Add files using upload-large-folder tool
Browse files- .gitattributes +8 -0
- LICENSE +29 -0
- LICENSE-QWEN-BASE +202 -0
- NOTICE +15 -0
- README.md +183 -0
- chat_template.jinja +154 -0
- config.json +108 -0
- generation_config.json +9 -0
- model-00001-of-00002.safetensors +3 -0
- model-00002-of-00002.safetensors +3 -0
- model.safetensors.index.json +626 -0
- modeling_marlin.py +533 -0
- preprocessor_config.json +21 -0
- processor_config.json +60 -0
- tokenizer.json +3 -0
- tokenizer_config.json +33 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,11 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
examples/caption_example.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
examples/find_example.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 39 |
+
examples/caption_example.jpg filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
examples/find_example.jpg filter=lfs diff=lfs merge=lfs -text
|
| 41 |
+
release_marlin_3up.png filter=lfs diff=lfs merge=lfs -text
|
| 42 |
+
examples/caption_example_test.png filter=lfs diff=lfs merge=lfs -text
|
| 43 |
+
examples/find_example_test.png filter=lfs diff=lfs merge=lfs -text
|
LICENSE
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
| License | Business Source License (BSL 1.1) |
|
| 2 |
+
| --- | --- |
|
| 3 |
+
| Licensor | NemoStation |
|
| 4 |
+
| Licensed Work | "Marlin 2B" including Model Weights and any Derivatives ("Derivatives" include fine-tunes, merges, quantizations, weight deltas, and other weight-level modifications or conversions.) |
|
| 5 |
+
| Additional Use Grant | You may make production use of the Licensed Work, provided Your use does not include offering the Licensed Work to third parties on a hosted or embedded basis in order to compete with NemoStation's paid version(s) of the Licensed Work. For purposes of this license:<br><br>A "competitive offering" is a Product that is offered to third parties on a paid basis, including through paid support arrangements, that significantly overlaps with the capabilities of NemoStation's paid version(s) of the Licensed Work. If Your Product is not a competitive offering when You first make it generally available, it will not become a competitive offering later due to NemoStation releasing a new version of the Licensed Work with additional capabilities. In addition, Products that are not provided on a paid basis are not competitive.<br><br>"Product" means software that is offered to end users to manage in their own environments or offered as a service on a hosted basis.<br><br>"Embedded" means including the source code or executable code from the Licensed Work in a competitive offering. "Embedded" also means packaging the competitive offering in such a way that the Licensed Work must be accessed or downloaded for the competitive offering to operate.<br><br>Hosting or using the Licensed Work(s) for internal purposes within an organization is not considered a competitive offering. NemoStation considers your organization to include all of your affiliates under common control. |
|
| 6 |
+
| Change Date | Two years after the first public release of this version of the Licensed Work |
|
| 7 |
+
| Change License | Apache License, Version 2.0 |
|
| 8 |
+
|
| 9 |
+
For information about alternative licensing arrangements for the Licensed Work, please contact [aryan@letsnemo.com](mailto:aryan@letsnemo.com).
|
| 10 |
+
|
| 11 |
+
The text of the Business Source License 1.1 follows. License text copyright (c) 2020 MariaDB Corporation Ab, All Rights Reserved. "Business Source License" is a trademark of MariaDB Corporation Ab.
|
| 12 |
+
|
| 13 |
+
## Terms
|
| 14 |
+
|
| 15 |
+
The Licensor hereby grants you the right to copy, modify, create derivative works, redistribute, and make non-production use of the Licensed Work. The Licensor may make an Additional Use Grant, above, permitting limited production use.
|
| 16 |
+
|
| 17 |
+
Effective on the Change Date, or the fourth anniversary of the first publicly available distribution of a specific version of the Licensed Work under this License, whichever comes first, the Licensor hereby grants you rights under the terms of the Change License, and the rights granted in the paragraph above terminate.
|
| 18 |
+
|
| 19 |
+
If your use of the Licensed Work does not comply with the requirements currently in effect as described in this License, you must purchase a commercial license from the Licensor, its affiliated entities, or authorized resellers, or you must refrain from using the Licensed Work.
|
| 20 |
+
|
| 21 |
+
All copies of the original and modified Licensed Work, and derivative works of the Licensed Work, are subject to this License. This License applies separately for each version of the Licensed Work and the Change Date may vary for each version of the Licensed Work released by Licensor.
|
| 22 |
+
|
| 23 |
+
You must conspicuously display this License on each original or modified copy of the Licensed Work. If you receive the Licensed Work in original or modified form from a third party, the terms and conditions set forth in this License apply to your use of that work.
|
| 24 |
+
|
| 25 |
+
Any use of the Licensed Work in violation of this License will automatically terminate your rights under this License for the current and all other versions of the Licensed Work.
|
| 26 |
+
|
| 27 |
+
This License does not grant you any right in any trademark or logo of Licensor or its affiliates (provided that you may use a trademark or logo of Licensor as expressly required by this License).
|
| 28 |
+
|
| 29 |
+
TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON AN "AS IS" BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND TITLE.
|
LICENSE-QWEN-BASE
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
Apache License
|
| 3 |
+
Version 2.0, January 2004
|
| 4 |
+
http://www.apache.org/licenses/
|
| 5 |
+
|
| 6 |
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
| 7 |
+
|
| 8 |
+
1. Definitions.
|
| 9 |
+
|
| 10 |
+
"License" shall mean the terms and conditions for use, reproduction,
|
| 11 |
+
and distribution as defined by Sections 1 through 9 of this document.
|
| 12 |
+
|
| 13 |
+
"Licensor" shall mean the copyright owner or entity authorized by
|
| 14 |
+
the copyright owner that is granting the License.
|
| 15 |
+
|
| 16 |
+
"Legal Entity" shall mean the union of the acting entity and all
|
| 17 |
+
other entities that control, are controlled by, or are under common
|
| 18 |
+
control with that entity. For the purposes of this definition,
|
| 19 |
+
"control" means (i) the power, direct or indirect, to cause the
|
| 20 |
+
direction or management of such entity, whether by contract or
|
| 21 |
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
| 22 |
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
| 23 |
+
|
| 24 |
+
"You" (or "Your") shall mean an individual or Legal Entity
|
| 25 |
+
exercising permissions granted by this License.
|
| 26 |
+
|
| 27 |
+
"Source" form shall mean the preferred form for making modifications,
|
| 28 |
+
including but not limited to software source code, documentation
|
| 29 |
+
source, and configuration files.
|
| 30 |
+
|
| 31 |
+
"Object" form shall mean any form resulting from mechanical
|
| 32 |
+
transformation or translation of a Source form, including but
|
| 33 |
+
not limited to compiled object code, generated documentation,
|
| 34 |
+
and conversions to other media types.
|
| 35 |
+
|
| 36 |
+
"Work" shall mean the work of authorship, whether in Source or
|
| 37 |
+
Object form, made available under the License, as indicated by a
|
| 38 |
+
copyright notice that is included in or attached to the work
|
| 39 |
+
(an example is provided in the Appendix below).
|
| 40 |
+
|
| 41 |
+
"Derivative Works" shall mean any work, whether in Source or Object
|
| 42 |
+
form, that is based on (or derived from) the Work and for which the
|
| 43 |
+
editorial revisions, annotations, elaborations, or other modifications
|
| 44 |
+
represent, as a whole, an original work of authorship. For the purposes
|
| 45 |
+
of this License, Derivative Works shall not include works that remain
|
| 46 |
+
separable from, or merely link (or bind by name) to the interfaces of,
|
| 47 |
+
the Work and Derivative Works thereof.
|
| 48 |
+
|
| 49 |
+
"Contribution" shall mean any work of authorship, including
|
| 50 |
+
the original version of the Work and any modifications or additions
|
| 51 |
+
to that Work or Derivative Works thereof, that is intentionally
|
| 52 |
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
| 53 |
+
or by an individual or Legal Entity authorized to submit on behalf of
|
| 54 |
+
the copyright owner. For the purposes of this definition, "submitted"
|
| 55 |
+
means any form of electronic, verbal, or written communication sent
|
| 56 |
+
to the Licensor or its representatives, including but not limited to
|
| 57 |
+
communication on electronic mailing lists, source code control systems,
|
| 58 |
+
and issue tracking systems that are managed by, or on behalf of, the
|
| 59 |
+
Licensor for the purpose of discussing and improving the Work, but
|
| 60 |
+
excluding communication that is conspicuously marked or otherwise
|
| 61 |
+
designated in writing by the copyright owner as "Not a Contribution."
|
| 62 |
+
|
| 63 |
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
| 64 |
+
on behalf of whom a Contribution has been received by Licensor and
|
| 65 |
+
subsequently incorporated within the Work.
|
| 66 |
+
|
| 67 |
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
| 68 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 69 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 70 |
+
copyright license to reproduce, prepare Derivative Works of,
|
| 71 |
+
publicly display, publicly perform, sublicense, and distribute the
|
| 72 |
+
Work and such Derivative Works in Source or Object form.
|
| 73 |
+
|
| 74 |
+
3. Grant of Patent License. Subject to the terms and conditions of
|
| 75 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 76 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 77 |
+
(except as stated in this section) patent license to make, have made,
|
| 78 |
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 79 |
+
where such license applies only to those patent claims licensable
|
| 80 |
+
by such Contributor that are necessarily infringed by their
|
| 81 |
+
Contribution(s) alone or by combination of their Contribution(s)
|
| 82 |
+
with the Work to which such Contribution(s) was submitted. If You
|
| 83 |
+
institute patent litigation against any entity (including a
|
| 84 |
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
| 85 |
+
or a Contribution incorporated within the Work constitutes direct
|
| 86 |
+
or contributory patent infringement, then any patent licenses
|
| 87 |
+
granted to You under this License for that Work shall terminate
|
| 88 |
+
as of the date such litigation is filed.
|
| 89 |
+
|
| 90 |
+
4. Redistribution. You may reproduce and distribute copies of the
|
| 91 |
+
Work or Derivative Works thereof in any medium, with or without
|
| 92 |
+
modifications, and in Source or Object form, provided that You
|
| 93 |
+
meet the following conditions:
|
| 94 |
+
|
| 95 |
+
(a) You must give any other recipients of the Work or
|
| 96 |
+
Derivative Works a copy of this License; and
|
| 97 |
+
|
| 98 |
+
(b) You must cause any modified files to carry prominent notices
|
| 99 |
+
stating that You changed the files; and
|
| 100 |
+
|
| 101 |
+
(c) You must retain, in the Source form of any Derivative Works
|
| 102 |
+
that You distribute, all copyright, patent, trademark, and
|
| 103 |
+
attribution notices from the Source form of the Work,
|
| 104 |
+
excluding those notices that do not pertain to any part of
|
| 105 |
+
the Derivative Works; and
|
| 106 |
+
|
| 107 |
+
(d) If the Work includes a "NOTICE" text file as part of its
|
| 108 |
+
distribution, then any Derivative Works that You distribute must
|
| 109 |
+
include a readable copy of the attribution notices contained
|
| 110 |
+
within such NOTICE file, excluding those notices that do not
|
| 111 |
+
pertain to any part of the Derivative Works, in at least one
|
| 112 |
+
of the following places: within a NOTICE text file distributed
|
| 113 |
+
as part of the Derivative Works; within the Source form or
|
| 114 |
+
documentation, if provided along with the Derivative Works; or,
|
| 115 |
+
within a display generated by the Derivative Works, if and
|
| 116 |
+
wherever such third-party notices normally appear. The contents
|
| 117 |
+
of the NOTICE file are for informational purposes only and
|
| 118 |
+
do not modify the License. You may add Your own attribution
|
| 119 |
+
notices within Derivative Works that You distribute, alongside
|
| 120 |
+
or as an addendum to the NOTICE text from the Work, provided
|
| 121 |
+
that such additional attribution notices cannot be construed
|
| 122 |
+
as modifying the License.
|
| 123 |
+
|
| 124 |
+
You may add Your own copyright statement to Your modifications and
|
| 125 |
+
may provide additional or different license terms and conditions
|
| 126 |
+
for use, reproduction, or distribution of Your modifications, or
|
| 127 |
+
for any such Derivative Works as a whole, provided Your use,
|
| 128 |
+
reproduction, and distribution of the Work otherwise complies with
|
| 129 |
+
the conditions stated in this License.
|
| 130 |
+
|
| 131 |
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
| 132 |
+
any Contribution intentionally submitted for inclusion in the Work
|
| 133 |
+
by You to the Licensor shall be under the terms and conditions of
|
| 134 |
+
this License, without any additional terms or conditions.
|
| 135 |
+
Notwithstanding the above, nothing herein shall supersede or modify
|
| 136 |
+
the terms of any separate license agreement you may have executed
|
| 137 |
+
with Licensor regarding such Contributions.
|
| 138 |
+
|
| 139 |
+
6. Trademarks. This License does not grant permission to use the trade
|
| 140 |
+
names, trademarks, service marks, or product names of the Licensor,
|
| 141 |
+
except as required for reasonable and customary use in describing the
|
| 142 |
+
origin of the Work and reproducing the content of the NOTICE file.
|
| 143 |
+
|
| 144 |
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
| 145 |
+
agreed to in writing, Licensor provides the Work (and each
|
| 146 |
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
| 147 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
| 148 |
+
implied, including, without limitation, any warranties or conditions
|
| 149 |
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 150 |
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 151 |
+
appropriateness of using or redistributing the Work and assume any
|
| 152 |
+
risks associated with Your exercise of permissions under this License.
|
| 153 |
+
|
| 154 |
+
8. Limitation of Liability. In no event and under no legal theory,
|
| 155 |
+
whether in tort (including negligence), contract, or otherwise,
|
| 156 |
+
unless required by applicable law (such as deliberate and grossly
|
| 157 |
+
negligent acts) or agreed to in writing, shall any Contributor be
|
| 158 |
+
liable to You for damages, including any direct, indirect, special,
|
| 159 |
+
incidental, or consequential damages of any character arising as a
|
| 160 |
+
result of this License or out of the use or inability to use the
|
| 161 |
+
Work (including but not limited to damages for loss of goodwill,
|
| 162 |
+
work stoppage, computer failure or malfunction, or any and all
|
| 163 |
+
other commercial damages or losses), even if such Contributor
|
| 164 |
+
has been advised of the possibility of such damages.
|
| 165 |
+
|
| 166 |
+
9. Accepting Warranty or Additional Liability. While redistributing
|
| 167 |
+
the Work or Derivative Works thereof, You may choose to offer,
|
| 168 |
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
| 169 |
+
or other liability obligations and/or rights consistent with this
|
| 170 |
+
License. However, in accepting such obligations, You may act only
|
| 171 |
+
on Your own behalf and on Your sole responsibility, not on behalf
|
| 172 |
+
of any other Contributor, and only if You agree to indemnify,
|
| 173 |
+
defend, and hold each Contributor harmless for any liability
|
| 174 |
+
incurred by, or claims asserted against, such Contributor by reason
|
| 175 |
+
of your accepting any such warranty or additional liability.
|
| 176 |
+
|
| 177 |
+
END OF TERMS AND CONDITIONS
|
| 178 |
+
|
| 179 |
+
APPENDIX: How to apply the Apache License to your work.
|
| 180 |
+
|
| 181 |
+
To apply the Apache License to your work, attach the following
|
| 182 |
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
| 183 |
+
replaced with your own identifying information. (Don't include
|
| 184 |
+
the brackets!) The text should be enclosed in the appropriate
|
| 185 |
+
comment syntax for the file format. We also recommend that a
|
| 186 |
+
file or class name and description of purpose be included on the
|
| 187 |
+
same "printed page" as the copyright notice for easier
|
| 188 |
+
identification within third-party archives.
|
| 189 |
+
|
| 190 |
+
Copyright 2026 Alibaba Cloud
|
| 191 |
+
|
| 192 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 193 |
+
you may not use this file except in compliance with the License.
|
| 194 |
+
You may obtain a copy of the License at
|
| 195 |
+
|
| 196 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 197 |
+
|
| 198 |
+
Unless required by applicable law or agreed to in writing, software
|
| 199 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 200 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 201 |
+
See the License for the specific language governing permissions and
|
| 202 |
+
limitations under the License.
|
NOTICE
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Marlin 2B
|
| 2 |
+
Copyright (c) 2026 NemoStation
|
| 3 |
+
|
| 4 |
+
This product includes weights derived from Qwen3.5-2B
|
| 5 |
+
(https://huggingface.co/Qwen/Qwen3.5-2B), Copyright (c) 2025 Alibaba Cloud,
|
| 6 |
+
used under the Apache License, Version 2.0 (see LICENSE-QWEN-BASE).
|
| 7 |
+
|
| 8 |
+
Modifications by NemoStation include: integration of a video-capable
|
| 9 |
+
visual tower, custom training data curation (~400K clip-level annotations
|
| 10 |
+
with Gemini-3-Flash teacher distillation), two-stage SFT + SimPO
|
| 11 |
+
post-training, and custom modeling code (modeling_marlin.py) exposing
|
| 12 |
+
the .caption() and .find() inference modes.
|
| 13 |
+
|
| 14 |
+
Marlin 2B and the modifications listed above are licensed under the
|
| 15 |
+
Business Source License 1.1 (see LICENSE).
|
README.md
ADDED
|
@@ -0,0 +1,183 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: other
|
| 3 |
+
license_name: bsl-1.1
|
| 4 |
+
license_link: LICENSE
|
| 5 |
+
language:
|
| 6 |
+
- en
|
| 7 |
+
base_model: Qwen/Qwen3.5-2B
|
| 8 |
+
pipeline_tag: video-text-to-text
|
| 9 |
+
library_name: transformers
|
| 10 |
+
tags:
|
| 11 |
+
- video
|
| 12 |
+
- multimodal
|
| 13 |
+
- video-captioning
|
| 14 |
+
- temporal-grounding
|
| 15 |
+
- qwen
|
| 16 |
+
- text-generation
|
| 17 |
+
- VLM
|
| 18 |
+
---
|
| 19 |
+
|
| 20 |
+
# video-scan
|
| 21 |
+
|
| 22 |
+
A 2B-parameter video VLM for dense captioning and natural-language temporal grounding. Given a video, it produces structured Scene + Event captions with second-precise timestamps, and resolves natural-language queries to `(start, end)` time spans in the video.
|
| 23 |
+
|
| 24 |
+
This repository is a redistribution of [`NemoStation/Marlin-2B`](https://huggingface.co/NemoStation/Marlin-2B) packaged for internal use. Weights are unmodified. Licensed under the Business Source License 1.1 — see [`LICENSE`](LICENSE) and [`NOTICE`](NOTICE) for terms and attribution. The internal Python module name (`modeling_marlin.py`) and class name (`MarlinForConditionalGeneration`) are preserved verbatim so that `trust_remote_code=True` loading via `auto_map` continues to work without modification.
|
| 25 |
+
|
| 26 |
+
## Capabilities
|
| 27 |
+
|
| 28 |
+
- **Caption mode**: returns `Scene: <paragraph>` followed by `Events: <X.X - Y.Y> <description>` lines.
|
| 29 |
+
- **Find mode**: given a natural-language event description, returns the matching time span as `From X.X to Y.Y.`.
|
| 30 |
+
- **Multichunk reasoning** (limited): `<think>`-style chunked-video reasoning with chunk-time to source-time arithmetic. Not exposed through the `.caption()` / `.find()` helpers — use a raw prompt to access it.
|
| 31 |
+
|
| 32 |
+
## Architecture
|
| 33 |
+
|
| 34 |
+
Fine-tune of Qwen3.5-2B with the video-capable visual tower kept intact. Custom modeling code in `modeling_marlin.py` exposes two convenience methods (`.caption()` and `.find()`) that wrap a single canonical training prompt per mode and parse the structured output into typed Python dicts. Raw `.generate()` is also available for custom prompts.
|
| 35 |
+
|
| 36 |
+
| Component | Value |
|
| 37 |
+
|---|---|
|
| 38 |
+
| Base model | Qwen/Qwen3.5-2B |
|
| 39 |
+
| Parameters | 2.21B (text + vision combined) |
|
| 40 |
+
| Precision | bfloat16 |
|
| 41 |
+
| Storage on disk | ~5.5 GB |
|
| 42 |
+
| Architecture string | `MarlinForConditionalGeneration` |
|
| 43 |
+
| `model_type` | `qwen3_5` |
|
| 44 |
+
| Context length | 262144 tokens |
|
| 45 |
+
|
| 46 |
+
## Training (upstream)
|
| 47 |
+
|
| 48 |
+
The following describes the upstream training pipeline as documented by NemoStation. We have not retrained or modified the weights.
|
| 49 |
+
|
| 50 |
+
- **Data**: ~400K clip-level annotations assembled from ActivityNet, LSMDC, Charades, Charades-Ego, TREC-VTT, WebVid-10M, HC-STVG, VidSTG, and TimeLens, with dense re-annotations distilled from Gemini-3-Flash and targeted human review on the highest-impact splits.
|
| 51 |
+
- **Stage 1**: supervised fine-tuning on the curated corpus with a fixed canonical prompt per mode and Tarsier-schema output formatting.
|
| 52 |
+
- **Stage 2**: SimPO (Simple Preference Optimization) on a teacher-distilled preference set, scored against Gemini-3-Flash on factual accuracy, completeness, and temporal alignment.
|
| 53 |
+
- **Hardware**: single H100.
|
| 54 |
+
|
| 55 |
+
## Evaluation (upstream-reported)
|
| 56 |
+
|
| 57 |
+
Upstream benchmarks Marlin-2B on three suites:
|
| 58 |
+
|
| 59 |
+
- **CaReBench** — [arXiv:2501.00513](https://arxiv.org/abs/2501.00513)
|
| 60 |
+
- **DREAM-1K** — [arXiv:2407.00634](https://arxiv.org/abs/2407.00634)
|
| 61 |
+
- **TimeLens-Bench** — [arXiv:2512.14698](https://arxiv.org/abs/2512.14698)
|
| 62 |
+
|
| 63 |
+
Headline numbers reported by NemoStation: tops the CaReBench leaderboard at the 2B scale, +6.4 mIoU over Qwen2.5-VL-7B on TimeLens-Bench (Charades / ActivityNet / QVHighlights). These numbers have not been independently re-verified in this repository.
|
| 64 |
+
|
| 65 |
+
## Quickstart
|
| 66 |
+
|
| 67 |
+
```python
|
| 68 |
+
import torch
|
| 69 |
+
from transformers import AutoModelForCausalLM
|
| 70 |
+
|
| 71 |
+
model = AutoModelForCausalLM.from_pretrained(
|
| 72 |
+
"cudabenchmarktest/video-scan",
|
| 73 |
+
trust_remote_code=True,
|
| 74 |
+
dtype=torch.bfloat16,
|
| 75 |
+
device_map={"": "cuda"},
|
| 76 |
+
)
|
| 77 |
+
model.compile() # optional — wraps torch.compile, faster after first call
|
| 78 |
+
```
|
| 79 |
+
|
| 80 |
+
### Caption mode
|
| 81 |
+
|
| 82 |
+
```python
|
| 83 |
+
result = model.caption("video.mp4")
|
| 84 |
+
|
| 85 |
+
print(result["caption"]) # full raw caption text (Scene: ... Events: ...)
|
| 86 |
+
print(result["scene"]) # parsed Scene paragraph
|
| 87 |
+
for ev in result["events"]:
|
| 88 |
+
print(f"<{ev['start']:.1f} - {ev['end']:.1f}> {ev['description']}")
|
| 89 |
+
```
|
| 90 |
+
|
| 91 |
+
Optional kwargs:
|
| 92 |
+
|
| 93 |
+
- `max_new_tokens=2048` — generation token cap (default).
|
| 94 |
+
- `prompt=None` — override the canonical training prompt. Almost always leave as `None`.
|
| 95 |
+
- `do_sample=False`, `temperature=1.0`, `top_p=1.0` — sampling controls.
|
| 96 |
+
|
| 97 |
+
### Find mode
|
| 98 |
+
|
| 99 |
+
```python
|
| 100 |
+
result = model.find("video.mp4", event="a person enters the room")
|
| 101 |
+
|
| 102 |
+
print(result["raw"]) # "From 14.3 to 18.2." raw model output
|
| 103 |
+
print(result["span"]) # (14.3, 18.2) tuple in seconds, or None on parse failure
|
| 104 |
+
print(result["format_ok"]) # True if output matched the trained format
|
| 105 |
+
```
|
| 106 |
+
|
| 107 |
+
## Raw inference
|
| 108 |
+
|
| 109 |
+
To bypass the helper methods and call `generate()` directly:
|
| 110 |
+
|
| 111 |
+
```python
|
| 112 |
+
import torch
|
| 113 |
+
from transformers import AutoModelForCausalLM, AutoProcessor
|
| 114 |
+
|
| 115 |
+
model = AutoModelForCausalLM.from_pretrained(
|
| 116 |
+
"cudabenchmarktest/video-scan",
|
| 117 |
+
trust_remote_code=True,
|
| 118 |
+
dtype=torch.bfloat16,
|
| 119 |
+
device_map={"": "cuda"},
|
| 120 |
+
)
|
| 121 |
+
processor = AutoProcessor.from_pretrained(
|
| 122 |
+
"cudabenchmarktest/video-scan", trust_remote_code=True
|
| 123 |
+
)
|
| 124 |
+
|
| 125 |
+
messages = [{"role": "user", "content": [
|
| 126 |
+
{"type": "video", "video": "video.mp4"},
|
| 127 |
+
{"type": "text", "text": "Your custom prompt here"},
|
| 128 |
+
]}]
|
| 129 |
+
inputs = processor.apply_chat_template(
|
| 130 |
+
messages, tokenize=True, add_generation_prompt=True,
|
| 131 |
+
return_tensors="pt", return_dict=True,
|
| 132 |
+
).to(model.device)
|
| 133 |
+
|
| 134 |
+
with torch.inference_mode():
|
| 135 |
+
out = model.generate(**inputs, max_new_tokens=512, do_sample=False)
|
| 136 |
+
out = out[:, inputs["input_ids"].shape[1]:]
|
| 137 |
+
text = processor.batch_decode(out, skip_special_tokens=True)[0]
|
| 138 |
+
print(text)
|
| 139 |
+
```
|
| 140 |
+
|
| 141 |
+
## Output format notes
|
| 142 |
+
|
| 143 |
+
The model emits a `<think>` token at the start of every response (an artifact of training with `add_non_thinking_prefix=True`). The `.caption()` and `.find()` helpers strip this automatically. When calling `generate()` directly, strip any leading `<think>...</think>` block (with or without closing tag) from the output before parsing.
|
| 144 |
+
|
| 145 |
+
## Requirements
|
| 146 |
+
|
| 147 |
+
- `transformers >= 5.7.0` (for native `qwen3_5` architecture)
|
| 148 |
+
- `torch >= 2.11.0`
|
| 149 |
+
- `torchcodec` (video decoding)
|
| 150 |
+
- `qwen-vl-utils >= 0.0.14`
|
| 151 |
+
- `av` (torchcodec system dependency)
|
| 152 |
+
- `pillow`
|
| 153 |
+
|
| 154 |
+
```bash
|
| 155 |
+
pip install "transformers>=5.7.0" "torch>=2.11.0" torchcodec "qwen-vl-utils>=0.0.14" av pillow
|
| 156 |
+
```
|
| 157 |
+
|
| 158 |
+
## Video preprocessing
|
| 159 |
+
|
| 160 |
+
The custom modeling code sets these environment variables internally to match the training-time setup. Override them in your shell **before** importing transformers if needed.
|
| 161 |
+
|
| 162 |
+
| Env var | Default | Purpose |
|
| 163 |
+
|---|---|---|
|
| 164 |
+
| `FORCE_QWENVL_VIDEO_READER` | `torchcodec` | Video decoder backend |
|
| 165 |
+
| `VIDEO_MAX_PIXELS` | `200704` | Max pixels per frame (~448x448) |
|
| 166 |
+
| `FPS` | `2.0` | Frame sampling rate |
|
| 167 |
+
| `FPS_MAX_FRAMES` | `240` | Cap on total frames (~2 min at 2 FPS) |
|
| 168 |
+
| `FPS_MIN_FRAMES` | `4` | Floor for very short videos |
|
| 169 |
+
|
| 170 |
+
## License and attribution
|
| 171 |
+
|
| 172 |
+
This redistribution is licensed under the **Business Source License 1.1**. The full license text is in [`LICENSE`](LICENSE). The Qwen3.5-2B base weights remain under Apache License 2.0 — see [`LICENSE-QWEN-BASE`](LICENSE-QWEN-BASE) and [`NOTICE`](NOTICE).
|
| 173 |
+
|
| 174 |
+
Key terms of BSL 1.1 as applied here:
|
| 175 |
+
|
| 176 |
+
- Copy, modify, redistribute, and non-production use are permitted.
|
| 177 |
+
- Production use is permitted **except** for offering this work to third parties on a hosted or embedded basis in a way that competes with NemoStation's paid version(s).
|
| 178 |
+
- Internal organizational use is explicitly not a competitive offering.
|
| 179 |
+
- On the **Change Date** (two years after upstream public release), the license converts to Apache License 2.0.
|
| 180 |
+
|
| 181 |
+
The "Marlin" name and any logos are trademarks of NemoStation and are not granted by this license. The class identifier `MarlinForConditionalGeneration` and the module name `modeling_marlin.py` are preserved only because `auto_map` requires them for `trust_remote_code` loading; they do not imply trademark use beyond technical interoperability.
|
| 182 |
+
|
| 183 |
+
Upstream source: <https://huggingface.co/NemoStation/Marlin-2B>
|
chat_template.jinja
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{%- set image_count = namespace(value=0) %}
|
| 2 |
+
{%- set video_count = namespace(value=0) %}
|
| 3 |
+
{%- macro render_content(content, do_vision_count, is_system_content=false) %}
|
| 4 |
+
{%- if content is string %}
|
| 5 |
+
{{- content }}
|
| 6 |
+
{%- elif content is iterable and content is not mapping %}
|
| 7 |
+
{%- for item in content %}
|
| 8 |
+
{%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
|
| 9 |
+
{%- if is_system_content %}
|
| 10 |
+
{{- raise_exception('System message cannot contain images.') }}
|
| 11 |
+
{%- endif %}
|
| 12 |
+
{%- if do_vision_count %}
|
| 13 |
+
{%- set image_count.value = image_count.value + 1 %}
|
| 14 |
+
{%- endif %}
|
| 15 |
+
{%- if add_vision_id %}
|
| 16 |
+
{{- 'Picture ' ~ image_count.value ~ ': ' }}
|
| 17 |
+
{%- endif %}
|
| 18 |
+
{{- '<|vision_start|><|image_pad|><|vision_end|>' }}
|
| 19 |
+
{%- elif 'video' in item or item.type == 'video' %}
|
| 20 |
+
{%- if is_system_content %}
|
| 21 |
+
{{- raise_exception('System message cannot contain videos.') }}
|
| 22 |
+
{%- endif %}
|
| 23 |
+
{%- if do_vision_count %}
|
| 24 |
+
{%- set video_count.value = video_count.value + 1 %}
|
| 25 |
+
{%- endif %}
|
| 26 |
+
{%- if add_vision_id %}
|
| 27 |
+
{{- 'Video ' ~ video_count.value ~ ': ' }}
|
| 28 |
+
{%- endif %}
|
| 29 |
+
{{- '<|vision_start|><|video_pad|><|vision_end|>' }}
|
| 30 |
+
{%- elif 'text' in item %}
|
| 31 |
+
{{- item.text }}
|
| 32 |
+
{%- else %}
|
| 33 |
+
{{- raise_exception('Unexpected item type in content.') }}
|
| 34 |
+
{%- endif %}
|
| 35 |
+
{%- endfor %}
|
| 36 |
+
{%- elif content is none or content is undefined %}
|
| 37 |
+
{{- '' }}
|
| 38 |
+
{%- else %}
|
| 39 |
+
{{- raise_exception('Unexpected content type.') }}
|
| 40 |
+
{%- endif %}
|
| 41 |
+
{%- endmacro %}
|
| 42 |
+
{%- if not messages %}
|
| 43 |
+
{{- raise_exception('No messages provided.') }}
|
| 44 |
+
{%- endif %}
|
| 45 |
+
{%- if tools and tools is iterable and tools is not mapping %}
|
| 46 |
+
{{- '<|im_start|>system\n' }}
|
| 47 |
+
{{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
|
| 48 |
+
{%- for tool in tools %}
|
| 49 |
+
{{- "\n" }}
|
| 50 |
+
{{- tool | tojson }}
|
| 51 |
+
{%- endfor %}
|
| 52 |
+
{{- "\n</tools>" }}
|
| 53 |
+
{{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
|
| 54 |
+
{%- if messages[0].role == 'system' %}
|
| 55 |
+
{%- set content = render_content(messages[0].content, false, true)|trim %}
|
| 56 |
+
{%- if content %}
|
| 57 |
+
{{- '\n\n' + content }}
|
| 58 |
+
{%- endif %}
|
| 59 |
+
{%- endif %}
|
| 60 |
+
{{- '<|im_end|>\n' }}
|
| 61 |
+
{%- else %}
|
| 62 |
+
{%- if messages[0].role == 'system' %}
|
| 63 |
+
{%- set content = render_content(messages[0].content, false, true)|trim %}
|
| 64 |
+
{{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
|
| 65 |
+
{%- endif %}
|
| 66 |
+
{%- endif %}
|
| 67 |
+
{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
|
| 68 |
+
{%- for message in messages[::-1] %}
|
| 69 |
+
{%- set index = (messages|length - 1) - loop.index0 %}
|
| 70 |
+
{%- if ns.multi_step_tool and message.role == "user" %}
|
| 71 |
+
{%- set content = render_content(message.content, false)|trim %}
|
| 72 |
+
{%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
|
| 73 |
+
{%- set ns.multi_step_tool = false %}
|
| 74 |
+
{%- set ns.last_query_index = index %}
|
| 75 |
+
{%- endif %}
|
| 76 |
+
{%- endif %}
|
| 77 |
+
{%- endfor %}
|
| 78 |
+
{%- if ns.multi_step_tool %}
|
| 79 |
+
{{- raise_exception('No user query found in messages.') }}
|
| 80 |
+
{%- endif %}
|
| 81 |
+
{%- for message in messages %}
|
| 82 |
+
{%- set content = render_content(message.content, true)|trim %}
|
| 83 |
+
{%- if message.role == "system" %}
|
| 84 |
+
{%- if not loop.first %}
|
| 85 |
+
{{- raise_exception('System message must be at the beginning.') }}
|
| 86 |
+
{%- endif %}
|
| 87 |
+
{%- elif message.role == "user" %}
|
| 88 |
+
{{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
|
| 89 |
+
{%- elif message.role == "assistant" %}
|
| 90 |
+
{%- set reasoning_content = '' %}
|
| 91 |
+
{%- if message.reasoning_content is string %}
|
| 92 |
+
{%- set reasoning_content = message.reasoning_content %}
|
| 93 |
+
{%- else %}
|
| 94 |
+
{%- if '</think>' in content %}
|
| 95 |
+
{%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
|
| 96 |
+
{%- set content = content.split('</think>')[-1].lstrip('\n') %}
|
| 97 |
+
{%- endif %}
|
| 98 |
+
{%- endif %}
|
| 99 |
+
{%- set reasoning_content = reasoning_content|trim %}
|
| 100 |
+
{%- if loop.index0 > ns.last_query_index %}
|
| 101 |
+
{{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
|
| 102 |
+
{%- else %}
|
| 103 |
+
{{- '<|im_start|>' + message.role + '\n' + content }}
|
| 104 |
+
{%- endif %}
|
| 105 |
+
{%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
|
| 106 |
+
{%- for tool_call in message.tool_calls %}
|
| 107 |
+
{%- if tool_call.function is defined %}
|
| 108 |
+
{%- set tool_call = tool_call.function %}
|
| 109 |
+
{%- endif %}
|
| 110 |
+
{%- if loop.first %}
|
| 111 |
+
{%- if content|trim %}
|
| 112 |
+
{{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
|
| 113 |
+
{%- else %}
|
| 114 |
+
{{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
|
| 115 |
+
{%- endif %}
|
| 116 |
+
{%- else %}
|
| 117 |
+
{{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
|
| 118 |
+
{%- endif %}
|
| 119 |
+
{%- if tool_call.arguments is defined %}
|
| 120 |
+
{%- for args_name, args_value in tool_call.arguments|items %}
|
| 121 |
+
{{- '<parameter=' + args_name + '>\n' }}
|
| 122 |
+
{%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
|
| 123 |
+
{{- args_value }}
|
| 124 |
+
{{- '\n</parameter>\n' }}
|
| 125 |
+
{%- endfor %}
|
| 126 |
+
{%- endif %}
|
| 127 |
+
{{- '</function>\n</tool_call>' }}
|
| 128 |
+
{%- endfor %}
|
| 129 |
+
{%- endif %}
|
| 130 |
+
{{- '<|im_end|>\n' }}
|
| 131 |
+
{%- elif message.role == "tool" %}
|
| 132 |
+
{%- if loop.previtem and loop.previtem.role != "tool" %}
|
| 133 |
+
{{- '<|im_start|>user' }}
|
| 134 |
+
{%- endif %}
|
| 135 |
+
{{- '\n<tool_response>\n' }}
|
| 136 |
+
{{- content }}
|
| 137 |
+
{{- '\n</tool_response>' }}
|
| 138 |
+
{%- if not loop.last and loop.nextitem.role != "tool" %}
|
| 139 |
+
{{- '<|im_end|>\n' }}
|
| 140 |
+
{%- elif loop.last %}
|
| 141 |
+
{{- '<|im_end|>\n' }}
|
| 142 |
+
{%- endif %}
|
| 143 |
+
{%- else %}
|
| 144 |
+
{{- raise_exception('Unexpected message role.') }}
|
| 145 |
+
{%- endif %}
|
| 146 |
+
{%- endfor %}
|
| 147 |
+
{%- if add_generation_prompt %}
|
| 148 |
+
{{- '<|im_start|>assistant\n' }}
|
| 149 |
+
{%- if enable_thinking is defined and enable_thinking is true %}
|
| 150 |
+
{{- '<think>\n' }}
|
| 151 |
+
{%- else %}
|
| 152 |
+
{{- '<think>\n\n</think>\n\n' }}
|
| 153 |
+
{%- endif %}
|
| 154 |
+
{%- endif %}
|
config.json
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"MarlinForConditionalGeneration"
|
| 4 |
+
],
|
| 5 |
+
"dtype": "bfloat16",
|
| 6 |
+
"eos_token_id": 248046,
|
| 7 |
+
"hidden_size": 2048,
|
| 8 |
+
"image_token_id": 248056,
|
| 9 |
+
"model_type": "qwen3_5",
|
| 10 |
+
"pad_token_id": 248044,
|
| 11 |
+
"text_config": {
|
| 12 |
+
"attention_bias": false,
|
| 13 |
+
"attention_dropout": 0.0,
|
| 14 |
+
"attn_output_gate": true,
|
| 15 |
+
"bos_token_id": null,
|
| 16 |
+
"dtype": "bfloat16",
|
| 17 |
+
"eos_token_id": 248044,
|
| 18 |
+
"full_attention_interval": 4,
|
| 19 |
+
"head_dim": 256,
|
| 20 |
+
"hidden_act": "silu",
|
| 21 |
+
"hidden_size": 2048,
|
| 22 |
+
"initializer_range": 0.02,
|
| 23 |
+
"intermediate_size": 6144,
|
| 24 |
+
"layer_types": [
|
| 25 |
+
"linear_attention",
|
| 26 |
+
"linear_attention",
|
| 27 |
+
"linear_attention",
|
| 28 |
+
"full_attention",
|
| 29 |
+
"linear_attention",
|
| 30 |
+
"linear_attention",
|
| 31 |
+
"linear_attention",
|
| 32 |
+
"full_attention",
|
| 33 |
+
"linear_attention",
|
| 34 |
+
"linear_attention",
|
| 35 |
+
"linear_attention",
|
| 36 |
+
"full_attention",
|
| 37 |
+
"linear_attention",
|
| 38 |
+
"linear_attention",
|
| 39 |
+
"linear_attention",
|
| 40 |
+
"full_attention",
|
| 41 |
+
"linear_attention",
|
| 42 |
+
"linear_attention",
|
| 43 |
+
"linear_attention",
|
| 44 |
+
"full_attention",
|
| 45 |
+
"linear_attention",
|
| 46 |
+
"linear_attention",
|
| 47 |
+
"linear_attention",
|
| 48 |
+
"full_attention"
|
| 49 |
+
],
|
| 50 |
+
"linear_conv_kernel_dim": 4,
|
| 51 |
+
"linear_key_head_dim": 128,
|
| 52 |
+
"linear_num_key_heads": 16,
|
| 53 |
+
"linear_num_value_heads": 16,
|
| 54 |
+
"linear_value_head_dim": 128,
|
| 55 |
+
"mamba_ssm_dtype": "float32",
|
| 56 |
+
"max_position_embeddings": 262144,
|
| 57 |
+
"mlp_only_layers": [],
|
| 58 |
+
"model_type": "qwen3_5_text",
|
| 59 |
+
"mtp_num_hidden_layers": 1,
|
| 60 |
+
"mtp_use_dedicated_embeddings": false,
|
| 61 |
+
"num_attention_heads": 8,
|
| 62 |
+
"num_hidden_layers": 24,
|
| 63 |
+
"num_key_value_heads": 2,
|
| 64 |
+
"pad_token_id": 248044,
|
| 65 |
+
"partial_rotary_factor": 0.25,
|
| 66 |
+
"rms_norm_eps": 1e-06,
|
| 67 |
+
"rope_parameters": {
|
| 68 |
+
"mrope_interleaved": true,
|
| 69 |
+
"mrope_section": [
|
| 70 |
+
11,
|
| 71 |
+
11,
|
| 72 |
+
10
|
| 73 |
+
],
|
| 74 |
+
"partial_rotary_factor": 0.25,
|
| 75 |
+
"rope_theta": 10000000,
|
| 76 |
+
"rope_type": "default"
|
| 77 |
+
},
|
| 78 |
+
"tie_word_embeddings": true,
|
| 79 |
+
"use_cache": false,
|
| 80 |
+
"vocab_size": 248320
|
| 81 |
+
},
|
| 82 |
+
"tie_word_embeddings": true,
|
| 83 |
+
"transformers_version": "5.7.0",
|
| 84 |
+
"use_cache": false,
|
| 85 |
+
"video_token_id": 248057,
|
| 86 |
+
"vision_config": {
|
| 87 |
+
"deepstack_visual_indexes": [],
|
| 88 |
+
"depth": 24,
|
| 89 |
+
"dtype": "bfloat16",
|
| 90 |
+
"hidden_act": "gelu_pytorch_tanh",
|
| 91 |
+
"hidden_size": 1024,
|
| 92 |
+
"in_channels": 3,
|
| 93 |
+
"initializer_range": 0.02,
|
| 94 |
+
"intermediate_size": 4096,
|
| 95 |
+
"model_type": "qwen3_5_vision",
|
| 96 |
+
"num_heads": 16,
|
| 97 |
+
"num_position_embeddings": 2304,
|
| 98 |
+
"out_hidden_size": 2048,
|
| 99 |
+
"patch_size": 16,
|
| 100 |
+
"spatial_merge_size": 2,
|
| 101 |
+
"temporal_patch_size": 2
|
| 102 |
+
},
|
| 103 |
+
"vision_end_token_id": 248054,
|
| 104 |
+
"vision_start_token_id": 248053,
|
| 105 |
+
"auto_map": {
|
| 106 |
+
"AutoModelForCausalLM": "modeling_marlin.MarlinForConditionalGeneration"
|
| 107 |
+
}
|
| 108 |
+
}
|
generation_config.json
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"eos_token_id": [
|
| 4 |
+
248044,
|
| 5 |
+
248046
|
| 6 |
+
],
|
| 7 |
+
"transformers_version": "5.7.0",
|
| 8 |
+
"use_cache": true
|
| 9 |
+
}
|
model-00001-of-00002.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5d78fa4dbd856dc89c01b99ffa92072fe31b8a1e6b31e87893734c80304983b7
|
| 3 |
+
size 4999157736
|
model-00002-of-00002.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:01d40ec9ccf4c2ad8e755604468dd6ee4a5c6551553e5739a03beb4c0673d0db
|
| 3 |
+
size 444519488
|
model.safetensors.index.json
ADDED
|
@@ -0,0 +1,626 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"metadata": {
|
| 3 |
+
"total_parameters": 2213241664,
|
| 4 |
+
"total_size": 5443602048
|
| 5 |
+
},
|
| 6 |
+
"weight_map": {
|
| 7 |
+
"lm_head.weight": "model-00001-of-00002.safetensors",
|
| 8 |
+
"model.language_model.embed_tokens.weight": "model-00001-of-00002.safetensors",
|
| 9 |
+
"model.language_model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 10 |
+
"model.language_model.layers.0.linear_attn.A_log": "model-00001-of-00002.safetensors",
|
| 11 |
+
"model.language_model.layers.0.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors",
|
| 12 |
+
"model.language_model.layers.0.linear_attn.dt_bias": "model-00001-of-00002.safetensors",
|
| 13 |
+
"model.language_model.layers.0.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors",
|
| 14 |
+
"model.language_model.layers.0.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors",
|
| 15 |
+
"model.language_model.layers.0.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors",
|
| 16 |
+
"model.language_model.layers.0.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors",
|
| 17 |
+
"model.language_model.layers.0.linear_attn.norm.weight": "model-00001-of-00002.safetensors",
|
| 18 |
+
"model.language_model.layers.0.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors",
|
| 19 |
+
"model.language_model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 20 |
+
"model.language_model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
|
| 21 |
+
"model.language_model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
|
| 22 |
+
"model.language_model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 23 |
+
"model.language_model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 24 |
+
"model.language_model.layers.1.linear_attn.A_log": "model-00001-of-00002.safetensors",
|
| 25 |
+
"model.language_model.layers.1.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors",
|
| 26 |
+
"model.language_model.layers.1.linear_attn.dt_bias": "model-00001-of-00002.safetensors",
|
| 27 |
+
"model.language_model.layers.1.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors",
|
| 28 |
+
"model.language_model.layers.1.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors",
|
| 29 |
+
"model.language_model.layers.1.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors",
|
| 30 |
+
"model.language_model.layers.1.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors",
|
| 31 |
+
"model.language_model.layers.1.linear_attn.norm.weight": "model-00001-of-00002.safetensors",
|
| 32 |
+
"model.language_model.layers.1.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors",
|
| 33 |
+
"model.language_model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 34 |
+
"model.language_model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
|
| 35 |
+
"model.language_model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
|
| 36 |
+
"model.language_model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 37 |
+
"model.language_model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 38 |
+
"model.language_model.layers.10.linear_attn.A_log": "model-00001-of-00002.safetensors",
|
| 39 |
+
"model.language_model.layers.10.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors",
|
| 40 |
+
"model.language_model.layers.10.linear_attn.dt_bias": "model-00001-of-00002.safetensors",
|
| 41 |
+
"model.language_model.layers.10.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors",
|
| 42 |
+
"model.language_model.layers.10.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors",
|
| 43 |
+
"model.language_model.layers.10.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors",
|
| 44 |
+
"model.language_model.layers.10.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors",
|
| 45 |
+
"model.language_model.layers.10.linear_attn.norm.weight": "model-00001-of-00002.safetensors",
|
| 46 |
+
"model.language_model.layers.10.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors",
|
| 47 |
+
"model.language_model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 48 |
+
"model.language_model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
|
| 49 |
+
"model.language_model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
|
| 50 |
+
"model.language_model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 51 |
+
"model.language_model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 52 |
+
"model.language_model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 53 |
+
"model.language_model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
|
| 54 |
+
"model.language_model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
|
| 55 |
+
"model.language_model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 56 |
+
"model.language_model.layers.11.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
|
| 57 |
+
"model.language_model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
|
| 58 |
+
"model.language_model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
| 59 |
+
"model.language_model.layers.11.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
|
| 60 |
+
"model.language_model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
|
| 61 |
+
"model.language_model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
|
| 62 |
+
"model.language_model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 63 |
+
"model.language_model.layers.12.linear_attn.A_log": "model-00001-of-00002.safetensors",
|
| 64 |
+
"model.language_model.layers.12.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors",
|
| 65 |
+
"model.language_model.layers.12.linear_attn.dt_bias": "model-00001-of-00002.safetensors",
|
| 66 |
+
"model.language_model.layers.12.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors",
|
| 67 |
+
"model.language_model.layers.12.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors",
|
| 68 |
+
"model.language_model.layers.12.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors",
|
| 69 |
+
"model.language_model.layers.12.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors",
|
| 70 |
+
"model.language_model.layers.12.linear_attn.norm.weight": "model-00001-of-00002.safetensors",
|
| 71 |
+
"model.language_model.layers.12.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors",
|
| 72 |
+
"model.language_model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 73 |
+
"model.language_model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
|
| 74 |
+
"model.language_model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
|
| 75 |
+
"model.language_model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 76 |
+
"model.language_model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 77 |
+
"model.language_model.layers.13.linear_attn.A_log": "model-00001-of-00002.safetensors",
|
| 78 |
+
"model.language_model.layers.13.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors",
|
| 79 |
+
"model.language_model.layers.13.linear_attn.dt_bias": "model-00001-of-00002.safetensors",
|
| 80 |
+
"model.language_model.layers.13.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors",
|
| 81 |
+
"model.language_model.layers.13.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors",
|
| 82 |
+
"model.language_model.layers.13.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors",
|
| 83 |
+
"model.language_model.layers.13.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors",
|
| 84 |
+
"model.language_model.layers.13.linear_attn.norm.weight": "model-00001-of-00002.safetensors",
|
| 85 |
+
"model.language_model.layers.13.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors",
|
| 86 |
+
"model.language_model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 87 |
+
"model.language_model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
|
| 88 |
+
"model.language_model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
|
| 89 |
+
"model.language_model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 90 |
+
"model.language_model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 91 |
+
"model.language_model.layers.14.linear_attn.A_log": "model-00001-of-00002.safetensors",
|
| 92 |
+
"model.language_model.layers.14.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors",
|
| 93 |
+
"model.language_model.layers.14.linear_attn.dt_bias": "model-00001-of-00002.safetensors",
|
| 94 |
+
"model.language_model.layers.14.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors",
|
| 95 |
+
"model.language_model.layers.14.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors",
|
| 96 |
+
"model.language_model.layers.14.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors",
|
| 97 |
+
"model.language_model.layers.14.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors",
|
| 98 |
+
"model.language_model.layers.14.linear_attn.norm.weight": "model-00001-of-00002.safetensors",
|
| 99 |
+
"model.language_model.layers.14.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors",
|
| 100 |
+
"model.language_model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 101 |
+
"model.language_model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
|
| 102 |
+
"model.language_model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
|
| 103 |
+
"model.language_model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 104 |
+
"model.language_model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 105 |
+
"model.language_model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 106 |
+
"model.language_model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
|
| 107 |
+
"model.language_model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
|
| 108 |
+
"model.language_model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 109 |
+
"model.language_model.layers.15.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
|
| 110 |
+
"model.language_model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
|
| 111 |
+
"model.language_model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
| 112 |
+
"model.language_model.layers.15.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
|
| 113 |
+
"model.language_model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
|
| 114 |
+
"model.language_model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
|
| 115 |
+
"model.language_model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 116 |
+
"model.language_model.layers.16.linear_attn.A_log": "model-00001-of-00002.safetensors",
|
| 117 |
+
"model.language_model.layers.16.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors",
|
| 118 |
+
"model.language_model.layers.16.linear_attn.dt_bias": "model-00001-of-00002.safetensors",
|
| 119 |
+
"model.language_model.layers.16.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors",
|
| 120 |
+
"model.language_model.layers.16.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors",
|
| 121 |
+
"model.language_model.layers.16.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors",
|
| 122 |
+
"model.language_model.layers.16.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors",
|
| 123 |
+
"model.language_model.layers.16.linear_attn.norm.weight": "model-00001-of-00002.safetensors",
|
| 124 |
+
"model.language_model.layers.16.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors",
|
| 125 |
+
"model.language_model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 126 |
+
"model.language_model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
|
| 127 |
+
"model.language_model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
|
| 128 |
+
"model.language_model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 129 |
+
"model.language_model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 130 |
+
"model.language_model.layers.17.linear_attn.A_log": "model-00001-of-00002.safetensors",
|
| 131 |
+
"model.language_model.layers.17.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors",
|
| 132 |
+
"model.language_model.layers.17.linear_attn.dt_bias": "model-00001-of-00002.safetensors",
|
| 133 |
+
"model.language_model.layers.17.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors",
|
| 134 |
+
"model.language_model.layers.17.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors",
|
| 135 |
+
"model.language_model.layers.17.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors",
|
| 136 |
+
"model.language_model.layers.17.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors",
|
| 137 |
+
"model.language_model.layers.17.linear_attn.norm.weight": "model-00001-of-00002.safetensors",
|
| 138 |
+
"model.language_model.layers.17.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors",
|
| 139 |
+
"model.language_model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 140 |
+
"model.language_model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
|
| 141 |
+
"model.language_model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
|
| 142 |
+
"model.language_model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 143 |
+
"model.language_model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 144 |
+
"model.language_model.layers.18.linear_attn.A_log": "model-00001-of-00002.safetensors",
|
| 145 |
+
"model.language_model.layers.18.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors",
|
| 146 |
+
"model.language_model.layers.18.linear_attn.dt_bias": "model-00001-of-00002.safetensors",
|
| 147 |
+
"model.language_model.layers.18.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors",
|
| 148 |
+
"model.language_model.layers.18.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors",
|
| 149 |
+
"model.language_model.layers.18.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors",
|
| 150 |
+
"model.language_model.layers.18.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors",
|
| 151 |
+
"model.language_model.layers.18.linear_attn.norm.weight": "model-00001-of-00002.safetensors",
|
| 152 |
+
"model.language_model.layers.18.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors",
|
| 153 |
+
"model.language_model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 154 |
+
"model.language_model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
|
| 155 |
+
"model.language_model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
|
| 156 |
+
"model.language_model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 157 |
+
"model.language_model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 158 |
+
"model.language_model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 159 |
+
"model.language_model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
|
| 160 |
+
"model.language_model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
|
| 161 |
+
"model.language_model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 162 |
+
"model.language_model.layers.19.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
|
| 163 |
+
"model.language_model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
|
| 164 |
+
"model.language_model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
| 165 |
+
"model.language_model.layers.19.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
|
| 166 |
+
"model.language_model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
|
| 167 |
+
"model.language_model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
|
| 168 |
+
"model.language_model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 169 |
+
"model.language_model.layers.2.linear_attn.A_log": "model-00001-of-00002.safetensors",
|
| 170 |
+
"model.language_model.layers.2.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors",
|
| 171 |
+
"model.language_model.layers.2.linear_attn.dt_bias": "model-00001-of-00002.safetensors",
|
| 172 |
+
"model.language_model.layers.2.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors",
|
| 173 |
+
"model.language_model.layers.2.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors",
|
| 174 |
+
"model.language_model.layers.2.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors",
|
| 175 |
+
"model.language_model.layers.2.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors",
|
| 176 |
+
"model.language_model.layers.2.linear_attn.norm.weight": "model-00001-of-00002.safetensors",
|
| 177 |
+
"model.language_model.layers.2.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors",
|
| 178 |
+
"model.language_model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 179 |
+
"model.language_model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
|
| 180 |
+
"model.language_model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
|
| 181 |
+
"model.language_model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 182 |
+
"model.language_model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 183 |
+
"model.language_model.layers.20.linear_attn.A_log": "model-00001-of-00002.safetensors",
|
| 184 |
+
"model.language_model.layers.20.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors",
|
| 185 |
+
"model.language_model.layers.20.linear_attn.dt_bias": "model-00001-of-00002.safetensors",
|
| 186 |
+
"model.language_model.layers.20.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors",
|
| 187 |
+
"model.language_model.layers.20.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors",
|
| 188 |
+
"model.language_model.layers.20.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors",
|
| 189 |
+
"model.language_model.layers.20.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors",
|
| 190 |
+
"model.language_model.layers.20.linear_attn.norm.weight": "model-00001-of-00002.safetensors",
|
| 191 |
+
"model.language_model.layers.20.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors",
|
| 192 |
+
"model.language_model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 193 |
+
"model.language_model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
|
| 194 |
+
"model.language_model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
|
| 195 |
+
"model.language_model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 196 |
+
"model.language_model.layers.21.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 197 |
+
"model.language_model.layers.21.linear_attn.A_log": "model-00001-of-00002.safetensors",
|
| 198 |
+
"model.language_model.layers.21.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors",
|
| 199 |
+
"model.language_model.layers.21.linear_attn.dt_bias": "model-00001-of-00002.safetensors",
|
| 200 |
+
"model.language_model.layers.21.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors",
|
| 201 |
+
"model.language_model.layers.21.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors",
|
| 202 |
+
"model.language_model.layers.21.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors",
|
| 203 |
+
"model.language_model.layers.21.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors",
|
| 204 |
+
"model.language_model.layers.21.linear_attn.norm.weight": "model-00001-of-00002.safetensors",
|
| 205 |
+
"model.language_model.layers.21.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors",
|
| 206 |
+
"model.language_model.layers.21.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 207 |
+
"model.language_model.layers.21.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
|
| 208 |
+
"model.language_model.layers.21.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
|
| 209 |
+
"model.language_model.layers.21.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 210 |
+
"model.language_model.layers.22.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 211 |
+
"model.language_model.layers.22.linear_attn.A_log": "model-00001-of-00002.safetensors",
|
| 212 |
+
"model.language_model.layers.22.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors",
|
| 213 |
+
"model.language_model.layers.22.linear_attn.dt_bias": "model-00001-of-00002.safetensors",
|
| 214 |
+
"model.language_model.layers.22.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors",
|
| 215 |
+
"model.language_model.layers.22.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors",
|
| 216 |
+
"model.language_model.layers.22.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors",
|
| 217 |
+
"model.language_model.layers.22.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors",
|
| 218 |
+
"model.language_model.layers.22.linear_attn.norm.weight": "model-00001-of-00002.safetensors",
|
| 219 |
+
"model.language_model.layers.22.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors",
|
| 220 |
+
"model.language_model.layers.22.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 221 |
+
"model.language_model.layers.22.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
|
| 222 |
+
"model.language_model.layers.22.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
|
| 223 |
+
"model.language_model.layers.22.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 224 |
+
"model.language_model.layers.23.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 225 |
+
"model.language_model.layers.23.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 226 |
+
"model.language_model.layers.23.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
|
| 227 |
+
"model.language_model.layers.23.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
|
| 228 |
+
"model.language_model.layers.23.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 229 |
+
"model.language_model.layers.23.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
|
| 230 |
+
"model.language_model.layers.23.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
|
| 231 |
+
"model.language_model.layers.23.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
| 232 |
+
"model.language_model.layers.23.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
|
| 233 |
+
"model.language_model.layers.23.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
|
| 234 |
+
"model.language_model.layers.23.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
|
| 235 |
+
"model.language_model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 236 |
+
"model.language_model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 237 |
+
"model.language_model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
|
| 238 |
+
"model.language_model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
|
| 239 |
+
"model.language_model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 240 |
+
"model.language_model.layers.3.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
|
| 241 |
+
"model.language_model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
|
| 242 |
+
"model.language_model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
| 243 |
+
"model.language_model.layers.3.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
|
| 244 |
+
"model.language_model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
|
| 245 |
+
"model.language_model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
|
| 246 |
+
"model.language_model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 247 |
+
"model.language_model.layers.4.linear_attn.A_log": "model-00001-of-00002.safetensors",
|
| 248 |
+
"model.language_model.layers.4.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors",
|
| 249 |
+
"model.language_model.layers.4.linear_attn.dt_bias": "model-00001-of-00002.safetensors",
|
| 250 |
+
"model.language_model.layers.4.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors",
|
| 251 |
+
"model.language_model.layers.4.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors",
|
| 252 |
+
"model.language_model.layers.4.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors",
|
| 253 |
+
"model.language_model.layers.4.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors",
|
| 254 |
+
"model.language_model.layers.4.linear_attn.norm.weight": "model-00001-of-00002.safetensors",
|
| 255 |
+
"model.language_model.layers.4.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors",
|
| 256 |
+
"model.language_model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 257 |
+
"model.language_model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
|
| 258 |
+
"model.language_model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
|
| 259 |
+
"model.language_model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 260 |
+
"model.language_model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 261 |
+
"model.language_model.layers.5.linear_attn.A_log": "model-00001-of-00002.safetensors",
|
| 262 |
+
"model.language_model.layers.5.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors",
|
| 263 |
+
"model.language_model.layers.5.linear_attn.dt_bias": "model-00001-of-00002.safetensors",
|
| 264 |
+
"model.language_model.layers.5.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors",
|
| 265 |
+
"model.language_model.layers.5.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors",
|
| 266 |
+
"model.language_model.layers.5.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors",
|
| 267 |
+
"model.language_model.layers.5.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors",
|
| 268 |
+
"model.language_model.layers.5.linear_attn.norm.weight": "model-00001-of-00002.safetensors",
|
| 269 |
+
"model.language_model.layers.5.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors",
|
| 270 |
+
"model.language_model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 271 |
+
"model.language_model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
|
| 272 |
+
"model.language_model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
|
| 273 |
+
"model.language_model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 274 |
+
"model.language_model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 275 |
+
"model.language_model.layers.6.linear_attn.A_log": "model-00001-of-00002.safetensors",
|
| 276 |
+
"model.language_model.layers.6.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors",
|
| 277 |
+
"model.language_model.layers.6.linear_attn.dt_bias": "model-00001-of-00002.safetensors",
|
| 278 |
+
"model.language_model.layers.6.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors",
|
| 279 |
+
"model.language_model.layers.6.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors",
|
| 280 |
+
"model.language_model.layers.6.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors",
|
| 281 |
+
"model.language_model.layers.6.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors",
|
| 282 |
+
"model.language_model.layers.6.linear_attn.norm.weight": "model-00001-of-00002.safetensors",
|
| 283 |
+
"model.language_model.layers.6.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors",
|
| 284 |
+
"model.language_model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 285 |
+
"model.language_model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
|
| 286 |
+
"model.language_model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
|
| 287 |
+
"model.language_model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 288 |
+
"model.language_model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 289 |
+
"model.language_model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 290 |
+
"model.language_model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
|
| 291 |
+
"model.language_model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
|
| 292 |
+
"model.language_model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 293 |
+
"model.language_model.layers.7.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
|
| 294 |
+
"model.language_model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
|
| 295 |
+
"model.language_model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
| 296 |
+
"model.language_model.layers.7.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
|
| 297 |
+
"model.language_model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
|
| 298 |
+
"model.language_model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
|
| 299 |
+
"model.language_model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 300 |
+
"model.language_model.layers.8.linear_attn.A_log": "model-00001-of-00002.safetensors",
|
| 301 |
+
"model.language_model.layers.8.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors",
|
| 302 |
+
"model.language_model.layers.8.linear_attn.dt_bias": "model-00001-of-00002.safetensors",
|
| 303 |
+
"model.language_model.layers.8.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors",
|
| 304 |
+
"model.language_model.layers.8.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors",
|
| 305 |
+
"model.language_model.layers.8.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors",
|
| 306 |
+
"model.language_model.layers.8.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors",
|
| 307 |
+
"model.language_model.layers.8.linear_attn.norm.weight": "model-00001-of-00002.safetensors",
|
| 308 |
+
"model.language_model.layers.8.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors",
|
| 309 |
+
"model.language_model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 310 |
+
"model.language_model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
|
| 311 |
+
"model.language_model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
|
| 312 |
+
"model.language_model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 313 |
+
"model.language_model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 314 |
+
"model.language_model.layers.9.linear_attn.A_log": "model-00001-of-00002.safetensors",
|
| 315 |
+
"model.language_model.layers.9.linear_attn.conv1d.weight": "model-00001-of-00002.safetensors",
|
| 316 |
+
"model.language_model.layers.9.linear_attn.dt_bias": "model-00001-of-00002.safetensors",
|
| 317 |
+
"model.language_model.layers.9.linear_attn.in_proj_a.weight": "model-00001-of-00002.safetensors",
|
| 318 |
+
"model.language_model.layers.9.linear_attn.in_proj_b.weight": "model-00001-of-00002.safetensors",
|
| 319 |
+
"model.language_model.layers.9.linear_attn.in_proj_qkv.weight": "model-00001-of-00002.safetensors",
|
| 320 |
+
"model.language_model.layers.9.linear_attn.in_proj_z.weight": "model-00001-of-00002.safetensors",
|
| 321 |
+
"model.language_model.layers.9.linear_attn.norm.weight": "model-00001-of-00002.safetensors",
|
| 322 |
+
"model.language_model.layers.9.linear_attn.out_proj.weight": "model-00001-of-00002.safetensors",
|
| 323 |
+
"model.language_model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 324 |
+
"model.language_model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
|
| 325 |
+
"model.language_model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
|
| 326 |
+
"model.language_model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 327 |
+
"model.language_model.norm.weight": "model-00001-of-00002.safetensors",
|
| 328 |
+
"model.visual.blocks.0.attn.proj.bias": "model-00001-of-00002.safetensors",
|
| 329 |
+
"model.visual.blocks.0.attn.proj.weight": "model-00001-of-00002.safetensors",
|
| 330 |
+
"model.visual.blocks.0.attn.qkv.bias": "model-00001-of-00002.safetensors",
|
| 331 |
+
"model.visual.blocks.0.attn.qkv.weight": "model-00001-of-00002.safetensors",
|
| 332 |
+
"model.visual.blocks.0.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
|
| 333 |
+
"model.visual.blocks.0.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
|
| 334 |
+
"model.visual.blocks.0.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
|
| 335 |
+
"model.visual.blocks.0.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
|
| 336 |
+
"model.visual.blocks.0.norm1.bias": "model-00001-of-00002.safetensors",
|
| 337 |
+
"model.visual.blocks.0.norm1.weight": "model-00001-of-00002.safetensors",
|
| 338 |
+
"model.visual.blocks.0.norm2.bias": "model-00001-of-00002.safetensors",
|
| 339 |
+
"model.visual.blocks.0.norm2.weight": "model-00001-of-00002.safetensors",
|
| 340 |
+
"model.visual.blocks.1.attn.proj.bias": "model-00001-of-00002.safetensors",
|
| 341 |
+
"model.visual.blocks.1.attn.proj.weight": "model-00001-of-00002.safetensors",
|
| 342 |
+
"model.visual.blocks.1.attn.qkv.bias": "model-00001-of-00002.safetensors",
|
| 343 |
+
"model.visual.blocks.1.attn.qkv.weight": "model-00001-of-00002.safetensors",
|
| 344 |
+
"model.visual.blocks.1.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
|
| 345 |
+
"model.visual.blocks.1.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
|
| 346 |
+
"model.visual.blocks.1.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
|
| 347 |
+
"model.visual.blocks.1.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
|
| 348 |
+
"model.visual.blocks.1.norm1.bias": "model-00001-of-00002.safetensors",
|
| 349 |
+
"model.visual.blocks.1.norm1.weight": "model-00001-of-00002.safetensors",
|
| 350 |
+
"model.visual.blocks.1.norm2.bias": "model-00001-of-00002.safetensors",
|
| 351 |
+
"model.visual.blocks.1.norm2.weight": "model-00001-of-00002.safetensors",
|
| 352 |
+
"model.visual.blocks.10.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 353 |
+
"model.visual.blocks.10.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 354 |
+
"model.visual.blocks.10.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 355 |
+
"model.visual.blocks.10.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 356 |
+
"model.visual.blocks.10.mlp.linear_fc1.bias": "model-00002-of-00002.safetensors",
|
| 357 |
+
"model.visual.blocks.10.mlp.linear_fc1.weight": "model-00002-of-00002.safetensors",
|
| 358 |
+
"model.visual.blocks.10.mlp.linear_fc2.bias": "model-00002-of-00002.safetensors",
|
| 359 |
+
"model.visual.blocks.10.mlp.linear_fc2.weight": "model-00002-of-00002.safetensors",
|
| 360 |
+
"model.visual.blocks.10.norm1.bias": "model-00002-of-00002.safetensors",
|
| 361 |
+
"model.visual.blocks.10.norm1.weight": "model-00002-of-00002.safetensors",
|
| 362 |
+
"model.visual.blocks.10.norm2.bias": "model-00002-of-00002.safetensors",
|
| 363 |
+
"model.visual.blocks.10.norm2.weight": "model-00002-of-00002.safetensors",
|
| 364 |
+
"model.visual.blocks.11.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 365 |
+
"model.visual.blocks.11.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 366 |
+
"model.visual.blocks.11.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 367 |
+
"model.visual.blocks.11.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 368 |
+
"model.visual.blocks.11.mlp.linear_fc1.bias": "model-00002-of-00002.safetensors",
|
| 369 |
+
"model.visual.blocks.11.mlp.linear_fc1.weight": "model-00002-of-00002.safetensors",
|
| 370 |
+
"model.visual.blocks.11.mlp.linear_fc2.bias": "model-00002-of-00002.safetensors",
|
| 371 |
+
"model.visual.blocks.11.mlp.linear_fc2.weight": "model-00002-of-00002.safetensors",
|
| 372 |
+
"model.visual.blocks.11.norm1.bias": "model-00002-of-00002.safetensors",
|
| 373 |
+
"model.visual.blocks.11.norm1.weight": "model-00002-of-00002.safetensors",
|
| 374 |
+
"model.visual.blocks.11.norm2.bias": "model-00002-of-00002.safetensors",
|
| 375 |
+
"model.visual.blocks.11.norm2.weight": "model-00002-of-00002.safetensors",
|
| 376 |
+
"model.visual.blocks.12.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 377 |
+
"model.visual.blocks.12.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 378 |
+
"model.visual.blocks.12.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 379 |
+
"model.visual.blocks.12.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 380 |
+
"model.visual.blocks.12.mlp.linear_fc1.bias": "model-00002-of-00002.safetensors",
|
| 381 |
+
"model.visual.blocks.12.mlp.linear_fc1.weight": "model-00002-of-00002.safetensors",
|
| 382 |
+
"model.visual.blocks.12.mlp.linear_fc2.bias": "model-00002-of-00002.safetensors",
|
| 383 |
+
"model.visual.blocks.12.mlp.linear_fc2.weight": "model-00002-of-00002.safetensors",
|
| 384 |
+
"model.visual.blocks.12.norm1.bias": "model-00002-of-00002.safetensors",
|
| 385 |
+
"model.visual.blocks.12.norm1.weight": "model-00002-of-00002.safetensors",
|
| 386 |
+
"model.visual.blocks.12.norm2.bias": "model-00002-of-00002.safetensors",
|
| 387 |
+
"model.visual.blocks.12.norm2.weight": "model-00002-of-00002.safetensors",
|
| 388 |
+
"model.visual.blocks.13.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 389 |
+
"model.visual.blocks.13.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 390 |
+
"model.visual.blocks.13.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 391 |
+
"model.visual.blocks.13.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 392 |
+
"model.visual.blocks.13.mlp.linear_fc1.bias": "model-00002-of-00002.safetensors",
|
| 393 |
+
"model.visual.blocks.13.mlp.linear_fc1.weight": "model-00002-of-00002.safetensors",
|
| 394 |
+
"model.visual.blocks.13.mlp.linear_fc2.bias": "model-00002-of-00002.safetensors",
|
| 395 |
+
"model.visual.blocks.13.mlp.linear_fc2.weight": "model-00002-of-00002.safetensors",
|
| 396 |
+
"model.visual.blocks.13.norm1.bias": "model-00002-of-00002.safetensors",
|
| 397 |
+
"model.visual.blocks.13.norm1.weight": "model-00002-of-00002.safetensors",
|
| 398 |
+
"model.visual.blocks.13.norm2.bias": "model-00002-of-00002.safetensors",
|
| 399 |
+
"model.visual.blocks.13.norm2.weight": "model-00002-of-00002.safetensors",
|
| 400 |
+
"model.visual.blocks.14.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 401 |
+
"model.visual.blocks.14.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 402 |
+
"model.visual.blocks.14.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 403 |
+
"model.visual.blocks.14.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 404 |
+
"model.visual.blocks.14.mlp.linear_fc1.bias": "model-00002-of-00002.safetensors",
|
| 405 |
+
"model.visual.blocks.14.mlp.linear_fc1.weight": "model-00002-of-00002.safetensors",
|
| 406 |
+
"model.visual.blocks.14.mlp.linear_fc2.bias": "model-00002-of-00002.safetensors",
|
| 407 |
+
"model.visual.blocks.14.mlp.linear_fc2.weight": "model-00002-of-00002.safetensors",
|
| 408 |
+
"model.visual.blocks.14.norm1.bias": "model-00002-of-00002.safetensors",
|
| 409 |
+
"model.visual.blocks.14.norm1.weight": "model-00002-of-00002.safetensors",
|
| 410 |
+
"model.visual.blocks.14.norm2.bias": "model-00002-of-00002.safetensors",
|
| 411 |
+
"model.visual.blocks.14.norm2.weight": "model-00002-of-00002.safetensors",
|
| 412 |
+
"model.visual.blocks.15.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 413 |
+
"model.visual.blocks.15.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 414 |
+
"model.visual.blocks.15.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 415 |
+
"model.visual.blocks.15.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 416 |
+
"model.visual.blocks.15.mlp.linear_fc1.bias": "model-00002-of-00002.safetensors",
|
| 417 |
+
"model.visual.blocks.15.mlp.linear_fc1.weight": "model-00002-of-00002.safetensors",
|
| 418 |
+
"model.visual.blocks.15.mlp.linear_fc2.bias": "model-00002-of-00002.safetensors",
|
| 419 |
+
"model.visual.blocks.15.mlp.linear_fc2.weight": "model-00002-of-00002.safetensors",
|
| 420 |
+
"model.visual.blocks.15.norm1.bias": "model-00002-of-00002.safetensors",
|
| 421 |
+
"model.visual.blocks.15.norm1.weight": "model-00002-of-00002.safetensors",
|
| 422 |
+
"model.visual.blocks.15.norm2.bias": "model-00002-of-00002.safetensors",
|
| 423 |
+
"model.visual.blocks.15.norm2.weight": "model-00002-of-00002.safetensors",
|
| 424 |
+
"model.visual.blocks.16.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 425 |
+
"model.visual.blocks.16.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 426 |
+
"model.visual.blocks.16.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 427 |
+
"model.visual.blocks.16.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 428 |
+
"model.visual.blocks.16.mlp.linear_fc1.bias": "model-00002-of-00002.safetensors",
|
| 429 |
+
"model.visual.blocks.16.mlp.linear_fc1.weight": "model-00002-of-00002.safetensors",
|
| 430 |
+
"model.visual.blocks.16.mlp.linear_fc2.bias": "model-00002-of-00002.safetensors",
|
| 431 |
+
"model.visual.blocks.16.mlp.linear_fc2.weight": "model-00002-of-00002.safetensors",
|
| 432 |
+
"model.visual.blocks.16.norm1.bias": "model-00002-of-00002.safetensors",
|
| 433 |
+
"model.visual.blocks.16.norm1.weight": "model-00002-of-00002.safetensors",
|
| 434 |
+
"model.visual.blocks.16.norm2.bias": "model-00002-of-00002.safetensors",
|
| 435 |
+
"model.visual.blocks.16.norm2.weight": "model-00002-of-00002.safetensors",
|
| 436 |
+
"model.visual.blocks.17.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 437 |
+
"model.visual.blocks.17.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 438 |
+
"model.visual.blocks.17.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 439 |
+
"model.visual.blocks.17.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 440 |
+
"model.visual.blocks.17.mlp.linear_fc1.bias": "model-00002-of-00002.safetensors",
|
| 441 |
+
"model.visual.blocks.17.mlp.linear_fc1.weight": "model-00002-of-00002.safetensors",
|
| 442 |
+
"model.visual.blocks.17.mlp.linear_fc2.bias": "model-00002-of-00002.safetensors",
|
| 443 |
+
"model.visual.blocks.17.mlp.linear_fc2.weight": "model-00002-of-00002.safetensors",
|
| 444 |
+
"model.visual.blocks.17.norm1.bias": "model-00002-of-00002.safetensors",
|
| 445 |
+
"model.visual.blocks.17.norm1.weight": "model-00002-of-00002.safetensors",
|
| 446 |
+
"model.visual.blocks.17.norm2.bias": "model-00002-of-00002.safetensors",
|
| 447 |
+
"model.visual.blocks.17.norm2.weight": "model-00002-of-00002.safetensors",
|
| 448 |
+
"model.visual.blocks.18.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 449 |
+
"model.visual.blocks.18.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 450 |
+
"model.visual.blocks.18.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 451 |
+
"model.visual.blocks.18.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 452 |
+
"model.visual.blocks.18.mlp.linear_fc1.bias": "model-00002-of-00002.safetensors",
|
| 453 |
+
"model.visual.blocks.18.mlp.linear_fc1.weight": "model-00002-of-00002.safetensors",
|
| 454 |
+
"model.visual.blocks.18.mlp.linear_fc2.bias": "model-00002-of-00002.safetensors",
|
| 455 |
+
"model.visual.blocks.18.mlp.linear_fc2.weight": "model-00002-of-00002.safetensors",
|
| 456 |
+
"model.visual.blocks.18.norm1.bias": "model-00002-of-00002.safetensors",
|
| 457 |
+
"model.visual.blocks.18.norm1.weight": "model-00002-of-00002.safetensors",
|
| 458 |
+
"model.visual.blocks.18.norm2.bias": "model-00002-of-00002.safetensors",
|
| 459 |
+
"model.visual.blocks.18.norm2.weight": "model-00002-of-00002.safetensors",
|
| 460 |
+
"model.visual.blocks.19.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 461 |
+
"model.visual.blocks.19.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 462 |
+
"model.visual.blocks.19.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 463 |
+
"model.visual.blocks.19.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 464 |
+
"model.visual.blocks.19.mlp.linear_fc1.bias": "model-00002-of-00002.safetensors",
|
| 465 |
+
"model.visual.blocks.19.mlp.linear_fc1.weight": "model-00002-of-00002.safetensors",
|
| 466 |
+
"model.visual.blocks.19.mlp.linear_fc2.bias": "model-00002-of-00002.safetensors",
|
| 467 |
+
"model.visual.blocks.19.mlp.linear_fc2.weight": "model-00002-of-00002.safetensors",
|
| 468 |
+
"model.visual.blocks.19.norm1.bias": "model-00002-of-00002.safetensors",
|
| 469 |
+
"model.visual.blocks.19.norm1.weight": "model-00002-of-00002.safetensors",
|
| 470 |
+
"model.visual.blocks.19.norm2.bias": "model-00002-of-00002.safetensors",
|
| 471 |
+
"model.visual.blocks.19.norm2.weight": "model-00002-of-00002.safetensors",
|
| 472 |
+
"model.visual.blocks.2.attn.proj.bias": "model-00001-of-00002.safetensors",
|
| 473 |
+
"model.visual.blocks.2.attn.proj.weight": "model-00001-of-00002.safetensors",
|
| 474 |
+
"model.visual.blocks.2.attn.qkv.bias": "model-00001-of-00002.safetensors",
|
| 475 |
+
"model.visual.blocks.2.attn.qkv.weight": "model-00001-of-00002.safetensors",
|
| 476 |
+
"model.visual.blocks.2.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
|
| 477 |
+
"model.visual.blocks.2.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
|
| 478 |
+
"model.visual.blocks.2.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
|
| 479 |
+
"model.visual.blocks.2.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
|
| 480 |
+
"model.visual.blocks.2.norm1.bias": "model-00001-of-00002.safetensors",
|
| 481 |
+
"model.visual.blocks.2.norm1.weight": "model-00001-of-00002.safetensors",
|
| 482 |
+
"model.visual.blocks.2.norm2.bias": "model-00001-of-00002.safetensors",
|
| 483 |
+
"model.visual.blocks.2.norm2.weight": "model-00001-of-00002.safetensors",
|
| 484 |
+
"model.visual.blocks.20.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 485 |
+
"model.visual.blocks.20.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 486 |
+
"model.visual.blocks.20.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 487 |
+
"model.visual.blocks.20.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 488 |
+
"model.visual.blocks.20.mlp.linear_fc1.bias": "model-00002-of-00002.safetensors",
|
| 489 |
+
"model.visual.blocks.20.mlp.linear_fc1.weight": "model-00002-of-00002.safetensors",
|
| 490 |
+
"model.visual.blocks.20.mlp.linear_fc2.bias": "model-00002-of-00002.safetensors",
|
| 491 |
+
"model.visual.blocks.20.mlp.linear_fc2.weight": "model-00002-of-00002.safetensors",
|
| 492 |
+
"model.visual.blocks.20.norm1.bias": "model-00002-of-00002.safetensors",
|
| 493 |
+
"model.visual.blocks.20.norm1.weight": "model-00002-of-00002.safetensors",
|
| 494 |
+
"model.visual.blocks.20.norm2.bias": "model-00002-of-00002.safetensors",
|
| 495 |
+
"model.visual.blocks.20.norm2.weight": "model-00002-of-00002.safetensors",
|
| 496 |
+
"model.visual.blocks.21.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 497 |
+
"model.visual.blocks.21.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 498 |
+
"model.visual.blocks.21.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 499 |
+
"model.visual.blocks.21.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 500 |
+
"model.visual.blocks.21.mlp.linear_fc1.bias": "model-00002-of-00002.safetensors",
|
| 501 |
+
"model.visual.blocks.21.mlp.linear_fc1.weight": "model-00002-of-00002.safetensors",
|
| 502 |
+
"model.visual.blocks.21.mlp.linear_fc2.bias": "model-00002-of-00002.safetensors",
|
| 503 |
+
"model.visual.blocks.21.mlp.linear_fc2.weight": "model-00002-of-00002.safetensors",
|
| 504 |
+
"model.visual.blocks.21.norm1.bias": "model-00002-of-00002.safetensors",
|
| 505 |
+
"model.visual.blocks.21.norm1.weight": "model-00002-of-00002.safetensors",
|
| 506 |
+
"model.visual.blocks.21.norm2.bias": "model-00002-of-00002.safetensors",
|
| 507 |
+
"model.visual.blocks.21.norm2.weight": "model-00002-of-00002.safetensors",
|
| 508 |
+
"model.visual.blocks.22.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 509 |
+
"model.visual.blocks.22.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 510 |
+
"model.visual.blocks.22.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 511 |
+
"model.visual.blocks.22.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 512 |
+
"model.visual.blocks.22.mlp.linear_fc1.bias": "model-00002-of-00002.safetensors",
|
| 513 |
+
"model.visual.blocks.22.mlp.linear_fc1.weight": "model-00002-of-00002.safetensors",
|
| 514 |
+
"model.visual.blocks.22.mlp.linear_fc2.bias": "model-00002-of-00002.safetensors",
|
| 515 |
+
"model.visual.blocks.22.mlp.linear_fc2.weight": "model-00002-of-00002.safetensors",
|
| 516 |
+
"model.visual.blocks.22.norm1.bias": "model-00002-of-00002.safetensors",
|
| 517 |
+
"model.visual.blocks.22.norm1.weight": "model-00002-of-00002.safetensors",
|
| 518 |
+
"model.visual.blocks.22.norm2.bias": "model-00002-of-00002.safetensors",
|
| 519 |
+
"model.visual.blocks.22.norm2.weight": "model-00002-of-00002.safetensors",
|
| 520 |
+
"model.visual.blocks.23.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 521 |
+
"model.visual.blocks.23.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 522 |
+
"model.visual.blocks.23.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 523 |
+
"model.visual.blocks.23.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 524 |
+
"model.visual.blocks.23.mlp.linear_fc1.bias": "model-00002-of-00002.safetensors",
|
| 525 |
+
"model.visual.blocks.23.mlp.linear_fc1.weight": "model-00002-of-00002.safetensors",
|
| 526 |
+
"model.visual.blocks.23.mlp.linear_fc2.bias": "model-00002-of-00002.safetensors",
|
| 527 |
+
"model.visual.blocks.23.mlp.linear_fc2.weight": "model-00002-of-00002.safetensors",
|
| 528 |
+
"model.visual.blocks.23.norm1.bias": "model-00002-of-00002.safetensors",
|
| 529 |
+
"model.visual.blocks.23.norm1.weight": "model-00002-of-00002.safetensors",
|
| 530 |
+
"model.visual.blocks.23.norm2.bias": "model-00002-of-00002.safetensors",
|
| 531 |
+
"model.visual.blocks.23.norm2.weight": "model-00002-of-00002.safetensors",
|
| 532 |
+
"model.visual.blocks.3.attn.proj.bias": "model-00001-of-00002.safetensors",
|
| 533 |
+
"model.visual.blocks.3.attn.proj.weight": "model-00001-of-00002.safetensors",
|
| 534 |
+
"model.visual.blocks.3.attn.qkv.bias": "model-00001-of-00002.safetensors",
|
| 535 |
+
"model.visual.blocks.3.attn.qkv.weight": "model-00001-of-00002.safetensors",
|
| 536 |
+
"model.visual.blocks.3.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
|
| 537 |
+
"model.visual.blocks.3.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
|
| 538 |
+
"model.visual.blocks.3.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
|
| 539 |
+
"model.visual.blocks.3.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
|
| 540 |
+
"model.visual.blocks.3.norm1.bias": "model-00001-of-00002.safetensors",
|
| 541 |
+
"model.visual.blocks.3.norm1.weight": "model-00001-of-00002.safetensors",
|
| 542 |
+
"model.visual.blocks.3.norm2.bias": "model-00001-of-00002.safetensors",
|
| 543 |
+
"model.visual.blocks.3.norm2.weight": "model-00001-of-00002.safetensors",
|
| 544 |
+
"model.visual.blocks.4.attn.proj.bias": "model-00001-of-00002.safetensors",
|
| 545 |
+
"model.visual.blocks.4.attn.proj.weight": "model-00001-of-00002.safetensors",
|
| 546 |
+
"model.visual.blocks.4.attn.qkv.bias": "model-00001-of-00002.safetensors",
|
| 547 |
+
"model.visual.blocks.4.attn.qkv.weight": "model-00001-of-00002.safetensors",
|
| 548 |
+
"model.visual.blocks.4.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
|
| 549 |
+
"model.visual.blocks.4.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
|
| 550 |
+
"model.visual.blocks.4.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
|
| 551 |
+
"model.visual.blocks.4.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
|
| 552 |
+
"model.visual.blocks.4.norm1.bias": "model-00001-of-00002.safetensors",
|
| 553 |
+
"model.visual.blocks.4.norm1.weight": "model-00001-of-00002.safetensors",
|
| 554 |
+
"model.visual.blocks.4.norm2.bias": "model-00001-of-00002.safetensors",
|
| 555 |
+
"model.visual.blocks.4.norm2.weight": "model-00001-of-00002.safetensors",
|
| 556 |
+
"model.visual.blocks.5.attn.proj.bias": "model-00001-of-00002.safetensors",
|
| 557 |
+
"model.visual.blocks.5.attn.proj.weight": "model-00001-of-00002.safetensors",
|
| 558 |
+
"model.visual.blocks.5.attn.qkv.bias": "model-00001-of-00002.safetensors",
|
| 559 |
+
"model.visual.blocks.5.attn.qkv.weight": "model-00001-of-00002.safetensors",
|
| 560 |
+
"model.visual.blocks.5.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
|
| 561 |
+
"model.visual.blocks.5.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
|
| 562 |
+
"model.visual.blocks.5.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
|
| 563 |
+
"model.visual.blocks.5.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
|
| 564 |
+
"model.visual.blocks.5.norm1.bias": "model-00001-of-00002.safetensors",
|
| 565 |
+
"model.visual.blocks.5.norm1.weight": "model-00001-of-00002.safetensors",
|
| 566 |
+
"model.visual.blocks.5.norm2.bias": "model-00001-of-00002.safetensors",
|
| 567 |
+
"model.visual.blocks.5.norm2.weight": "model-00001-of-00002.safetensors",
|
| 568 |
+
"model.visual.blocks.6.attn.proj.bias": "model-00001-of-00002.safetensors",
|
| 569 |
+
"model.visual.blocks.6.attn.proj.weight": "model-00001-of-00002.safetensors",
|
| 570 |
+
"model.visual.blocks.6.attn.qkv.bias": "model-00001-of-00002.safetensors",
|
| 571 |
+
"model.visual.blocks.6.attn.qkv.weight": "model-00001-of-00002.safetensors",
|
| 572 |
+
"model.visual.blocks.6.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
|
| 573 |
+
"model.visual.blocks.6.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
|
| 574 |
+
"model.visual.blocks.6.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
|
| 575 |
+
"model.visual.blocks.6.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
|
| 576 |
+
"model.visual.blocks.6.norm1.bias": "model-00001-of-00002.safetensors",
|
| 577 |
+
"model.visual.blocks.6.norm1.weight": "model-00001-of-00002.safetensors",
|
| 578 |
+
"model.visual.blocks.6.norm2.bias": "model-00001-of-00002.safetensors",
|
| 579 |
+
"model.visual.blocks.6.norm2.weight": "model-00001-of-00002.safetensors",
|
| 580 |
+
"model.visual.blocks.7.attn.proj.bias": "model-00001-of-00002.safetensors",
|
| 581 |
+
"model.visual.blocks.7.attn.proj.weight": "model-00001-of-00002.safetensors",
|
| 582 |
+
"model.visual.blocks.7.attn.qkv.bias": "model-00001-of-00002.safetensors",
|
| 583 |
+
"model.visual.blocks.7.attn.qkv.weight": "model-00001-of-00002.safetensors",
|
| 584 |
+
"model.visual.blocks.7.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
|
| 585 |
+
"model.visual.blocks.7.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
|
| 586 |
+
"model.visual.blocks.7.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
|
| 587 |
+
"model.visual.blocks.7.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
|
| 588 |
+
"model.visual.blocks.7.norm1.bias": "model-00001-of-00002.safetensors",
|
| 589 |
+
"model.visual.blocks.7.norm1.weight": "model-00001-of-00002.safetensors",
|
| 590 |
+
"model.visual.blocks.7.norm2.bias": "model-00001-of-00002.safetensors",
|
| 591 |
+
"model.visual.blocks.7.norm2.weight": "model-00001-of-00002.safetensors",
|
| 592 |
+
"model.visual.blocks.8.attn.proj.bias": "model-00001-of-00002.safetensors",
|
| 593 |
+
"model.visual.blocks.8.attn.proj.weight": "model-00001-of-00002.safetensors",
|
| 594 |
+
"model.visual.blocks.8.attn.qkv.bias": "model-00001-of-00002.safetensors",
|
| 595 |
+
"model.visual.blocks.8.attn.qkv.weight": "model-00001-of-00002.safetensors",
|
| 596 |
+
"model.visual.blocks.8.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
|
| 597 |
+
"model.visual.blocks.8.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
|
| 598 |
+
"model.visual.blocks.8.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
|
| 599 |
+
"model.visual.blocks.8.mlp.linear_fc2.weight": "model-00002-of-00002.safetensors",
|
| 600 |
+
"model.visual.blocks.8.norm1.bias": "model-00002-of-00002.safetensors",
|
| 601 |
+
"model.visual.blocks.8.norm1.weight": "model-00002-of-00002.safetensors",
|
| 602 |
+
"model.visual.blocks.8.norm2.bias": "model-00002-of-00002.safetensors",
|
| 603 |
+
"model.visual.blocks.8.norm2.weight": "model-00002-of-00002.safetensors",
|
| 604 |
+
"model.visual.blocks.9.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 605 |
+
"model.visual.blocks.9.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 606 |
+
"model.visual.blocks.9.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 607 |
+
"model.visual.blocks.9.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 608 |
+
"model.visual.blocks.9.mlp.linear_fc1.bias": "model-00002-of-00002.safetensors",
|
| 609 |
+
"model.visual.blocks.9.mlp.linear_fc1.weight": "model-00002-of-00002.safetensors",
|
| 610 |
+
"model.visual.blocks.9.mlp.linear_fc2.bias": "model-00002-of-00002.safetensors",
|
| 611 |
+
"model.visual.blocks.9.mlp.linear_fc2.weight": "model-00002-of-00002.safetensors",
|
| 612 |
+
"model.visual.blocks.9.norm1.bias": "model-00002-of-00002.safetensors",
|
| 613 |
+
"model.visual.blocks.9.norm1.weight": "model-00002-of-00002.safetensors",
|
| 614 |
+
"model.visual.blocks.9.norm2.bias": "model-00002-of-00002.safetensors",
|
| 615 |
+
"model.visual.blocks.9.norm2.weight": "model-00002-of-00002.safetensors",
|
| 616 |
+
"model.visual.merger.linear_fc1.bias": "model-00002-of-00002.safetensors",
|
| 617 |
+
"model.visual.merger.linear_fc1.weight": "model-00002-of-00002.safetensors",
|
| 618 |
+
"model.visual.merger.linear_fc2.bias": "model-00002-of-00002.safetensors",
|
| 619 |
+
"model.visual.merger.linear_fc2.weight": "model-00002-of-00002.safetensors",
|
| 620 |
+
"model.visual.merger.norm.bias": "model-00002-of-00002.safetensors",
|
| 621 |
+
"model.visual.merger.norm.weight": "model-00002-of-00002.safetensors",
|
| 622 |
+
"model.visual.patch_embed.proj.bias": "model-00002-of-00002.safetensors",
|
| 623 |
+
"model.visual.patch_embed.proj.weight": "model-00002-of-00002.safetensors",
|
| 624 |
+
"model.visual.pos_embed.weight": "model-00002-of-00002.safetensors"
|
| 625 |
+
}
|
| 626 |
+
}
|
modeling_marlin.py
ADDED
|
@@ -0,0 +1,533 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Custom HuggingFace modeling code for Marlin.
|
| 2 |
+
|
| 3 |
+
This module subclasses the upstream ``Qwen3_5ForConditionalGeneration``
|
| 4 |
+
(native in ``transformers >= 5.7.0``) and adds two convenience methods —
|
| 5 |
+
:meth:`MarlinForConditionalGeneration.caption` and
|
| 6 |
+
:meth:`MarlinForConditionalGeneration.find` — that mirror moondream's
|
| 7 |
+
image-SDK ergonomics for video captioning and temporal grounding.
|
| 8 |
+
|
| 9 |
+
The forward pass is **not** modified: we only add chat-template + generate +
|
| 10 |
+
post-processing wrappers. Loading the model through
|
| 11 |
+
``AutoModelForCausalLM.from_pretrained(..., trust_remote_code=True)`` returns
|
| 12 |
+
this subclass thanks to the ``auto_map`` entry in ``config.json``.
|
| 13 |
+
|
| 14 |
+
Required environment for video inference (set before importing transformers)::
|
| 15 |
+
|
| 16 |
+
FORCE_QWENVL_VIDEO_READER=torchcodec
|
| 17 |
+
VIDEO_MAX_PIXELS=200704
|
| 18 |
+
FPS=2.0
|
| 19 |
+
FPS_MAX_FRAMES=240
|
| 20 |
+
FPS_MIN_FRAMES=4
|
| 21 |
+
|
| 22 |
+
System requirements:
|
| 23 |
+
|
| 24 |
+
* transformers >= 5.7.0
|
| 25 |
+
* torch >= 2.11.0
|
| 26 |
+
* torchcodec
|
| 27 |
+
* qwen-vl-utils >= 0.0.14
|
| 28 |
+
* av, pillow
|
| 29 |
+
"""
|
| 30 |
+
|
| 31 |
+
from __future__ import annotations
|
| 32 |
+
|
| 33 |
+
import os
|
| 34 |
+
import re
|
| 35 |
+
from typing import Any, Dict, List, Optional, Tuple, TypedDict, Union
|
| 36 |
+
|
| 37 |
+
import torch
|
| 38 |
+
|
| 39 |
+
# ``Qwen3_5ForConditionalGeneration`` is the native HF class for Marlin's
|
| 40 |
+
# backbone (Qwen3.5-2B with vision tower). It ships in transformers >= 5.7.0.
|
| 41 |
+
# We import it lazily-friendly at module top so AutoModelForCausalLM resolution
|
| 42 |
+
# works correctly when this file is loaded via ``trust_remote_code=True``.
|
| 43 |
+
from transformers import Qwen3_5ForConditionalGeneration
|
| 44 |
+
|
| 45 |
+
__all__ = [
|
| 46 |
+
"CAPTION_PROMPT",
|
| 47 |
+
"GROUNDING_PROMPT_TEMPLATE",
|
| 48 |
+
"CaptionResult",
|
| 49 |
+
"FindResult",
|
| 50 |
+
"Event",
|
| 51 |
+
"MarlinForConditionalGeneration",
|
| 52 |
+
"strip_thinking",
|
| 53 |
+
"parse_caption",
|
| 54 |
+
"parse_span",
|
| 55 |
+
]
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
# ---------------------------------------------------------------------------
|
| 59 |
+
# Canonical training-time prompts — DO NOT EDIT
|
| 60 |
+
# ---------------------------------------------------------------------------
|
| 61 |
+
#
|
| 62 |
+
# These strings must match exactly what the model was fine-tuned on. Diverging
|
| 63 |
+
# from them silently degrades quality.
|
| 64 |
+
|
| 65 |
+
CAPTION_PROMPT: str = (
|
| 66 |
+
"Provide a spatial description of this clip followed by time-ranged events.\n"
|
| 67 |
+
"For each event, give the time range as <start - end> and a short description."
|
| 68 |
+
)
|
| 69 |
+
|
| 70 |
+
GROUNDING_PROMPT_TEMPLATE: str = (
|
| 71 |
+
'Identify the timestamps during which "{event}" takes place. '
|
| 72 |
+
'Output the time range as "From <start> to <end>." (numbers in seconds).'
|
| 73 |
+
)
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
# ---------------------------------------------------------------------------
|
| 77 |
+
# Thinking-tag stripping
|
| 78 |
+
# ---------------------------------------------------------------------------
|
| 79 |
+
#
|
| 80 |
+
# ms-swift's Marlin training template uses ``add_non_thinking_prefix=True``,
|
| 81 |
+
# which prefixes every response with a bare ``<think>\n`` (no close tag). The
|
| 82 |
+
# model occasionally also emits a complete ``<think>...</think>`` block. Strip
|
| 83 |
+
# both robustly.
|
| 84 |
+
|
| 85 |
+
_THINK_BLOCK = re.compile(r"<think>.*?</think>\s*", re.DOTALL)
|
| 86 |
+
_THINK_PREFIX = re.compile(r"^\s*<think>\s*\n*", re.IGNORECASE)
|
| 87 |
+
_THINK_CLOSE = re.compile(r"</think>\s*", re.IGNORECASE)
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def strip_thinking(text: str) -> str:
|
| 91 |
+
"""Remove ``<think>...</think>`` blocks and bare ``<think>`` prefixes.
|
| 92 |
+
|
| 93 |
+
Parameters
|
| 94 |
+
----------
|
| 95 |
+
text:
|
| 96 |
+
Raw model output.
|
| 97 |
+
|
| 98 |
+
Returns
|
| 99 |
+
-------
|
| 100 |
+
str
|
| 101 |
+
The text with any thinking artifacts removed and outer whitespace
|
| 102 |
+
stripped.
|
| 103 |
+
"""
|
| 104 |
+
out = _THINK_BLOCK.sub("", text)
|
| 105 |
+
out = _THINK_PREFIX.sub("", out)
|
| 106 |
+
out = _THINK_CLOSE.sub("", out)
|
| 107 |
+
return out.strip()
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
# ---------------------------------------------------------------------------
|
| 111 |
+
# Mode 1 — dense caption parser
|
| 112 |
+
# ---------------------------------------------------------------------------
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
class Event(TypedDict):
|
| 116 |
+
"""A single time-ranged event extracted from a dense caption."""
|
| 117 |
+
|
| 118 |
+
start: float
|
| 119 |
+
end: float
|
| 120 |
+
description: str
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
# Tolerates ``<1.2 - 3.4>`` / ``1.2 - 3.4`` / ``1.2-3.4`` with optional units.
|
| 124 |
+
# Unit alternation is ordered longest-first so e.g. ``"1.8 seconds"`` consumes
|
| 125 |
+
# the full word instead of leaving ``"econds"`` in the description.
|
| 126 |
+
_EVENT_LINE = re.compile(
|
| 127 |
+
r"^\s*<?\s*(\d+\.?\d*)\s*(?:seconds?|secs?|s)?\s*-\s*"
|
| 128 |
+
r"(\d+\.?\d*)\s*(?:seconds?|secs?|s)?\s*>?\s*[:\-]?\s*(.+?)\s*$"
|
| 129 |
+
)
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def _parse_events(events_block: str) -> List[Event]:
|
| 133 |
+
"""Parse a multi-line events block into a list of :class:`Event` dicts."""
|
| 134 |
+
out: List[Event] = []
|
| 135 |
+
for raw_line in events_block.splitlines():
|
| 136 |
+
line = raw_line.strip()
|
| 137 |
+
if not line:
|
| 138 |
+
continue
|
| 139 |
+
m = _EVENT_LINE.match(line)
|
| 140 |
+
if not m:
|
| 141 |
+
continue
|
| 142 |
+
start = float(m.group(1))
|
| 143 |
+
end = float(m.group(2))
|
| 144 |
+
desc = m.group(3).strip().lstrip("-").strip()
|
| 145 |
+
if end <= start or not desc:
|
| 146 |
+
continue
|
| 147 |
+
out.append(Event(start=start, end=end, description=desc))
|
| 148 |
+
return out
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
def parse_caption(text: str) -> Tuple[str, str, List[Event]]:
|
| 152 |
+
"""Parse a Mode 1 caption into ``(caption, scene, events)``.
|
| 153 |
+
|
| 154 |
+
The model is trained to produce::
|
| 155 |
+
|
| 156 |
+
Scene: <one-paragraph spatial description>
|
| 157 |
+
|
| 158 |
+
Events:
|
| 159 |
+
<start - end> <description>
|
| 160 |
+
<start - end> <description>
|
| 161 |
+
|
| 162 |
+
The parser is tolerant: if explicit ``Scene:`` / ``Events:`` headers are
|
| 163 |
+
missing, ``scene`` falls back to everything before the first event line and
|
| 164 |
+
``events`` is whatever event-shaped lines were detected.
|
| 165 |
+
|
| 166 |
+
Parameters
|
| 167 |
+
----------
|
| 168 |
+
text:
|
| 169 |
+
Raw model output. Thinking artifacts will be stripped.
|
| 170 |
+
|
| 171 |
+
Returns
|
| 172 |
+
-------
|
| 173 |
+
tuple
|
| 174 |
+
``(caption, scene, events)`` — the post-thinking full text, the parsed
|
| 175 |
+
scene paragraph, and a list of :class:`Event` dicts in emission order.
|
| 176 |
+
"""
|
| 177 |
+
cleaned = strip_thinking(text)
|
| 178 |
+
|
| 179 |
+
scene_match = re.search(
|
| 180 |
+
r"(?:^|\n)\s*Scene\s*:\s*(.*?)(?=\n\s*Events\s*:|\Z)",
|
| 181 |
+
cleaned,
|
| 182 |
+
re.IGNORECASE | re.DOTALL,
|
| 183 |
+
)
|
| 184 |
+
events_match = re.search(
|
| 185 |
+
r"(?:^|\n)\s*Events\s*:\s*(.*)\Z",
|
| 186 |
+
cleaned,
|
| 187 |
+
re.IGNORECASE | re.DOTALL,
|
| 188 |
+
)
|
| 189 |
+
|
| 190 |
+
if scene_match:
|
| 191 |
+
scene = scene_match.group(1).strip()
|
| 192 |
+
else:
|
| 193 |
+
# Fallback: scene = everything before the first event-shaped line.
|
| 194 |
+
scene_lines: List[str] = []
|
| 195 |
+
for line in cleaned.splitlines():
|
| 196 |
+
if _EVENT_LINE.match(line.strip()):
|
| 197 |
+
break
|
| 198 |
+
scene_lines.append(line)
|
| 199 |
+
scene = "\n".join(scene_lines).strip()
|
| 200 |
+
|
| 201 |
+
events_block = events_match.group(1) if events_match else cleaned
|
| 202 |
+
events = _parse_events(events_block)
|
| 203 |
+
|
| 204 |
+
return cleaned, scene, events
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
# ---------------------------------------------------------------------------
|
| 208 |
+
# Mode 2 — temporal grounding parser
|
| 209 |
+
# ---------------------------------------------------------------------------
|
| 210 |
+
|
| 211 |
+
# Tolerates ``From 1.2 to 3.4.``, ``From 1.2s to 3.4 sec``; trailing period
|
| 212 |
+
# optional.
|
| 213 |
+
_SPAN_RE = re.compile(
|
| 214 |
+
r"From\s+(\d+\.?\d*)\s*(?:s|sec)?\s+to\s+(\d+\.?\d*)\s*(?:s|sec)?\.?",
|
| 215 |
+
re.IGNORECASE,
|
| 216 |
+
)
|
| 217 |
+
|
| 218 |
+
|
| 219 |
+
def parse_span(text: str) -> Tuple[str, Optional[Tuple[float, float]]]:
|
| 220 |
+
"""Parse a Mode 2 grounding output into ``(text, span)``.
|
| 221 |
+
|
| 222 |
+
Parameters
|
| 223 |
+
----------
|
| 224 |
+
text:
|
| 225 |
+
Raw model output. Thinking artifacts will be stripped.
|
| 226 |
+
|
| 227 |
+
Returns
|
| 228 |
+
-------
|
| 229 |
+
tuple
|
| 230 |
+
``(cleaned, span)`` — the post-thinking text and ``(start, end)`` in
|
| 231 |
+
seconds, or ``None`` if no valid ``"From X to Y"`` substring was found
|
| 232 |
+
or the span was non-positive.
|
| 233 |
+
"""
|
| 234 |
+
cleaned = strip_thinking(text)
|
| 235 |
+
m = _SPAN_RE.search(cleaned)
|
| 236 |
+
if not m:
|
| 237 |
+
return cleaned, None
|
| 238 |
+
start = float(m.group(1))
|
| 239 |
+
end = float(m.group(2))
|
| 240 |
+
if end <= start:
|
| 241 |
+
return cleaned, None
|
| 242 |
+
return cleaned, (start, end)
|
| 243 |
+
|
| 244 |
+
|
| 245 |
+
# ---------------------------------------------------------------------------
|
| 246 |
+
# Result dicts
|
| 247 |
+
# ---------------------------------------------------------------------------
|
| 248 |
+
|
| 249 |
+
|
| 250 |
+
class CaptionResult(TypedDict):
|
| 251 |
+
"""Return type for :meth:`MarlinForConditionalGeneration.caption`.
|
| 252 |
+
|
| 253 |
+
Keys
|
| 254 |
+
----
|
| 255 |
+
caption : str
|
| 256 |
+
Post-thinking model output (e.g. ``"Scene: ...\\n\\nEvents:\\n..."``).
|
| 257 |
+
scene : str
|
| 258 |
+
Parsed ``Scene:`` paragraph.
|
| 259 |
+
events : list of :class:`Event`
|
| 260 |
+
Parsed ``{start, end, description}`` dicts in emission order.
|
| 261 |
+
raw : str
|
| 262 |
+
Raw model output *before* thinking-prefix stripping (for debugging).
|
| 263 |
+
"""
|
| 264 |
+
|
| 265 |
+
caption: str
|
| 266 |
+
scene: str
|
| 267 |
+
events: List[Event]
|
| 268 |
+
raw: str
|
| 269 |
+
|
| 270 |
+
|
| 271 |
+
class FindResult(TypedDict):
|
| 272 |
+
"""Return type for :meth:`MarlinForConditionalGeneration.find`.
|
| 273 |
+
|
| 274 |
+
Keys
|
| 275 |
+
----
|
| 276 |
+
raw : str
|
| 277 |
+
Raw post-thinking model output (e.g. ``"From 1.2 to 3.4."``).
|
| 278 |
+
span : tuple of (float, float) or None
|
| 279 |
+
``(start, end)`` in seconds, or ``None`` if parsing failed.
|
| 280 |
+
format_ok : bool
|
| 281 |
+
``True`` iff the output matched the trained ``"From X to Y."`` format.
|
| 282 |
+
"""
|
| 283 |
+
|
| 284 |
+
raw: str
|
| 285 |
+
span: Optional[Tuple[float, float]]
|
| 286 |
+
format_ok: bool
|
| 287 |
+
|
| 288 |
+
|
| 289 |
+
# ---------------------------------------------------------------------------
|
| 290 |
+
# Default video-preprocessing env vars
|
| 291 |
+
# ---------------------------------------------------------------------------
|
| 292 |
+
#
|
| 293 |
+
# qwen-vl-utils reads these from the environment when ``apply_chat_template``
|
| 294 |
+
# decodes a video. We populate them here as a safety net for users who forget
|
| 295 |
+
# to set them before importing transformers. Existing env values are NEVER
|
| 296 |
+
# overwritten — explicit user settings always win.
|
| 297 |
+
|
| 298 |
+
_DEFAULT_VIDEO_ENV: Dict[str, str] = {
|
| 299 |
+
"FORCE_QWENVL_VIDEO_READER": "torchcodec",
|
| 300 |
+
"VIDEO_MAX_PIXELS": "200704",
|
| 301 |
+
"FPS": "2.0",
|
| 302 |
+
"FPS_MAX_FRAMES": "240",
|
| 303 |
+
"FPS_MIN_FRAMES": "4",
|
| 304 |
+
}
|
| 305 |
+
|
| 306 |
+
for _k, _v in _DEFAULT_VIDEO_ENV.items():
|
| 307 |
+
os.environ.setdefault(_k, _v)
|
| 308 |
+
|
| 309 |
+
|
| 310 |
+
# ---------------------------------------------------------------------------
|
| 311 |
+
# The actual model class
|
| 312 |
+
# ---------------------------------------------------------------------------
|
| 313 |
+
|
| 314 |
+
|
| 315 |
+
class MarlinForConditionalGeneration(Qwen3_5ForConditionalGeneration):
|
| 316 |
+
"""Marlin with ``.caption()`` and ``.find()`` convenience methods.
|
| 317 |
+
|
| 318 |
+
Inherits the full forward / generate / from_pretrained machinery from
|
| 319 |
+
:class:`transformers.Qwen3_5ForConditionalGeneration`; only adds two
|
| 320 |
+
helpers that wrap chat-template construction, generation, and the trained
|
| 321 |
+
output parsers.
|
| 322 |
+
|
| 323 |
+
Use it via the standard auto class::
|
| 324 |
+
|
| 325 |
+
from transformers import AutoModelForCausalLM
|
| 326 |
+
model = AutoModelForCausalLM.from_pretrained(
|
| 327 |
+
"NemoStation/Marlin-2B",
|
| 328 |
+
trust_remote_code=True,
|
| 329 |
+
dtype=torch.bfloat16,
|
| 330 |
+
device_map={"": "cuda"},
|
| 331 |
+
)
|
| 332 |
+
result = model.caption("video.mp4")
|
| 333 |
+
span = model.find("video.mp4", event="a person enters the room")
|
| 334 |
+
"""
|
| 335 |
+
|
| 336 |
+
# ------------------------------------------------------------------ utils
|
| 337 |
+
|
| 338 |
+
@property
|
| 339 |
+
def processor(self): # type: ignore[override]
|
| 340 |
+
"""Lazily-loaded :class:`~transformers.AutoProcessor` for this checkpoint.
|
| 341 |
+
|
| 342 |
+
Cached on the instance to avoid the expensive HF Hub lookup on every
|
| 343 |
+
call.
|
| 344 |
+
"""
|
| 345 |
+
cached = getattr(self, "_processor", None)
|
| 346 |
+
if cached is None:
|
| 347 |
+
from transformers import AutoProcessor
|
| 348 |
+
|
| 349 |
+
cached = AutoProcessor.from_pretrained(
|
| 350 |
+
self.config._name_or_path,
|
| 351 |
+
trust_remote_code=True,
|
| 352 |
+
)
|
| 353 |
+
self._processor = cached
|
| 354 |
+
return cached
|
| 355 |
+
|
| 356 |
+
def compile(self, *args: Any, **kwargs: Any) -> "MarlinForConditionalGeneration":
|
| 357 |
+
"""Optional ``torch.compile`` wrapper around the model.
|
| 358 |
+
|
| 359 |
+
Returns ``self`` so it chains naturally after ``from_pretrained``::
|
| 360 |
+
|
| 361 |
+
model = AutoModelForCausalLM.from_pretrained(...).compile()
|
| 362 |
+
|
| 363 |
+
All positional / keyword args are forwarded to ``torch.compile``.
|
| 364 |
+
"""
|
| 365 |
+
# ``torch.compile`` replaces the module's forward with a compiled
|
| 366 |
+
# version in-place; we still return self for fluent chaining.
|
| 367 |
+
torch.compile(self, *args, **kwargs)
|
| 368 |
+
return self
|
| 369 |
+
|
| 370 |
+
# ----------------------------------------------------------- core generate
|
| 371 |
+
|
| 372 |
+
def _generate_video(
|
| 373 |
+
self,
|
| 374 |
+
video_path: Union[str, os.PathLike],
|
| 375 |
+
prompt: str,
|
| 376 |
+
max_tokens: int,
|
| 377 |
+
*,
|
| 378 |
+
do_sample: bool = False,
|
| 379 |
+
temperature: float = 1.0,
|
| 380 |
+
top_p: float = 1.0,
|
| 381 |
+
) -> str:
|
| 382 |
+
"""Build a chat message with one video + one text turn and decode.
|
| 383 |
+
|
| 384 |
+
Returns the raw decoded string (with any ``<think>`` artifacts still
|
| 385 |
+
attached — callers are expected to run :func:`strip_thinking`).
|
| 386 |
+
"""
|
| 387 |
+
messages = [
|
| 388 |
+
{
|
| 389 |
+
"role": "user",
|
| 390 |
+
"content": [
|
| 391 |
+
{"type": "video", "video": str(video_path)},
|
| 392 |
+
{"type": "text", "text": prompt},
|
| 393 |
+
],
|
| 394 |
+
}
|
| 395 |
+
]
|
| 396 |
+
|
| 397 |
+
inputs = self.processor.apply_chat_template(
|
| 398 |
+
messages,
|
| 399 |
+
tokenize=True,
|
| 400 |
+
add_generation_prompt=True,
|
| 401 |
+
return_tensors="pt",
|
| 402 |
+
return_dict=True,
|
| 403 |
+
).to(self.device)
|
| 404 |
+
|
| 405 |
+
with torch.inference_mode():
|
| 406 |
+
out = self.generate(
|
| 407 |
+
**inputs,
|
| 408 |
+
max_new_tokens=max_tokens,
|
| 409 |
+
do_sample=do_sample,
|
| 410 |
+
temperature=temperature if do_sample else 1.0,
|
| 411 |
+
top_p=top_p if do_sample else 1.0,
|
| 412 |
+
)
|
| 413 |
+
|
| 414 |
+
# Strip the prompt prefix so we only return the model's continuation.
|
| 415 |
+
prompt_len = inputs["input_ids"].shape[1]
|
| 416 |
+
out = out[:, prompt_len:]
|
| 417 |
+
return self.processor.batch_decode(out, skip_special_tokens=True)[0]
|
| 418 |
+
|
| 419 |
+
# ---------------------------------------------------------------- caption
|
| 420 |
+
|
| 421 |
+
def caption(
|
| 422 |
+
self,
|
| 423 |
+
video_path: Union[str, os.PathLike],
|
| 424 |
+
*,
|
| 425 |
+
prompt: Optional[str] = None,
|
| 426 |
+
do_sample: bool = False,
|
| 427 |
+
temperature: float = 1.0,
|
| 428 |
+
top_p: float = 1.0,
|
| 429 |
+
max_new_tokens: int = 2048,
|
| 430 |
+
) -> CaptionResult:
|
| 431 |
+
"""Generate a dense caption for a video.
|
| 432 |
+
|
| 433 |
+
Parameters
|
| 434 |
+
----------
|
| 435 |
+
video_path:
|
| 436 |
+
Local path to a video file (mp4, webm, etc.).
|
| 437 |
+
prompt:
|
| 438 |
+
Override the canonical training prompt. Almost always leave at
|
| 439 |
+
``None``; diverging from training silently degrades quality.
|
| 440 |
+
do_sample:
|
| 441 |
+
If ``True``, switch to nucleus sampling. Defaults to greedy.
|
| 442 |
+
temperature, top_p:
|
| 443 |
+
Sampling params, only used when ``do_sample=True``.
|
| 444 |
+
max_new_tokens:
|
| 445 |
+
Generation cap. Default 2048 is enough for any dense caption the
|
| 446 |
+
model produces in practice.
|
| 447 |
+
|
| 448 |
+
Returns
|
| 449 |
+
-------
|
| 450 |
+
CaptionResult
|
| 451 |
+
Dict with keys ``caption``, ``scene``, ``events``, ``raw``.
|
| 452 |
+
"""
|
| 453 |
+
prompt_text = prompt if prompt is not None else CAPTION_PROMPT
|
| 454 |
+
raw = self._generate_video(
|
| 455 |
+
video_path,
|
| 456 |
+
prompt_text,
|
| 457 |
+
max_tokens=max_new_tokens,
|
| 458 |
+
do_sample=do_sample,
|
| 459 |
+
temperature=temperature,
|
| 460 |
+
top_p=top_p,
|
| 461 |
+
)
|
| 462 |
+
cleaned, scene, events = parse_caption(raw)
|
| 463 |
+
return CaptionResult(
|
| 464 |
+
caption=cleaned,
|
| 465 |
+
scene=scene,
|
| 466 |
+
events=events,
|
| 467 |
+
raw=raw,
|
| 468 |
+
)
|
| 469 |
+
|
| 470 |
+
# ------------------------------------------------------------------- find
|
| 471 |
+
|
| 472 |
+
def find(
|
| 473 |
+
self,
|
| 474 |
+
video_path: Union[str, os.PathLike],
|
| 475 |
+
event: str,
|
| 476 |
+
*,
|
| 477 |
+
prompt_template: Optional[str] = None,
|
| 478 |
+
do_sample: bool = False,
|
| 479 |
+
temperature: float = 1.0,
|
| 480 |
+
top_p: float = 1.0,
|
| 481 |
+
max_new_tokens: int = 64,
|
| 482 |
+
) -> FindResult:
|
| 483 |
+
"""Locate when a natural-language event occurs in a video.
|
| 484 |
+
|
| 485 |
+
Parameters
|
| 486 |
+
----------
|
| 487 |
+
video_path:
|
| 488 |
+
Local path to a video file.
|
| 489 |
+
event:
|
| 490 |
+
Free-form description of the event to locate, e.g.
|
| 491 |
+
``"a person enters the room"``. Inserted into the trained prompt
|
| 492 |
+
via the ``{event}`` placeholder.
|
| 493 |
+
prompt_template:
|
| 494 |
+
Override the canonical training prompt template. Must include a
|
| 495 |
+
``{event}`` placeholder. Almost always leave at ``None``.
|
| 496 |
+
do_sample:
|
| 497 |
+
If ``True``, switch to nucleus sampling. Defaults to greedy.
|
| 498 |
+
temperature, top_p:
|
| 499 |
+
Sampling params, only used when ``do_sample=True``.
|
| 500 |
+
max_new_tokens:
|
| 501 |
+
Output budget. 64 is plenty for the one-line trained format.
|
| 502 |
+
|
| 503 |
+
Returns
|
| 504 |
+
-------
|
| 505 |
+
FindResult
|
| 506 |
+
Dict with keys ``raw``, ``span`` and ``format_ok``.
|
| 507 |
+
|
| 508 |
+
Raises
|
| 509 |
+
------
|
| 510 |
+
ValueError
|
| 511 |
+
If ``event`` is empty or whitespace-only.
|
| 512 |
+
"""
|
| 513 |
+
event_str = (event or "").strip()
|
| 514 |
+
if not event_str:
|
| 515 |
+
raise ValueError("`event` must be a non-empty string")
|
| 516 |
+
|
| 517 |
+
template = prompt_template if prompt_template is not None else GROUNDING_PROMPT_TEMPLATE
|
| 518 |
+
prompt_text = template.format(event=event_str)
|
| 519 |
+
|
| 520 |
+
raw = self._generate_video(
|
| 521 |
+
video_path,
|
| 522 |
+
prompt_text,
|
| 523 |
+
max_tokens=max_new_tokens,
|
| 524 |
+
do_sample=do_sample,
|
| 525 |
+
temperature=temperature,
|
| 526 |
+
top_p=top_p,
|
| 527 |
+
)
|
| 528 |
+
cleaned, span = parse_span(raw)
|
| 529 |
+
return FindResult(
|
| 530 |
+
raw=cleaned,
|
| 531 |
+
span=span,
|
| 532 |
+
format_ok=span is not None,
|
| 533 |
+
)
|
preprocessor_config.json
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"size": {
|
| 3 |
+
"longest_edge": 16777216,
|
| 4 |
+
"shortest_edge": 65536
|
| 5 |
+
},
|
| 6 |
+
"patch_size": 16,
|
| 7 |
+
"temporal_patch_size": 2,
|
| 8 |
+
"merge_size": 2,
|
| 9 |
+
"image_mean": [
|
| 10 |
+
0.5,
|
| 11 |
+
0.5,
|
| 12 |
+
0.5
|
| 13 |
+
],
|
| 14 |
+
"image_std": [
|
| 15 |
+
0.5,
|
| 16 |
+
0.5,
|
| 17 |
+
0.5
|
| 18 |
+
],
|
| 19 |
+
"processor_class": "Qwen3VLProcessor",
|
| 20 |
+
"image_processor_type": "Qwen2VLImageProcessorFast"
|
| 21 |
+
}
|
processor_config.json
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"image_processor": {
|
| 3 |
+
"do_convert_rgb": true,
|
| 4 |
+
"do_normalize": true,
|
| 5 |
+
"do_rescale": true,
|
| 6 |
+
"do_resize": true,
|
| 7 |
+
"image_mean": [
|
| 8 |
+
0.5,
|
| 9 |
+
0.5,
|
| 10 |
+
0.5
|
| 11 |
+
],
|
| 12 |
+
"image_processor_type": "Qwen2VLImageProcessor",
|
| 13 |
+
"image_std": [
|
| 14 |
+
0.5,
|
| 15 |
+
0.5,
|
| 16 |
+
0.5
|
| 17 |
+
],
|
| 18 |
+
"merge_size": 2,
|
| 19 |
+
"patch_size": 16,
|
| 20 |
+
"resample": 3,
|
| 21 |
+
"rescale_factor": 0.00392156862745098,
|
| 22 |
+
"size": {
|
| 23 |
+
"longest_edge": 16777216,
|
| 24 |
+
"shortest_edge": 65536
|
| 25 |
+
},
|
| 26 |
+
"temporal_patch_size": 2
|
| 27 |
+
},
|
| 28 |
+
"processor_class": "Qwen3VLProcessor",
|
| 29 |
+
"video_processor": {
|
| 30 |
+
"do_convert_rgb": true,
|
| 31 |
+
"do_normalize": true,
|
| 32 |
+
"do_rescale": true,
|
| 33 |
+
"do_resize": true,
|
| 34 |
+
"do_sample_frames": true,
|
| 35 |
+
"fps": 2,
|
| 36 |
+
"image_mean": [
|
| 37 |
+
0.5,
|
| 38 |
+
0.5,
|
| 39 |
+
0.5
|
| 40 |
+
],
|
| 41 |
+
"image_std": [
|
| 42 |
+
0.5,
|
| 43 |
+
0.5,
|
| 44 |
+
0.5
|
| 45 |
+
],
|
| 46 |
+
"max_frames": 768,
|
| 47 |
+
"merge_size": 2,
|
| 48 |
+
"min_frames": 4,
|
| 49 |
+
"patch_size": 16,
|
| 50 |
+
"resample": 3,
|
| 51 |
+
"rescale_factor": 0.00392156862745098,
|
| 52 |
+
"return_metadata": false,
|
| 53 |
+
"size": {
|
| 54 |
+
"longest_edge": 25165824,
|
| 55 |
+
"shortest_edge": 4096
|
| 56 |
+
},
|
| 57 |
+
"temporal_patch_size": 2,
|
| 58 |
+
"video_processor_type": "Qwen3VLVideoProcessor"
|
| 59 |
+
}
|
| 60 |
+
}
|
tokenizer.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523
|
| 3 |
+
size 19989325
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_prefix_space": false,
|
| 3 |
+
"audio_bos_token": "<|audio_start|>",
|
| 4 |
+
"audio_eos_token": "<|audio_end|>",
|
| 5 |
+
"audio_token": "<|audio_pad|>",
|
| 6 |
+
"backend": "tokenizers",
|
| 7 |
+
"bos_token": null,
|
| 8 |
+
"clean_up_tokenization_spaces": false,
|
| 9 |
+
"eos_token": "<|im_end|>",
|
| 10 |
+
"errors": "replace",
|
| 11 |
+
"image_token": "<|image_pad|>",
|
| 12 |
+
"is_local": true,
|
| 13 |
+
"local_files_only": false,
|
| 14 |
+
"model_max_length": 262144,
|
| 15 |
+
"model_specific_special_tokens": {
|
| 16 |
+
"audio_bos_token": "<|audio_start|>",
|
| 17 |
+
"audio_eos_token": "<|audio_end|>",
|
| 18 |
+
"audio_token": "<|audio_pad|>",
|
| 19 |
+
"image_token": "<|image_pad|>",
|
| 20 |
+
"video_token": "<|video_pad|>",
|
| 21 |
+
"vision_bos_token": "<|vision_start|>",
|
| 22 |
+
"vision_eos_token": "<|vision_end|>"
|
| 23 |
+
},
|
| 24 |
+
"pad_token": "<|endoftext|>",
|
| 25 |
+
"pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
|
| 26 |
+
"processor_class": "Qwen3VLProcessor",
|
| 27 |
+
"split_special_tokens": false,
|
| 28 |
+
"tokenizer_class": "Qwen2Tokenizer",
|
| 29 |
+
"unk_token": null,
|
| 30 |
+
"video_token": "<|video_pad|>",
|
| 31 |
+
"vision_bos_token": "<|vision_start|>",
|
| 32 |
+
"vision_eos_token": "<|vision_end|>"
|
| 33 |
+
}
|