Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,33 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.model filter=lfs diff=lfs merge=lfs -text
+model.safetensors filter=lfs diff=lfs merge=lfs -text
+cookbooks/assets/grounding/depth_1.jpg filter=lfs diff=lfs merge=lfs -text
+cookbooks/assets/grounding/depth_2.jpg filter=lfs diff=lfs merge=lfs -text
+cookbooks/assets/grounding/dinner.jpg filter=lfs diff=lfs merge=lfs -text
+cookbooks/assets/grounding/macaron.jpg filter=lfs diff=lfs merge=lfs -text
+cookbooks/assets/grounding/peoples.jpg filter=lfs diff=lfs merge=lfs -text
+cookbooks/assets/grounding/test_example_point_01.png filter=lfs diff=lfs merge=lfs -text
+cookbooks/assets/grounding/test_example_point_02.png filter=lfs diff=lfs merge=lfs -text
+cookbooks/assets/ocr/example1_1.png filter=lfs diff=lfs merge=lfs -text
+cookbooks/assets/ocr/example1_2.png filter=lfs diff=lfs merge=lfs -text
+cookbooks/assets/ocr/example2.png filter=lfs diff=lfs merge=lfs -text
+cookbooks/assets/ocr/example4.png filter=lfs diff=lfs merge=lfs -text
+cookbooks/assets/ocr/example5.png filter=lfs diff=lfs merge=lfs -text
+cookbooks/assets/ocr/example6.png filter=lfs diff=lfs merge=lfs -text
+cookbooks/assets/ocr/example7.png filter=lfs diff=lfs merge=lfs -text
+cookbooks/assets/ocr/example7_2.png filter=lfs diff=lfs merge=lfs -text
+cookbooks/assets/reasoning/chemistry.png filter=lfs diff=lfs merge=lfs -text
+cookbooks/assets/reasoning/geometry.png filter=lfs diff=lfs merge=lfs -text
+cookbooks/assets/video/example_video_1.mp4 filter=lfs diff=lfs merge=lfs -text
+cookbooks/assets/video/example_video_2.mp4 filter=lfs diff=lfs merge=lfs -text
+cookbooks/assets/video/example_video_3.mp4 filter=lfs diff=lfs merge=lfs -text
+cookbooks/assets/video/example_video_4.mp4 filter=lfs diff=lfs merge=lfs -text
+cookbooks/assets/video/example_video_5.mp4 filter=lfs diff=lfs merge=lfs -text
+cookbooks/assets/video/example_video_6.mp4 filter=lfs diff=lfs merge=lfs -text
+cookbooks/assets/video/example_video_7.mp4 filter=lfs diff=lfs merge=lfs -text
+cookbooks/assets/video/example_video_8.mp4 filter=lfs diff=lfs merge=lfs -text
+cookbooks/assets/video/example_video_9.mp4 filter=lfs diff=lfs merge=lfs -text
+cookbooks/grounding.ipynb filter=lfs diff=lfs merge=lfs -text
+doc/technical_report.pdf filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,34 @@
+OPENPANGU MODEL LICENSE AGREEMENT VERSION 1.0
+
+This OPENPANGU MODEL LICENSE AGREEMENT VERSION 1.0 (the "Agreement") is a legal agreement between You and Huawei Technologies Co., Ltd. ("Huawei", "We" or "Us"), and it governs Your reproducing, use, modification, and distribution of openPangu as made available by Huawei under this Agreement. 
+
+By using, reproducing, modifying, distributing, performing or displaying any portion or element of openPangu, or otherwise accepting the terms of this Agreement, You agree to be bound by this Agreement.
+
+1.	Definitions. 
+1.1.	“openPangu” or “Model” means openPangu large language models and software, including trained model weights, parameters (including optimizer states), accompanying source code and scripts released under this Agreement. 
+1.2.	“Derivative Model” means all (1) modifications to the Model, (2) works based on the Model, and (3) any other derivative works of the Model. For clarity, information or content results from operating or otherwise using the Model is not a Derivative Model. 
+1.3.	“You” or “Your” means an individual or Legal Entity exercising permissions granted by this Agreement and/or using the Model for any purpose.
+1.4.	“Third Party” or “Third Parties” means individuals or legal entities that are not under common control with Us or You.
+
+2.	License Grant. Subject to Your full compliance with the terms and conditions of this Agreement, We hereby grant to You a perpetual, worldwide, non-exclusive, non-transferable, no-charge, royalty-free license (except as stated in Section 3) to use, reproduce, modify, and distribute the Model. 
+
+3.	Conditions for License Grant. You represent and warrant that You will not, access, download, install, run, deploy, integrate, modify, or otherwise use the Model, directly or indirectly, within the European Union.
+
+
+4.	Redistribution. 
+4.1.	If You distribute the Model or Derivative Model, You shall retain in Your distribution (1) a copy of this agreement, and (2) all copyright notices and other notices of origin included in the Model that are applicable to Your distribution.
+4.2.	Further, if You distribute or make available to Third Parties a product or service (including another AI model) based on the Model, You are required to (1) display the acknowledgement “Powered by openPangu” and (2) include a trademark notice “openPangu is a trademark of Huawei Technologies Co., Ltd.” on related webpages, user manuals, product documentations or other advertising materials mentioning features of the Model.
+4.3.	You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for Derivative Model made by You as a whole, provided Your use, reproduction, and distribution of the Model otherwise complies with the terms and conditions of this Agreement.
+
+5.	Ownership. We do not claim ownership to any information or content generated using the Model or Derivative Model that are made by You. You are solely responsible for evaluating the accuracy and appropriateness of such information or content for Your use case.
+
+6.	Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of Huawei, except as required for complying with Section 4.2.
+
+7.	Indemnity. You will indemnify and hold harmless Huawei from and against any claim by any third party arising out of or related to Your use or distribution of the Model or Derivative Model made by You (e.g. a violation against Section 3). For avoidance of doubt, “third party” in this clause include supervisory authorities.
+
+8.	THE MODEL IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE, NONINFRINGEMENT, ACCURACY, OR THE ABSENCE OF LATENT OR OTHER DEFECTS OR ERRORS, WHETHER OR NOT DISCOVERABLE, ALL TO THE GREATEST EXTENT PERMISSIBLE UNDER APPLICABLE LAW.
+
+9.	IN NO EVENT SHALL WE BE LIABLE TO YOU FOR ANY DAMAGES, INCLUDING, BUT NOT LIMITED TO ANY DIRECT, OR INDIRECT, SPECIAL OR CONSEQUENTIAL DAMAGES ARISING FROM YOUR USE OR INABILITY TO USE THE MODEL, IN WHOLE OR IN PART, NO MATTER HOW IT’S CAUSED OR THE LEGAL THEORY IT IS BASED ON, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
+
+
+END OF THE TERMS AND CONDITIONS

OPEN SOURCE SOFTWARE NOTICE

@@ -0,0 +1,635 @@
+OPEN SOURCE SOFTWARE NOTICE
+
+Please note we provide an open source software notice along with this product and/or this product firmware (in the following just “this product”). The open source software licenses are granted by the respective right holders. And the open source licenses prevail all other license information with regard to the respective open source software contained in the product, including but not limited to End User Software Licensing Agreement. This notice is provided on behalf of Huawei Technologies Co. Ltd. and any of its local subsidiaries which may have provided this product to you in your local country. 
+
+Warranty Disclaimer    
+THE OPEN SOURCE SOFTWARE IN THIS PRODUCT IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS.
+
+Copyright Notice and License Texts 
+
+Software: transformers 4.52.4
+Copyright notice: 
+Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
+
+License Text:
+----------------------------------------
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+Software: vllm 0.9.1
+Copyright notice: 
+Copyright 2025 The vLLM team.
+
+License Text:
+----------------------------------------
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+Software: vllm-ascend 0.9.1
+Copyright notice: 
+Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved.
+
+License Text:
+----------------------------------------
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -1,5 +1,109 @@
----
-license: other
-license_name: openpangu-model-license-agreement-version-1.0
-license_link: https://ai.gitcode.com/ascend-tribe/openPangu-VL-7B/blob/main/LICENSE
----
+# openPangu-VL-7B
+中文 | [English](README_EN.md) | [技术报告](doc/technical_report.pdf)
+
+## 1. 简介
+
+openPangu-VL-7B 是基于昇腾 NPU ，基于openPangu-Embedded-7B-V1.1语言基模和openPangu-ViT-600M视觉编码器训练的高效多模态模型。openPangu-VL-7B 训练了约 3T tokens，具备通用视觉对话、文档理解、目标定位与计数、视频理解、视觉高阶推理等能力。该模型为快思考模型。
+
+## 2. 模型架构
+
+|                               |   openPangu-VL-7B   |
+| :---------------------------: | :----------------: |
+| **语言模型** | |
+|       Architecture       |       Dense        |
+|     Parameters (Non-Embedding)    |         7B         |
+|     Number of Layers      |         34         |
+|     Hidden Dimension     |       12800        |
+|   Attention Mechanism    |     GQA      |
+| Number of Attention Heads | 32 for Q，8 for KV |
+|      Vocabulary Size      |        153k        |
+|      Context Length (Natively)       |        128k         |
+| **视觉编码器** | |
+|       Architecture       |       22 Window Attention + 4 Full Attention        |
+|     Number of Layers      |         26         |
+|     Attention Hidden Size    |     1536             |
+|     FFN Hidden Size    |     4608             |
+| Number of Attention Heads | 16 |
+|      Parameters      |        615M        |
+
+## 3. 测评结果
+
+|     测评集     |      openPangu-VL-7B |
+| :------------: | :--------: |
+|  **通用视觉对话**  |                     |  
+|   MMBench V1.1 DEV    |    86.5    |
+|     MMStar      |         70.1    |
+| RealWorldQA | 76.1    |
+|     AI2D     |      84.7    |
+|  **光学符号识别/文档理解**  |                     |   
+|    OCRBench    |        907    |
+|     TextVQA     |       85.1    |
+|     DocVQA     |       96.0    |
+|     ChartQA     |       88.3    |
+|     CharXiv dq/rq     |      83.9/54.3    |
+|  **视觉学科能力**  |                     |  
+| MMMU | 65.2 |
+| MMMU-Pro | 52.6 | 
+| MathVista | 75.0 |
+|  **目标定位/计数**  |                     |  
+| RefCOCO-avg  | 90.6    |
+|     ODinW-13      |        51.5   |
+|     CountBench      |        96.1   |
+|     Point-Bench      |        65.4   |
+|  **多图**  |                     |  
+| BLINK  | 63.3    |
+| MUIRBench  | 61.6    |
+|  **视频理解**  |                     |  
+| MVBench  | 74.0    |
+| VideoMME (w/o sub)  | 68.0    |
+| MLVU  | 76.9    |
+|  **语言能力**  |                     |  
+| MMLU-Pro  | 78.2    |
+| MMLU-Redux  | 87.3    |
+| GPQA-Diamond  | 65.2    |
+| C-Eval | 83.2 |
+| AIME25  | 36.5    |
+| Math-500  | 89.4    |
+| LiveCodeBenchV6 | 24.6 |
+| MBPP+ | 68.5 |
+| IFEval | 83.0 |
+
+**注：** 评测使用**vllm-ascend部署推理，系统prompt为空**。一般而言，图片最小分辨率设置为2304\*28\*28能获得最优的测评效果。（OCRBench中的极小图OCR除外，建议设置为不大于64\*28\*28。）具体prompt和分辨率设置参见[技术报告](doc/technical_report.pdf)附录。
+
+## 4. 部署和使用
+
+### vllm-ascend部署（推荐）
+- 使用vllm-ascend推理框架，参考[[vllm_ascend_for_openpangu_vl_7b](doc/vllm_ascend_for_openpangu_vl_7b.md)]进行服务部署。
+
+- 完成推理服务部署后，使用此脚本测试是否部署成功。
+```bash
+cd inference/vllm_ascend/examples; python quick_start.py
+```
+
+### 直接推理
+环境配置：
+- python==3.10
+- CANN==8.1.RC1
+```bash
+cd inference; pip install -r requirements.txt
+```
+
+推理：
+```bash
+cd inference; python generate.py
+```
+
+### 能力展示
+- 更多推理样例和能力展示，请参见`cookbooks`。
+
+## 5. 模型许可证
+除文件中对开源许可证另有约定外，openPangu-VL-7B 模型根据 OPENPANGU MODEL LICENSE AGREEMENT VERSION 1.0 授权，旨在允许使用并促进人工智能技术的进一步发展。有关详细信息，请参阅模型存储库根目录中的 [LICENSE](LICENSE) 文件。
+
+## 6. 免责声明
+由于 openPangu-VL-7B （“模型”）所依赖的技术固有的限制，以及人工智能生成的内容是由盘古自动生成的，华为无法对以下事项做出任何保证：
+- 该模型的输出通过AI算法自动生成，不能排除某些信息可能存在缺陷、不合理或引起不适的可能性，生成的内容不代表华为的态度或立场； 
+- 无法保证该模型100%准确、可靠、功能齐全、及时、安全、无错误、不间断、持续稳定或无任何故障； 
+- 该模型的输出内容不构成任何建议或决策，也不保证生成的内容的真实性、完整性、准确性、及时性、合法性、功能性或实用性。生成的内容不能替代医疗、法律等领域的专业人士回答您的问题。生成的内容仅供参考，不代表华为的任何态度、立场或观点。您需要根据实际情况做出独立判断，华为不承担任何责任。
+
+## 7. 反馈
+如果有任何意见和建议，请提交issue或联系[openPangu@huawei.com](url)。

README_EN.md

@@ -0,0 +1,113 @@
+# openPangu-VL-7B
+
+[中文](README.md) | English | [Technical Report](doc/technical_report.pdf)
+
+## 1. Introduction
+The openPangu-VL-7B is an efficient multimodal model based on the Ascend NPU, trained using the openPangu-Embedded-7B-V1.1 base language model and the openPangu-ViT-600M vision encoder. The openPangu-VL-7B has been trained on approximately 3T tokens and possesses capabilities such as general Visual Question Answering, chart and document understanding, object grounding and counting, video understanding, and advanced visual reasoning. This model is designed for fast thinking mode.
+
+
+## 2. Model Architecture
+
+|                               |   openPangu-VL-7B   |
+| :---------------------------: | :----------------: |
+| **LLM** | |
+|       Architecture       |       Dense        |
+|     Parameters (Non-Embedding)    |         7B         |
+|     Number of Layers      |         34         |
+|     Hidden Dimension     |       12800        |
+|   Attention Mechanism    |     GQA      |
+| Number of Attention Heads | 32 for Q，8 for KV |
+|      Vocabulary Size      |        153k        |
+|      Context Length (Natively)       |        128k         |
+| **Vision Encoder** | |
+|       Architecture       |       22 Window Attention + 4 Full Attention        |
+|     Number of Layers      |         26         |
+|     Attention Hidden Size    |     1536             |
+|     FFN Hidden Size    |     4608             |
+| Number of Attention Heads | 16 |
+|      Parameters      |        615M        |
+
+
+## 3. Results
+
+|     Benchmark     |      openPangu-VL-7B |
+| :------------: | :--------: |
+|  **General VQA**  |                     |  
+|   MMBench V1.1 DEV    |    86.5    |
+|     MMStar      |         70.1    |
+| RealWorldQA | 76.1    |
+|     AI2D     |      84.7    |
+|  **OCR & Chart/Document Understanding**  |   |
+|    OCRBench    |        907    |
+|     TextVQA     |       85.1    |
+|     DocVQA     |       96.0    |
+|     ChartQA     |       88.3    |
+|     CharXiv dq/rq     |      83.9/54.3    |
+|  **STEM**  |                     |  
+| MMMU | 65.2 |
+| MMMU-Pro | 52.6 | 
+| MathVista | 75.0 |
+|  **Object Grounding/Counting**  |  |
+| RefCOCO-avg  | 90.6    |
+|     ODinW-13      |        51.5   |
+|     CountBench      |        96.1   |
+|     Point-Bench      |        65.4   |
+|  **Multi-Image**  |                     |  
+| BLINK  | 63.3    |
+| MUIRBench  | 61.6    |
+|  **Video Understanding**  |                     |  
+| MVBench  | 74.0    |
+| VideoMME (w/o sub)  | 68.0    |
+| MLVU  | 76.9    |
+|  **Text-Centric Benchmark**  |                     | 
+| MMLU-Pro  | 78.2    |
+| MMLU-Redux  | 87.3    |
+| GPQA-Diamond  | 65.2    |
+| C-Eval | 83.2 |
+| AIME25  | 36.5    |
+| Math-500  | 89.4    |
+| LiveCodeBenchV6 | 24.6 |
+| MBPP+ | 68.5 |
+| IFEval | 83.0 |
+
+**Note:** The evaluation is conducted with **vllm-ascend deploy** and **the system prompt remains empty**. Generally, setting the minimum resolution to 2304\*28\*28 can yield the best evaluation results. (Except for the extremely small image OCR in OCRBench, it is recommended to set the resolution to no more than 64\*28\*28.) Detailed settings for different benchmarks can be found in [Technical Report](doc/technical_report.pdf).
+
+## 4. Deployment
+
+### vllm-ascend deploy (recommended)
+- vllm-ascend：please refer to [[vllm_ascend_for_openpangu_vl_7b](doc/vllm_ascend_for_openpangu_vl_7b_EN.md)] to deploy the inference serving.
+
+- After finish deploying, you can test the api with the following script.
+```bash
+cd inference/vllm_ascend/examples; python quick_start.py
+```
+
+### Direct inference
+
+Environment:
+- python==3.10
+- CANN==8.1.RC1
+```bash
+cd inference; pip install -r requirements.txt
+```
+
+Inference:
+```bash
+cd inference; python generate.py
+```
+
+### Model abilities
+- For more examples and demomstrations of model abilities, please refer to `cookbooks`.
+
+## 5. Model License
+Unless otherwise noted, the openPangu-VL-7B model is licensed under the terms and conditions of OPENPANGU MODEL LICENSE AGREEMENT VERSION 1.0, which is intended to be used permissively and enable the further development of artificial intelligence technologies. Please refer to the [LICENSE](LICENSE) file located in the root directory of the model repository for details.
+
+## 6. Disclaimer
+Due to the technical limitations inherent in the technology on which the openPangu-VL-7B (“Model”) relies and the fact that the artificial intelligence generated content is automatically produced by Model, Huawei cannot make any guarantees regarding the following matters:
+
+- The output of this Model is automatically generated via AI algorithms, it does not rule out the possibility that some of the information may be flawed, unreasonable, or cause discomfort, and the generated content does not represent Huawei's attitude or standpoint; 
+- There is no guarantee that this Model is 100% accurate, reliable, functional, timely, secure and safety, error-free, uninterrupted, continuously stable, or free of any faults; 
+- The output of this Model does not constitute any advices or decisions for you, and it does not guarantee the authenticity, completeness, accuracy, timeliness, legality, functionality, or practicality of the generated content. The generated content cannot replace professionals in medical, legal, and other fields in answering your questions. The generated content is for your reference only and does not represent any attitude, standpoint, or position of Huawei. You need to make independent judgments based on your actual situation, and Huawei does not assume any responsibilities.
+
+## 7. Contact
+If you have any question, please raise an issue or contact us at [openPangu@huawei.com](url).

chat_template.json

@@ -0,0 +1,3 @@
+{
+    "chat_template": "{% macro role_name(role) -%}{%- if role == 'assistant' -%}助手{%- elif role == 'system' -%}系统{%- elif role == 'user' -%}用户{%- elif role == 'tool' -%}工具{%- elif role == 'function' -%}方法{%- else -%}{{ role }}{%- endif -%}{%- endmacro %}{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message.role != 'system' %}<s>[unused9]系统：[unused10]{% endif %}{% if loop.first and message.role == 'system' %}<s>{% endif %}[unused9]{{ role_name(message.role) }}：{% if message.content is string %}{{ message.content }}[unused10]{% else %}{% for content in message.content %}{% set ctype = content.type|default('') %}{% set is_img = (ctype == 'image') or ('image' in content) or ('image_url' in content) %}{% set is_vid = (ctype == 'video') or ('video' in content) %}{% if is_img %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}图片 {{ image_count.value }}: {% endif %}[unused18][unused19][unused20]{% elif is_vid %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}视频 {{ video_count.value }}: {% endif %}[unused18][unused32][unused20]{% elif content.text is defined %}{{ content.text }}{% endif %}{% endfor %}[unused10]{% endif %}{% endfor %}{% if add_generation_prompt %}[unused9]助手：{% endif %}"
+}

config.json

@@ -0,0 +1,69 @@
+{
+  "architectures": [
+    "OpenPanguVLForConditionalGeneration"
+  ],
+  "bias": true,
+  "auto_map": {
+    "AutoConfig": "configuration_openpangu_vl.OpenPanguVLConfig",
+    "AutoModel": "modeling_openpangu_vl.OpenPanguVLModel",
+    "AutoModelForCausalLM": "modeling_openpangu_vl.OpenPanguVL"
+  },
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 45892,
+  "vision_start_token_id": 45986,
+  "vision_end_token_id": 46007,
+  "image_token_id": 46005,
+  "video_token_id": 144208,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 12800,
+  "max_position_embeddings": 131072,
+  "max_window_layers": 34,
+  "model_type": "openpangu_vl",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 34,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 64000000.0,
+
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.52.4",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vision_config": {
+    "rms_norm_eps": 1e-06,
+    "depth": 26,
+    "hidden_act": "gelu",
+    "hidden_size": 1536,
+    "intermediate_size": 4608,
+    "num_heads": 16,
+    "in_chans": 3,
+    "out_hidden_size": 3584,
+    "patch_size": 14,
+    "spatial_merge_size": 2,
+    "spatial_patch_size": 14,
+    "window_size": 112,
+    "fullatt_block_indexes": [
+      5,
+      12,
+      19,
+      25
+    ],
+    "tokens_per_second": 2,
+    "temporal_patch_size": 2
+  },
+  "rope_scaling": {
+    "rope_type": "default",
+    "type": "default",
+    "mrope_section": [
+      10,
+      27,
+      27
+    ],
+    "mrope_interleaved": true
+  },
+  "vocab_size": 153376
+}

configuration_openpangu_vl.py

@@ -0,0 +1,148 @@
+# coding=utf-8
+# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved.
+# Copyright 2025 The HuggingFace Inc. team.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.modeling_rope_utils import rope_config_validation
+
+
+class OpenPanguVLVisionConfig(PretrainedConfig):
+    model_type = "openpangu_vl"
+    base_config_key = "vision_config"
+
+    def __init__(
+        self,
+        depth=26,
+        num_heads=16,
+        rms_norm_eps=1e-06,
+        hidden_size=1536,
+        hidden_act="gelu",
+        intermediate_size=4608,
+        out_hidden_size=3584,
+        in_chans=3,
+        patch_size=14,
+        spatial_merge_size=2,
+        window_size=112,
+        fullatt_block_indexes=[5, 12, 19, 25],
+        tokens_per_second=2,
+        temporal_patch_size=2,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.depth = depth
+        self.num_heads = num_heads
+        self.rms_norm_eps = rms_norm_eps
+        self.hidden_size = hidden_size
+        self.hidden_act = hidden_act
+        self.intermediate_size = intermediate_size
+        self.out_hidden_size = out_hidden_size
+        self.in_channels = in_chans
+        self.patch_size = patch_size
+        self.spatial_merge_size = spatial_merge_size
+        self.window_size = window_size
+        self.fullatt_block_indexes = fullatt_block_indexes
+        self.tokens_per_second = tokens_per_second
+        self.temporal_patch_size = temporal_patch_size
+
+
+class OpenPanguVLTextConfig(PretrainedConfig):
+
+    model_type = "openpangu_vl_text"
+    base_config_key = "text_config"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+        self,
+        num_hidden_layers=34,
+        num_attention_heads=32,
+        num_key_value_heads=8,
+        rms_norm_eps=1e-05,
+        hidden_size=4096,
+        hidden_act="silu",
+        intermediate_size=12800,
+        initializer_range=0.02,
+        tie_word_embeddings=False,
+        use_sliding_window=False,
+        sliding_window=None,
+        max_window_layers=80,
+        vocab_size=153376,
+        max_position_embeddings=32768,
+        use_cache=True,
+        rope_theta=64000000.0,
+        attention_dropout=0.0,
+        rope_scaling=None,
+        image_token_id=None,
+        video_token_id=None,
+        **kwargs,
+    ):
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.num_key_value_heads = num_key_value_heads
+        self.rms_norm_eps = rms_norm_eps
+        self.hidden_size = hidden_size
+        self.hidden_act = hidden_act
+        self.intermediate_size = intermediate_size
+        self.initializer_range = initializer_range
+        self.use_sliding_window = use_sliding_window
+        self.sliding_window = sliding_window
+        self.max_window_layers = max_window_layers
+        self.vocab_size = vocab_size
+        self.max_position_embeddings = max_position_embeddings
+        self.use_cache = use_cache
+        self.rope_theta = rope_theta
+        self.attention_dropout = attention_dropout
+        self.rope_scaling = rope_scaling
+
+        if self.rope_scaling is not None and "type" in self.rope_scaling:
+            if self.rope_scaling["type"] == "mrope":
+                self.rope_scaling["type"] = "default"
+            self.rope_scaling["rope_type"] = self.rope_scaling["type"]
+        rope_config_validation(self, ignore_keys={"mrope_section", "mrope_interleaved"})
+        self.image_token_id = image_token_id
+        self.video_token_id = video_token_id
+
+        super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
+
+
+class OpenPanguVLConfig(PretrainedConfig):
+
+    model_type = "openpangu_vl"
+    sub_configs = {"vision_config": OpenPanguVLVisionConfig, "text_config": OpenPanguVLTextConfig}
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+        self,
+        text_config=None,
+        vision_config=None,
+        image_token_id=46005,
+        video_token_id=144208,
+        **kwargs,
+    ):
+        if isinstance(vision_config, dict):
+            self.vision_config = self.sub_configs["vision_config"](**vision_config)
+        elif vision_config is None:
+            self.vision_config = self.sub_configs["vision_config"]()
+
+        if isinstance(text_config, dict):
+            self.text_config = self.sub_configs["text_config"](**text_config)
+        elif text_config is None:
+            self.text_config = self.sub_configs["text_config"](**kwargs)
+
+        self.image_token_id = image_token_id
+        self.video_token_id = video_token_id
+
+        super().__init__(**kwargs)
+
+
+__all__ = ["OpenPanguVLConfig", "OpenPanguVLTextConfig"]

cookbooks/assets/video/example_video_1.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a39adc894b3e3b3a2031ea2d0f957ddfad34362b76397321b11bfefa62c40de
+size 12700201

cookbooks/assets/video/example_video_2.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a90a370b6a768b10c3b003faf2b36c0e3fb6a6c6afecd2de7f60678351d7bed
+size 14892846

cookbooks/assets/video/example_video_3.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a005fdc45c1ea3e8fccb151a4275bb9bde53a0ef7034ef3f52688dbc22b782d3
+size 26385309

cookbooks/assets/video/example_video_4.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc2a87bf37fee1c608eb81856d0201ff3716de4e6ce972c8dac3c468d46f8a5c
+size 8082173

cookbooks/assets/video/example_video_5.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0613a6189fb546dbc35085fb48efb909a30898bb674476391a6179a202dbeef7
+size 4770828

cookbooks/assets/video/example_video_6.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2bd56ba8ef4d1eb1e9d3744bbe2d97ff962126cfc9586796a0447e54085a00ca
+size 19854080

cookbooks/assets/video/example_video_7.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3020815b636cbffde7a3504d6f9d8589d79e80a5f839dfb6e65a4a76422c7cc
+size 23466837

cookbooks/assets/video/example_video_8.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf41e2c4a51111ddcc0ac04c46804932d9ae0c35d0bdd37c64233e9c171cf961
+size 25891046

cookbooks/assets/video/example_video_9.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:121f77f79e2898bba71fa0a51810567e9ea4f86364d71acd7ed3c3672dcd0ff4
+size 16076994

cookbooks/grounding.ipynb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68a1d3e9b01755f894b65b86393200858304fea466a1ada8ac09cd11ec1414b8
+size 10670893

cookbooks/video.ipynb

@@ -0,0 +1,329 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "96bbbe85-9ff1-436c-b3ed-bf01b891c3e8",
+   "metadata": {},
+   "source": [
+    "### Video understanding with openPangu-VL-7B"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0dc96c01-6327-4495-9c2d-02b525a84cf2",
+   "metadata": {},
+   "source": [
+    "#### Inference with api"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "d6a397f0-8ca9-4ad3-b2c7-30c58a52350e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "import base64\n",
+    "import os\n",
+    "import requests\n",
+    "import json\n",
+    "from PIL import Image, ImageDraw, ImageFont\n",
+    "import ast\n",
+    "import re\n",
+    "def encode_image_to_base64(img_path):\n",
+    "    \"\"\"\n",
+    "    :param img_path: image path\n",
+    "    :return: Base64 encoding\n",
+    "    \"\"\"\n",
+    "    try:\n",
+    "        with open(img_path, 'rb') as img_file:\n",
+    "            img_data = img_file.read()\n",
+    "        base64_str = base64.b64encode(img_data).decode('utf-8')\n",
+    "        return base64_str\n",
+    "    except Exception as e:\n",
+    "        print(f\"Encoding Error: {e}\")\n",
+    "        return None\n",
+    "\n",
+    "def encode_video_to_base64(video_path):\n",
+    "    \"\"\"\n",
+    "    :param video_path: video path\n",
+    "    :return: Base64 encoding \n",
+    "    \"\"\"\n",
+    "    try:\n",
+    "        with open(video_path, 'rb') as video_file:\n",
+    "            video_data = video_file.read()\n",
+    "        base64_str = base64.b64encode(video_data).decode('utf-8')\n",
+    "        return base64_str\n",
+    "    except Exception as e:\n",
+    "        print(f\"Encoding Error: {e}\")\n",
+    "        return None\n",
+    "    \n",
+    "def replace_grouding_tokens(response):\n",
+    "    return response.replace(\"[unused23]\",\"\").replace(\"[unused24]\",\":\").replace(\"[unused21]\",\"\").replace(\"[unused22]\",\"\\n\")\n",
+    "\n",
+    "def extract_with_regex(data):\n",
+    "    text_list = []\n",
+    "    coordinates_list = []\n",
+    "    \n",
+    "    # 使用正则表达式匹配模式\n",
+    "    pattern = r'([^:]+):\\((\\d+),(\\d+)\\),\\((\\d+),(\\d+)\\)'\n",
+    "    matches = re.findall(pattern, data)\n",
+    "    \n",
+    "    for match in matches:\n",
+    "        text_list.append(match[0])\n",
+    "        coord1 = (int(match[1]), int(match[2]))\n",
+    "        coord2 = (int(match[3]), int(match[4]))\n",
+    "        coordinates_list.append([coord1, coord2])\n",
+    "    \n",
+    "    return text_list, coordinates_list\n",
+    "\n",
+    "def plot_text_bounding_boxes(image_path, response):\n",
+    "    \"\"\"\n",
+    "    Plots bounding boxes on an image.\n",
+    "\n",
+    "    Args:\n",
+    "        image_path: The path to the image file.\n",
+    "        response: model response without grouding tokens.\n",
+    "    \"\"\"\n",
+    "    \n",
+    "    text_list, coordinates_list = extract_with_regex(response)\n",
+    "    img = Image.open(image_path)\n",
+    "    width, height = img.size\n",
+    "    draw = ImageDraw.Draw(img)\n",
+    "    font = ImageFont.truetype(\"NotoSansCJK-Regular.ttc\", size=10)\n",
+    "\n",
+    "    for text, coord in zip(text_list,coordinates_list):\n",
+    "        color = 'green'\n",
+    "\n",
+    "        abs_y1 = int(coord[0][1]/999 * height)\n",
+    "        abs_x1 = int(coord[0][0]/999 * width)\n",
+    "        abs_y2 = int(coord[1][1]/999 * height)\n",
+    "        abs_x2 = int(coord[1][0]/999 * width)\n",
+    "\n",
+    "        if abs_x1 > abs_x2:\n",
+    "            abs_x1, abs_x2 = abs_x2, abs_x1\n",
+    "\n",
+    "        if abs_y1 > abs_y2:\n",
+    "            abs_y1, abs_y2 = abs_y2, abs_y1\n",
+    "\n",
+    "        draw.rectangle(\n",
+    "            ((abs_x1, abs_y1), (abs_x2, abs_y2)), outline=color, width=1\n",
+    "        )\n",
+    "        draw.text((abs_x1, abs_y2), text, fill=color, font=font)\n",
+    "    img.show()\n",
+    "\n",
+    "\n",
+    "def infer_image_with_api(image_path, prompt):\n",
+    "    url = \"http://127.0.0.1:8000/v1/chat/completions\"\n",
+    "    base64_image = encode_image_to_base64(image_path)\n",
+    "    payload = json.dumps({\n",
+    "      \"messages\": [\n",
+    "          {\n",
+    "                \"role\": \"user\",\n",
+    "                \"content\": [\n",
+    "                    {\"type\": \"image_url\", \"image_url\": {\"url\": f\"data:image/jpg;base64,{base64_image }\"}},\n",
+    "                    {\"type\": \"text\", \"text\": prompt},     \n",
+    "                ]\n",
+    "          }\n",
+    "      ],    \n",
+    "      \"model\": \"pangu_vl\",\n",
+    "      \"max_tokens\": 2048,\n",
+    "      \"temperature\": 0,\n",
+    "      \"stream\": False,\n",
+    "\n",
+    "    })\n",
+    "\n",
+    "    headers = {\n",
+    "      'Content-Type': 'application/json'\n",
+    "    }\n",
+    "\n",
+    "    response = requests.request(\"POST\", url, headers=headers, data=payload)\n",
+    "    return json.loads(response.text)[\"choices\"][0][\"message\"][\"content\"]\n",
+    "\n",
+    "def infer_video_with_api(video_path, prompt):\n",
+    "    url = \"http://127.0.0.1:8000/v1/chat/completions\"\n",
+    "    base64_str_video = encode_video_to_base64(video_path)\n",
+    "    payload = json.dumps({\n",
+    "      \"messages\": [\n",
+    "          {\n",
+    "                \"role\": \"user\",\n",
+    "                \"content\": [\n",
+    "                    {\"type\": \"video_url\", \"video_url\": {\"url\": f\"data:video/mp4;base64,{base64_str_video }\"}},\n",
+    "                    {\"type\": \"text\", \"text\": prompt}, \n",
+    "                ]\n",
+    "          }\n",
+    "      ],    \n",
+    "      \"model\": \"pangu_vl\",\n",
+    "      \"max_tokens\": 2048,\n",
+    "      \"temperature\": 0,\n",
+    "      \"stream\": False,\n",
+    "\n",
+    "    })\n",
+    "\n",
+    "    headers = {\n",
+    "      'Content-Type': 'application/json'\n",
+    "    }\n",
+    "\n",
+    "    response = requests.request(\"POST\", url, headers=headers, data=payload)\n",
+    "    return json.loads(response.text)[\"choices\"][0][\"message\"][\"content\"]\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4f9a1b5c",
+   "metadata": {},
+   "source": [
+    "#### Video description"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "54f4630a",
+   "metadata": {},
+   "outputs": [
+    {
+     "output_type": "stream",
+     "name": "stdout",
+     "text": [
+      "The video begins with a person standing next to a small wooden table. On the table, there is a black and silver electric kettle and a pink mug with a white bottle inside it. The person is wearing a pink and white striped crop top and black pants with white stripes on the sides. They are holding a black laptop in their hands, showing the keyboard side to the camera. The person then closes the laptop and sets it aside. Next, they pick up a smartphone, showing the back side to the camera, and then the front side. After that, they pick up a stainless steel pot with a lid, showing it to the camera. They then set the pot aside and pick up a large kitchen knife, showing it to the camera. Finally, they pick up a pink garment, possibly a dress or a shirt, and show it to the camera. Throughout the video, the person is standing in the same position, and the background remains consistent with a white wall and a window with sheer curtains.\n",
+      "\n",
+      "\n",
+      "视频开始时，画面展示了一张带有黄色和灰色花纹的桌布，上面放置着一个透明的玻璃杯，杯中装有深色液体，旁边是一个黑色的遥控器。画面中央有一只手正在用黑色的笔在一张白色的便签纸上写字。便签纸上有水平的蓝色线条。手正在书写字母“a”，随后继续书写，最终写出了“art”这个词。写完“art”后，手将笔放下，便签纸被拿起并展示给镜头，可以看到“art”这个词清晰地写在纸上。接着，手将便签纸放回原位，然后画面中出现了一个绿色的字母“t”形状的纸片，被放置在“art”这个词的下方。随后，一个蓝色的字母“a”形状的纸片被放置在绿色“t”旁边，最后，一个红色的字母“c”形状的纸片被放置在蓝色“a”旁边，形成了“t a c”这三个字母。整个过程中，背景和物品的位置没有发生变化。\n"
+     ]
+    }
+   ],
+   "source": [
+    "video_path = \"assets/video/example_video_1.mp4\"\n",
+    "prompt = \"Please describe the video in detail.\"\n",
+    "response = infer_video_with_api(video_path, prompt)\n",
+    "print(response)\n",
+    "\n",
+    "print(\"\")\n",
+    "print(\"\")\n",
+    "\n",
+    "video_path = \"assets/video/example_video_2.mp4\"\n",
+    "prompt = \"请使用中文详细描述该视频。\"\n",
+    "response = infer_video_with_api(video_path, prompt)\n",
+    "print(response)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "594ab293",
+   "metadata": {},
+   "source": [
+    "#### Temporal Action Localization"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "c177b6f1",
+   "metadata": {},
+   "outputs": [
+    {
+     "output_type": "stream",
+     "name": "stdout",
+     "text": [
+      "[unused14]0.0 - 4.2 seconds[unused15], A person is putting objects on the floor.[unused14]4.2 - 9.2 seconds[unused15], They put a ball, a cup, and a box on the floor.[unused14]9.2 - 20.4 seconds[unused15], They close the door.\n"
+     ]
+    }
+   ],
+   "source": [
+    "video_path = \"assets/video/example_video_3.mp4\"\n",
+    "prompt = \"Please locate all the actions in the video and output the corresponding start and end timestamps.\"\n",
+    "response = infer_video_with_api(video_path, prompt)\n",
+    "print(response)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ba6ecf29",
+   "metadata": {},
+   "source": [
+    "#### Video General QA"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "66aef615",
+   "metadata": {},
+   "outputs": [
+    {
+     "output_type": "stream",
+     "name": "stdout",
+     "text": [
+      "视频中出现了两次五角星。第一次是在视频开始时，双手合拢后张开手掌，可以看到左手掌心有一个五角星图案。第二次是在视频接近尾声时，双手再次合拢后张开手掌，可以看到左手掌心又出现了一个五角星图案。\n",
+      "\n",
+      "\n",
+      "他穿着短袖。\n",
+      "\n",
+      "\n",
+      "The second item placed on the table is a clear glass mug with floral designs.\n",
+      "\n",
+      "\n",
+      "The word 'SUGAR' is written on the bottle on the left.\n"
+     ]
+    }
+   ],
+   "source": [
+    "\n",
+    "video_path = \"assets/video/example_video_4.mp4\"\n",
+    "prompt = \"视频中出现了几次五角星？在什么位置？\"\n",
+    "response = infer_video_with_api(video_path, prompt)\n",
+    "print(response)\n",
+    "\n",
+    "print(\"\")\n",
+    "print(\"\")\n",
+    "\n",
+    "video_path = \"assets/video/example_video_5.mp4\"\n",
+    "prompt = \"他穿着长袖还是短袖？\"\n",
+    "response = infer_video_with_api(video_path, prompt)\n",
+    "print(response)\n",
+    "\n",
+    "print(\"\")\n",
+    "print(\"\")\n",
+    "\n",
+    "video_path = \"assets/video/example_video_6.mp4\"\n",
+    "prompt = \"What is the second item placed on the table by the person in the video?\"\n",
+    "response = infer_video_with_api(video_path, prompt)\n",
+    "print(response)\n",
+    "\n",
+    "print(\"\")\n",
+    "print(\"\")\n",
+    "\n",
+    "video_path = \"assets/video/example_video_7.mp4\"\n",
+    "prompt = \"What words are written on the bottle on the left?\"\n",
+    "response = infer_video_with_api(video_path, prompt)\n",
+    "print(response)\n",
+    "\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "name": "python3",
+   "display_name": "Python 3.11.13 64-bit"
+  },
+  "language_info": {
+   "name": "",
+   "version": "3.11.13"
+  },
+  "interpreter": {
+   "hash": "32cd74d773da33073fa79be3834e1375de230f7ebe90dceaad933455c7929c10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

doc/technical_report.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d9b7f7fd8719aafb014cda4ca4358fd1bb778ce08501182c2c11bb084a0b2bd
+size 1255829

doc/vllm_ascend_for_openpangu_vl_7b.md

@@ -0,0 +1,227 @@
+## openPangu-VL-7B  在[vllm-ascend](https://github.com/vllm-project/vllm-ascend)部署指导文档
+
+### 部署环境要求
+
+Atlas 800T A2(64GB) 1、2、4、8卡均可部署openPangu-VL-7B
+
+### 镜像构建和启动
+
+选用vllm-ascend社区镜像v0.9.1
+
+拉取方式如下：
+
+```bash
+docker pull quay.io/ascend/vllm-ascend:v0.9.1
+```
+
+以下操作需在每个节点都执行。
+启动镜像。
+
+```bash
+# Update the vllm-ascend image
+export IMAGE=quay.io/ascend/vllm-ascend:v0.9.1  # Use correct image id
+export NAME=vllm-ascend  # Custom docker name
+
+# Run the container using the defined variables
+# Note if you are running bridge network with docker, Please expose available ports for multiple nodes communication in advance
+# To prevent device interference from other docker containers, add the argument "--privileged"
+docker run --rm \
+--name $NAME \
+--network host \
+--ipc=host \
+--device /dev/davinci0 \
+--device /dev/davinci1 \
+--device /dev/davinci2 \
+--device /dev/davinci3 \
+--device /dev/davinci4 \
+--device /dev/davinci5 \
+--device /dev/davinci6 \
+--device /dev/davinci7 \
+--device /dev/davinci_manager \
+--device /dev/devmm_svm \
+--device /dev/hisi_hdc \
+-v /usr/local/dcmi:/usr/local/dcmi \
+-v /usr/local/Ascend/driver/tools/hccn_tool:/usr/local/Ascend/driver/tools/hccn_tool \
+-v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
+-v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
+-v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
+-v /etc/ascend_install.info:/etc/ascend_install.info \
+-v /mnt/sfs_turbo/.cache:/root/.cache \
+-it $IMAGE bash
+```
+
+需要保证模型权重和本项目代码可在容器中访问。如果未进入容器，需以root用户进入容器。
+```bash
+docker exec -itu root $NAME /bin/bash
+```
+
+### PD混部推理
+
+示例启动脚本：`LOAD_CKPT_DIR=xxx bash examples/start_serving_openpangu_vl_7b.sh`。该启动脚本为8卡推理（变量TENSOR_PARALLEL_SIZE_LOCAL=8）。拉起服务后，可向首节点（主节点）发送请求。
+
+### 发请求测试
+
+服务启动后，可发送测试请求。推荐使用示例中的system prompt。
+
+推理示例：图片+文字
+
+```python
+import json
+import base64
+import os
+import requests
+import json
+
+def encode_image_to_base64(img_path, img_name):
+    #load image to base64
+    try:
+        with open(os.path.join(img_path, img_name), 'rb') as img_file:
+            img_data = img_file.read()
+        base64_str = base64.b64encode(img_data).decode('utf-8')
+        return base64_str
+    except Exception as e:
+        print(f"image load failed: {e}")
+        return None
+
+base64_image = encode_image_to_base64("/image_path", "image_name.jpg")
+
+
+payload_image_example = json.dumps({
+  "messages": [
+      {
+            "role": "system",
+            "content": [
+                {"type": "text", "text": "你是华为公司开发的多模态大模型，名字是openPangu-VL-7B。你能够处理文本和视觉模态的输入，并给出文本输出。"},     
+            ]
+      },
+      {
+            "role": "user",
+            "content": [
+                {"type": "image_url", "image_url": {"url": f"data:image/jpg;base64,{base64_image}"}},
+                {"type": "text", "text": "Please describe this picture."}, 
+            ]
+      }
+  ],    
+  "model": "pangu_vl",
+  "max_tokens": 500,
+  "temperature": 1.0,
+  "stream": False,
+})
+
+
+url = "http://127.0.0.1:8000/v1/chat/completions"
+headers = {
+  'Content-Type': 'application/json'
+}
+
+response_image_example = requests.request("POST", url, headers=headers, data=payload_image_example)
+print(f"the response of image example is {response_image_example.text}")
+
+```
+
+
+推理示例：视频+文字
+
+```python
+import json
+import base64
+import os
+import requests
+import json
+
+def encode_video_to_base64(video_path, video_name):
+    #load video to base64
+    try:
+        with open(os.path.join(video_path, video_name), 'rb') as video_file:
+            video_data = video_file.read()
+        base64_str = base64.b64encode(video_data).decode('utf-8')
+        return base64_str
+    except Exception as e:
+        print(f"video load failed: {e}")
+        return None
+
+base64_video = encode_video_to_base64("/video_path", "video_name.mp4")
+
+payload_video_example = json.dumps({
+  "messages": [
+      {
+            "role": "system",
+            "content": [
+                {"type": "text", "text": "你是华为公司开发的多模态大模型，名字是openPangu-VL-7B。你能够处理文本和视觉模态的输入，并给出文本输出。"},     
+            ]
+      },
+      {
+            "role": "user",
+            "content": [
+                {"type": "video_url", "video_url": {"url": f"data:video/mp4;base64,{base64_video}"}},
+                {"type": "text", "text": "Please describe this video."},  
+            ]
+      }
+  ],    
+  "model": "pangu_vl",
+  "max_tokens": 500,
+  "temperature": 1.0,
+  "stream": False,
+})
+
+
+url = "http://127.0.0.1:8000/v1/chat/completions"
+headers = {
+  'Content-Type': 'application/json'
+}
+
+response_video_example = requests.request("POST", url, headers=headers, data=payload_video_example)
+print(f"the response of video example is {response_video_example.text}")
+
+```
+
+### 128k 视频长序列推理
+在/preprocessor_config.json中添加字段,输入视频会被抽取为768帧
+```
+"num_frames": 768,
+"sample_fps": -1.0
+```
+
+
+启动脚本(/inference/vllm_ascend/examples/start_serving_openpangu_vl_7b.sh)内设置参数：
+```
+export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
+MAX_MODEL_LEN=128000
+MAX_NUM_BATCHED_TOKENS=100000
+GPU_MEMORY_UTILIZATION=0.7
+
+--no-enable-chunked-prefill \
+--no-enable-prefix-caching \
+```
+
+
+### Int8推理
+
+#### ModelSlim量化
+
+openPangu-VL-7B模型支持使用开源量化框架ModelSlim,参考[[ModelSlim_openPangu-VL-7B]](https://gitcode.com/Ascend/msit/blob/msModelslim_Pangu_VL/msmodelslim/example/multimodal_vlm/openPangu-VL/ReadMe.md)，当前模型支持W8A8权重激活量化。
+
+##### openPangu-VL-7B W8A8 动态量化
+
+```bash
+export QUANT_PATH=your_quant_save_dir
+export MODEL_PATH=your_model_ckpt_dir
+export CALI_DATASET=your_cali_dataset_dir
+python quant_pangu_vl.py \
+--model_path $MODEL_PATH --calib_images $CALI_DATASET \
+--save_directory $QUANT_PATH --w_bit 8 --a_bit 8 --device_type npu \
+--trust_remote_code True --anti_method m2 --act_method 3 --is_dynamic True
+```
+
+相较于BF16模型，int8量化模型的config.json增加以下字段：
+```
+"quantize": "w8a8_dynamic",
+```
+
+ModelSlim量化脚本生成量化模型后会自动追加上述字段到config.json中。
+
+#### Int8推理
+
+相较于BF16模型推理，int8量化模型推理可使用同样的启动脚本，仅需：
+* 减少节点数、卡数；
+* 修改模型Checkpoint路径。

doc/vllm_ascend_for_openpangu_vl_7b_EN.md

@@ -0,0 +1,225 @@
+## Deployment Guide of openPangu-VL-7B Based on [vllm-ascend](https://github.com/vllm-project/vllm-ascend)
+
+### Deployment Environment Description
+
+Atlas 800T A2(64GB), openPangu-VL-7B can be deployed on 1、2、4 or 8 cards.
+
+### Docker Boot and Inference Code
+
+Use the vllm-ascend community image v0.9.1.
+
+Pull the image with the following command:
+
+```bash
+docker pull quay.io/ascend/vllm-ascend:v0.9.1
+```
+
+The following operations need to be performed on each node.
+Start the container.
+
+```bash
+# Update the vllm-ascend image
+export IMAGE=quay.io/ascend/vllm-ascend:v0.9.1  # Use correct image id
+export NAME=vllm-ascend  # Custom docker name
+
+# Run the container using the defined variables
+# Note if you are running bridge network with docker, Please expose available ports for multiple nodes communication in advance
+# To prevent device interference from other docker containers, add the argument "--privileged"
+docker run --rm \
+--name $NAME \
+--network host \
+--ipc=host \
+--device /dev/davinci0 \
+--device /dev/davinci1 \
+--device /dev/davinci2 \
+--device /dev/davinci3 \
+--device /dev/davinci4 \
+--device /dev/davinci5 \
+--device /dev/davinci6 \
+--device /dev/davinci7 \
+--device /dev/davinci_manager \
+--device /dev/devmm_svm \
+--device /dev/hisi_hdc \
+-v /usr/local/dcmi:/usr/local/dcmi \
+-v /usr/local/Ascend/driver/tools/hccn_tool:/usr/local/Ascend/driver/tools/hccn_tool \
+-v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
+-v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
+-v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
+-v /etc/ascend_install.info:/etc/ascend_install.info \
+-v /mnt/sfs_turbo/.cache:/root/.cache \
+-it $IMAGE bash
+```
+
+Ensure that the model checkpoint and the project code are accessible within the container. If not inside the container, enter the container as the root user:
+```bash
+docker exec -itu root $NAME /bin/bash
+```
+
+### PD Aggregation Inference
+
+Example startup script: `LOAD_CKPT_DIR=xxx bash examples/start_serving_openpangu_vl_7b.sh`。This example script requires 8 nodes(TENSOR_PARALLEL_SIZE_LOCAL=8）) to deploy the openPangu-VL-7B model. After starting the service, we can send requests to the first node (master node).
+
+### Send Testing Requests
+
+After the service is started, we can send testing requests. It is recommended to use the system prompt provided in the examples.
+
+example：image + text
+
+```python
+import json
+import base64
+import os
+import requests
+import json
+
+def encode_image_to_base64(img_path, img_name):
+    #load image to base64
+    try:
+        with open(os.path.join(img_path, img_name), 'rb') as img_file:
+            img_data = img_file.read()
+        base64_str = base64.b64encode(img_data).decode('utf-8')
+        return base64_str
+    except Exception as e:
+        print(f"image load failed: {e}")
+        return None
+
+base64_image = encode_image_to_base64("/image_path", "image_name.jpg")
+
+
+payload_image_example = json.dumps({
+  "messages": [
+      {
+            "role": "system",
+            "content": [
+                {"type": "text", "text": "You are a multimodal large model developed by Huawei, named openPangu-VL-7B. You can process both text and visual inputs and generate text outputs."},     
+            ]
+      },
+      {
+            "role": "user",
+            "content": [
+                {"type": "image_url", "image_url": {"url": f"data:image/jpg;base64,{base64_image}"}},
+                {"type": "text", "text": "Please describe this picture."}, 
+            ]
+      }
+  ],    
+  "model": "pangu_vl",
+  "max_tokens": 500,
+  "temperature": 1.0,
+  "stream": False,
+})
+
+
+url = "http://127.0.0.1:8000/v1/chat/completions"
+headers = {
+  'Content-Type': 'application/json'
+}
+
+response_image_example = requests.request("POST", url, headers=headers, data=payload_image_example)
+print(f"the response of image example is {response_image_example.text}")
+
+```
+
+
+example：video + text
+
+```python
+import json
+import base64
+import os
+import requests
+import json
+
+def encode_video_to_base64(video_path, video_name):
+    #load video to base64
+    try:
+        with open(os.path.join(video_path, video_name), 'rb') as video_file:
+            video_data = video_file.read()
+        base64_str = base64.b64encode(video_data).decode('utf-8')
+        return base64_str
+    except Exception as e:
+        print(f"video load failed: {e}")
+        return None
+
+base64_video = encode_video_to_base64("/video_path", "video_name.mp4")
+
+payload_video_example = json.dumps({
+  "messages": [
+      {
+            "role": "system",
+            "content": [
+                {"type": "text", "text": "You are a multimodal large model developed by Huawei, named openPangu-VL-7B. You can process both text and visual inputs and generate text outputs."},     
+            ]
+      },
+      {
+            "role": "user",
+            "content": [
+                {"type": "video_url", "video_url": {"url": f"data:video/mp4;base64,{base64_video}"}},
+                {"type": "text", "text": "Please describe this video."},  
+            ]
+      }
+  ],    
+  "model": "pangu_vl",
+  "max_tokens": 500,
+  "temperature": 1.0,
+  "stream": False,
+})
+
+
+url = "http://127.0.0.1:8000/v1/chat/completions"
+headers = {
+  'Content-Type': 'application/json'
+}
+
+response_video_example = requests.request("POST", url, headers=headers, data=payload_video_example)
+print(f"the response of video example is {response_video_example.text}")
+
+```
+
+### 128k Sequence Video Inference
+Add fields to /preprocessor_config.json,The input video will be sampled into 768 frames.
+```
+"num_frames": 768,
+"sample_fps": -1.0
+```
+
+In the startup script(/inference/vllm_ascend/examples/start_serving_openpangu_vl_7b.sh) setting the parameters as follows：
+```
+export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
+MAX_MODEL_LEN=128000
+MAX_NUM_BATCHED_TOKENS=100000
+GPU_MEMORY_UTILIZATION=0.7
+
+--no-enable-chunked-prefill \
+--no-enable-prefix-caching \
+```
+
+### Int8 Inference
+
+#### ModelSlim Quantization
+
+openPangu-VL-7B model supports quantization using the open source quantization framework,Please refer to [[ModelSlim_openPangu-VL-7B_README]](https://gitcode.com/Ascend/msit/blob/msModelslim_Pangu_VL/msmodelslim/example/multimodal_vlm/openPangu-VL/ReadMe.md)，The current model supports W8A8 quantization.
+
+##### openPangu-VL-7B W8A8 Dynamic quantization
+
+```bash
+export QUANT_PATH=your_quant_save_dir
+export MODEL_PATH=your_model_ckpt_dir
+export CALI_DATASET=your_cali_dataset_dir
+python quant_pangu_vl.py \
+--model_path $MODEL_PATH --calib_images $CALI_DATASET \
+--save_directory $QUANT_PATH --w_bit 8 --a_bit 8 --device_type npu \
+--trust_remote_code True --anti_method m2 --act_method 3 --is_dynamic True
+```
+
+Compared with the BF16 model, the following fields are added to the config.json file of the int8 quantization model:
+```
+"quantize": "w8a8_dynamic",
+```
+
+After the ModelSlim quantization script generates a quantization model, the preceding fields are automatically added to the config.json file.
+
+#### Int8 Inference
+
+Compared to BF16 model inference, the int8 quantized model inference uses the same startup script, requiring only:
+* Reducing the number of nodes and GPUs;
+* Modifying the model checkpoint path.

generation_config.json

@@ -0,0 +1,11 @@
+{
+  "bos_token_id": 1,
+  "pad_token_id": 2,
+  "do_sample": true,
+  "eos_token_id": [
+    45892
+  ],
+  "temperature": 0.000001,
+  "top_k": 1,
+  "transformers_version": "4.53.2"
+}

imageprocessor_openpangu_vl.py

@@ -0,0 +1,418 @@
+# coding=utf-8
+# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved.
+# Copyright 2025 The HuggingFace Inc. team.
+# Copyright 2025 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved.
+# Adapted from transformers/models/qwen2_vl/image_processing_qwen2_vl_fast.py
+
+# 
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
+# and OPT implementations in this library. It has been modified from its
+# original forms to accommodate minor architectural differences compared
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Optional, Union
+from types import SimpleNamespace
+from transformers.models.qwen2_vl.image_processing_qwen2_vl_fast import Qwen2VLImageProcessorFast
+from functools import partial, lru_cache
+from transformers.image_processing_utils import BatchFeature
+from transformers.image_utils import (
+    ChannelDimension,
+    SizeDict,
+    make_flat_list_of_images,
+    valid_images,
+    pil_torch_interpolation_mapping,
+)
+from torchvision.transforms.v2 import functional as F
+import torch
+from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize
+# from transformers.image_processing_utils_fast import (
+#     group_images_by_shape,
+#     reorder_images,
+# )
+
+
+def rescale(image, scale):
+    return image * scale
+
+
+def normalize(image, mean, std):
+    return F.normalize(image, mean, std)
+
+
+@lru_cache(maxsize=10)
+def _fuse_mean_std_and_rescale_factor(
+    do_normalize: Optional[bool] = None,
+    image_mean: Optional[Union[float, list[float]]] = None,
+    image_std: Optional[Union[float, list[float]]] = None,
+    do_rescale: Optional[bool] = None,
+    rescale_factor: Optional[float] = None,
+    device: Optional["torch.device"] = None,
+) -> tuple:
+    if do_rescale and do_normalize:
+        # Fused rescale and normalize
+        image_mean = torch.tensor(image_mean, device=device) * (1.0 / rescale_factor)
+        image_std = torch.tensor(image_std, device=device) * (1.0 / rescale_factor)
+        do_rescale = False
+    return image_mean, image_std, do_rescale
+
+
+def rescale_and_normalize(
+    images: "torch.Tensor",
+    do_rescale: bool,
+    rescale_factor: float,
+    do_normalize: bool,
+    image_mean: Union[float, list[float]],
+    image_std: Union[float, list[float]],
+) -> "torch.Tensor":
+    """
+    Rescale and normalize images.
+    """
+    image_mean, image_std, do_rescale = _fuse_mean_std_and_rescale_factor(
+        do_normalize=do_normalize,
+        image_mean=image_mean,
+        image_std=image_std,
+        do_rescale=do_rescale,
+        rescale_factor=rescale_factor,
+        device=images.device,
+    )
+    # if/elif as we use fused rescale and normalize if both are set to True
+    if do_normalize:
+        images = normalize(images.to(dtype=torch.float32), image_mean, image_std)
+    elif do_rescale:
+        images = rescale(images, rescale_factor)
+    images = images.to(OpenPanguVLImageProcessorFast.dtype)
+
+    return images
+
+# This part will be removed in the future.
+from collections import defaultdict
+def _group_images_by_shape(nested_images, is_nested: bool = False):
+    """Helper function to flatten a single level of nested image structures and group by shape."""
+    grouped_images = defaultdict(list)
+    grouped_images_index = {}
+    nested_images = [nested_images] if not is_nested else nested_images
+    for i, sublist in enumerate(nested_images):
+        for j, image in enumerate(sublist):
+            key = (i, j) if is_nested else j
+            shape = image.shape[1:]
+            grouped_images[shape].append(image)
+            grouped_images_index[key] = (shape, len(grouped_images[shape]) - 1)
+
+    return grouped_images, grouped_images_index
+
+
+def _reconstruct_nested_structure(indices, processed_images):
+    """Helper function to reconstruct a single level nested structure."""
+    # Find the maximum outer index
+    max_outer_idx = max(idx[0] for idx in indices.keys())
+
+    # Create the outer list
+    result = [None] * (max_outer_idx + 1)
+
+    # Group indices by outer index
+    nested_indices = defaultdict(list)
+    for i, j in indices.keys():
+        nested_indices[i].append(j)
+
+    for i in range(max_outer_idx + 1):
+        if i in nested_indices:
+            inner_max_idx = max(nested_indices[i])
+            inner_list = [None] * (inner_max_idx + 1)
+            for j in range(inner_max_idx + 1):
+                if (i, j) in indices:
+                    shape, idx = indices[(i, j)]
+                    inner_list[j] = processed_images[shape][idx]
+            result[i] = inner_list
+
+    return result
+
+
+def group_images_by_shape(
+    images: Union[list["torch.Tensor"], "torch.Tensor"],
+    disable_grouping: bool,
+    is_nested: bool = False,
+) -> tuple[
+    dict[tuple[int, int], list["torch.Tensor"]], dict[Union[int, tuple[int, int]], tuple[tuple[int, int], int]]
+]:
+    # If disable grouping is not explicitely provided, we favor disabling it if the images are on CPU, and enabling it otherwise.
+    if disable_grouping is None:
+        device = images[0][0].device if is_nested else images[0].device
+        disable_grouping = device == "cpu"
+
+    if disable_grouping:
+        if is_nested:
+            return {(i, j): images[i][j].unsqueeze(0) for i in range(len(images)) for j in range(len(images[i]))}, {
+                (i, j): ((i, j), 0) for i in range(len(images)) for j in range(len(images[i]))
+            }
+        else:
+            return {i: images[i].unsqueeze(0) for i in range(len(images))}, {i: (i, 0) for i in range(len(images))}
+
+    # Handle single level nested structure
+    grouped_images, grouped_images_index = _group_images_by_shape(images, is_nested)
+
+    # Stack images with the same shape
+    grouped_images = {shape: torch.stack(images_list, dim=0) for shape, images_list in grouped_images.items()}
+
+    return grouped_images, grouped_images_index
+
+
+def reorder_images(
+    processed_images: dict[tuple[int, int], "torch.Tensor"],
+    grouped_images_index: dict[Union[int, tuple[int, int]], tuple[tuple[int, int], int]],
+    is_nested: bool = False,
+) -> Union[list["torch.Tensor"], "torch.Tensor"]:
+    if not is_nested:
+        return [
+            processed_images[grouped_images_index[i][0]][grouped_images_index[i][1]]
+            for i in range(len(grouped_images_index))
+        ]
+
+    return _reconstruct_nested_structure(grouped_images_index, processed_images)
+
+
+class OpenPanguVLImageProcessorFast(Qwen2VLImageProcessorFast):
+    temporal_patch_size = 1
+    min_pxl = 28
+    min_edge = 56
+    dtype = torch.bfloat16
+
+    def _prepare_input_images(
+        self,
+        images,
+        do_convert_rgb,
+        input_data_format,
+        device,
+    ) -> list["torch.Tensor"]:
+        """
+        Prepare the input images for processing.
+        """
+        images = self._prepare_images_structure(images)
+        process_image_fn = partial(
+            self._process_image,
+            do_convert_rgb=do_convert_rgb,
+            input_data_format=input_data_format,
+            device=device,
+        )
+
+        processed_images = []
+        for image in images:
+            if image.size[0] <= OpenPanguVLImageProcessorFast.min_pxl or image.size[1] <= OpenPanguVLImageProcessorFast.min_pxl:
+                
+                if image.size[0] >= image.size[1]:
+                    aspect_ratio = OpenPanguVLImageProcessorFast.min_edge * 1.0 / image.size[1]
+                    new_image_height = OpenPanguVLImageProcessorFast.min_edge
+                    new_image_width = int(aspect_ratio * image.size[0])
+                else:
+                    aspect_ratio = OpenPanguVLImageProcessorFast.min_edge * 1.0 / image.size[0]
+                    new_image_height = int(aspect_ratio * image.size[1])
+                    new_image_width = OpenPanguVLImageProcessorFast.min_edge
+                image = image.resize((new_image_width, new_image_height))
+
+            processed_images.append(process_image_fn(image))
+        return processed_images
+
+    def preprocess(
+        self,
+        images = None,
+        videos = None,
+        do_resize = None,
+        size = None,
+        resample = None,
+        do_rescale = None,
+        rescale_factor = None,
+        do_normalize = None,
+        image_mean = None,
+        image_std = None,
+        min_pixels = None,
+        max_pixels = None,
+        patch_size = None,
+        temporal_patch_size = None,
+        merge_size = None,
+        do_convert_rgb = None,
+        return_tensors = None,
+        data_format = ChannelDimension.FIRST,
+        input_data_format = None,
+        device = None,
+        disable_grouping = False,
+        **kwargs,
+    ):
+        temporal_patch_size=OpenPanguVLImageProcessorFast.temporal_patch_size
+        params = self._resolve_preprocess_params(
+            do_resize=do_resize,
+            size=size,
+            min_pixels=min_pixels,
+            max_pixels=max_pixels,
+            resample=resample,
+            do_rescale=do_rescale,
+            rescale_factor=rescale_factor,
+            do_normalize=do_normalize,
+            image_mean=image_mean,
+            image_std=image_std,
+            patch_size=patch_size,
+            temporal_patch_size=temporal_patch_size,
+            merge_size=merge_size,
+            do_convert_rgb=do_convert_rgb,
+        )
+
+        data = self._process_images(
+            images,
+            params,
+            input_data_format,
+            device,
+            disable_grouping,
+            return_tensors
+        )
+
+        return data
+
+    def _resolve_preprocess_params(self, **kwargs):
+        params = SimpleNamespace()
+        for key, value in kwargs.items():
+            setattr(params, key, value if value is not None else getattr(self, key))
+        if params.size is None:
+            params.size = {"shortest_edge": params.min_pixels, "longest_edge": params.max_pixels}
+        params.size = SizeDict(**params.size)
+        params.image_mean = tuple(params.image_mean) if params.image_mean else None
+        params.image_std = tuple(params.image_std) if params.image_std else None
+        return params
+
+    def _process_images(self, images, params, input_data_format, device, disable_grouping, return_tensors):
+        images = make_flat_list_of_images(images)
+        if not valid_images(images):
+            raise ValueError("Invalid image type.")
+
+        images = self._prepare_input_images(
+            images=images,
+            do_convert_rgb=params.do_convert_rgb,
+            input_data_format=input_data_format,
+            device=device,
+        )
+
+        data = self._preprocess(
+                images=images,
+                do_resize=params.do_resize,
+                size=params.size,
+                interpolation=pil_torch_interpolation_mapping.get(params.resample, params.resample),
+                do_rescale=params.do_rescale,
+                rescale_factor=params.rescale_factor,
+                do_normalize=params.do_normalize,
+                image_mean=params.image_mean,
+                image_std=params.image_std,
+                patch_size=params.patch_size,
+                temporal_patch_size=params.temporal_patch_size,
+                merge_size=params.merge_size,
+                do_convert_rgb=params.do_convert_rgb,
+                input_data_format=input_data_format,
+                device=device,
+                disable_grouping=disable_grouping,
+                return_tensors=return_tensors,
+            )
+        
+        return data
+
+    def _preprocess(
+        self,
+        images: list["torch.Tensor"],
+        do_resize: bool,
+        size: SizeDict,
+        interpolation: Optional["F.InterpolationMode"],
+        do_rescale: bool,
+        rescale_factor: float,
+        do_normalize: bool,
+        image_mean: Optional[Union[float, list[float]]],
+        image_std: Optional[Union[float, list[float]]],
+        patch_size: int,
+        temporal_patch_size: int,
+        merge_size: int,
+        disable_grouping: Optional[bool],
+        return_tensors,
+        **kwargs,
+    ):
+        # Group images by size for batched resizing
+        grouped_images, grouped_images_index = group_images_by_shape(images, disable_grouping=disable_grouping)
+        resized_images_grouped = {}
+        for shape, stacked_images in grouped_images.items():
+            height, width = stacked_images.shape[-2:]
+            if do_resize:
+                resized_height, resized_width = smart_resize(
+                    height,
+                    width,
+                    factor=patch_size * merge_size,
+                    min_pixels=size["shortest_edge"],
+                    max_pixels=size["longest_edge"],
+                )
+                stacked_images = self.resize(
+                    image=stacked_images,
+                    size=SizeDict(height=resized_height, width=resized_width),
+                    interpolation=interpolation,
+                )
+            resized_images_grouped[shape] = stacked_images
+        resized_images = reorder_images(resized_images_grouped, grouped_images_index)
+
+        # Group images by size for further processing
+        # Needed in case do_resize is False, or resize returns images with different sizes
+        grouped_images, grouped_images_index = group_images_by_shape(resized_images, disable_grouping=disable_grouping)
+        processed_images_grouped = {}
+        processed_grids = {}
+        for shape, stacked_images in grouped_images.items():
+            resized_height, resized_width = stacked_images.shape[-2:]
+            # Fused rescale and normalize
+            # patches = rescale_and_normalize(
+            #     stacked_images, do_rescale, rescale_factor, do_normalize, image_mean, image_std
+            # )
+            patches = stacked_images
+            if patches.ndim == 4:
+                # add a temporal dimension if we have images
+                patches = patches.unsqueeze(1)
+            if patches.shape[1] % temporal_patch_size != 0:
+                repeats = patches[:, -1:].repeat(1, temporal_patch_size - 1, 1, 1, 1)
+                patches = torch.cat([patches, repeats], dim=1)
+            batch_size, grid_t, channel = patches.shape[:3]
+            grid_t = grid_t // temporal_patch_size
+            grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
+
+            patches = patches.view(
+                batch_size,
+                grid_t,
+                temporal_patch_size,
+                channel,
+                grid_h // merge_size,
+                merge_size,
+                patch_size,
+                grid_w // merge_size,
+                merge_size,
+                patch_size,
+            )
+            # Reorder dimensions to group grid and patch information for subsequent flattening.
+            # (batch, grid_t, grid_h, grid_w, merge_h, merge_w, channel, temp_patch_size, patch_h, patch_w)
+            patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9)
+            flatten_patches = patches.reshape(
+                batch_size,
+                grid_t * grid_h * grid_w,
+                channel * temporal_patch_size * patch_size * patch_size,
+            )
+
+            processed_images_grouped[shape] = flatten_patches
+            processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size
+
+        processed_images = reorder_images(processed_images_grouped, grouped_images_index)
+        processed_grids = reorder_images(processed_grids, grouped_images_index)
+        pixel_values = torch.cat(processed_images, dim=0)
+        image_grid_thw = torch.tensor(processed_grids)
+
+        return BatchFeature(
+            data={"pixel_values": pixel_values, 
+                  "image_grid_thw": image_grid_thw}, tensor_type=return_tensors
+        )

inference/generate.py

@@ -0,0 +1,58 @@
+from transformers import AutoProcessor
+from transformers import AutoModelForCausalLM
+from qwen_vl_utils import process_vision_info
+
+model_path="../"
+
+print(f"LOAD MODEL FROM: {model_path}")
+
+
+
+key_mapping = {
+        "^visual": "model.visual",
+        r"^model(?!\.(language_model|visual))": "model.language_model",
+    }
+
+
+model =  AutoModelForCausalLM.from_pretrained(
+        model_path,
+        trust_remote_code=True,
+        torch_dtype='auto',
+        key_mapping=key_mapping).eval().cuda()
+
+conversation = [
+    {
+        "role": "system",
+        "content": [
+            {"type": "text", "text": "你是华为公司开发的多模态大模型，名字是openPangu-VL-7B。你能够处理文本和视觉模态的输入，并给出文本输出。"},     
+        ]
+    },
+    {
+        "role": "user",
+        "content": [
+            {"type": "text", "text": "你好，你是谁？"},     
+        ]
+    }
+] 
+
+
+processor = AutoProcessor.from_pretrained(model_path, trust_remote_code=True)
+text = processor.apply_chat_template(conversation, tokenize=False, add_generation_prompt=True)
+
+image_inputs, video_inputs = process_vision_info(conversation)
+
+inputs = processor(
+    text=[text],
+    images=image_inputs,
+    videos=video_inputs,
+    padding=False,
+    return_tensors="pt",
+)
+inputs = inputs.to(model.device)
+
+generated_ids = model.generate(**inputs, max_new_tokens=128)
+generated_ids_trimmed = [out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)]
+res = processor.batch_decode(
+    generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
+)
+print(f"OUTPUT: {res}")

inference/requirements.txt

@@ -0,0 +1,4 @@
+torch==2.5.1
+torch_npu==2.5.1
+transformers==4.53.2
+qwen_vl_utils==0.0.14

inference/vllm_ascend/examples/quick_start.py

@@ -0,0 +1,89 @@
+import json
+import base64
+import os
+import requests
+import json
+from PIL import Image, ImageDraw, ImageFont
+import ast
+import re
+def encode_image_to_base64(img_path):
+    """
+    :param img_path: image path
+    :return: Base64 encoding
+    """
+    try:
+        with open(img_path, 'rb') as img_file:
+            img_data = img_file.read()
+        base64_str = base64.b64encode(img_data).decode('utf-8')
+        return base64_str
+    except Exception as e:
+        print(f"Encoding Error: {e}")
+        return None
+
+def infer_image_with_api(image_path, prompt):
+    url = "http://127.0.0.1:8000/v1/chat/completions"
+    base64_image = encode_image_to_base64(image_path)
+    payload = json.dumps({
+      "messages": [
+          {
+                "role": "system",
+                "content": [
+                    {"type": "text", "text": "你是华为公司开发的多模态大模型，名字是openPangu-VL-7B。你能够处理文本和视觉模态的输入，并给出文本输出。"},     
+                ]
+          },
+          {
+                "role": "user",
+                "content": [
+                    {"type": "image_url", "image_url": {"url": f"data:image/jpg;base64,{base64_image }"}},
+                    {"type": "text", "text": prompt},     
+                ]
+          }
+      ],    
+      "model": "pangu_vl",
+      "max_tokens": 2048,
+      "temperature": 0,
+      "stream": False,
+
+    })
+
+    headers = {
+      'Content-Type': 'application/json'
+    }
+
+    response = requests.request("POST", url, headers=headers, data=payload)
+    return json.loads(response.text)["choices"][0]["message"]["content"]
+
+
+def infer_message_with_api(prompt):
+    url = "http://127.0.0.1:8000/v1/chat/completions"
+    payload = json.dumps({
+      "messages": [
+          {
+                "role": "system",
+                "content": [
+                    {"type": "text", "text": "你是华为公司开发的多模态大模型，名字是openPangu-VL-7B。你能够处理文本和视觉模态的输入，并给出文本输出。"},     
+                ]
+          },
+          {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": prompt},     
+                ]
+          }
+      ],    
+      "model": "pangu_vl",
+      "max_tokens": 2048,
+      "temperature": 0,
+      "stream": False,
+
+    })
+
+    headers = {
+      'Content-Type': 'application/json'
+    }
+
+    response = requests.request("POST", url, headers=headers, data=payload)
+    return json.loads(response.text)["choices"][0]["message"]["content"]
+
+res = infer_message_with_api("你好，你是谁？")
+print(res)