Duplicate from NexaAI/OmniNeural-4B

Co-authored-by: Alex Chen <alexchen4ai@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,46 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+files-1-1.nexa filter=lfs diff=lfs merge=lfs -text
+weights-1-8.nexa filter=lfs diff=lfs merge=lfs -text
+weights-2-8.nexa filter=lfs diff=lfs merge=lfs -text
+weights-3-8.nexa filter=lfs diff=lfs merge=lfs -text
+weights-4-8.nexa filter=lfs diff=lfs merge=lfs -text
+weights-5-8.nexa filter=lfs diff=lfs merge=lfs -text
+weights-6-8.nexa filter=lfs diff=lfs merge=lfs -text
+weights-7-8.nexa filter=lfs diff=lfs merge=lfs -text
+weights-8-8.nexa filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
+files-1-1.json filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,85 @@
+NEXA AI LICENSE AGREEMENT
+Release Date: August 17, 2025
+
+Copyright © 2025 Nexa AI, Inc., a Delaware corporation with its principal place of business in Cupertino, California, USA.
+
+Friendly Summary (not a substitute for the terms below)
+- You can use, modify, and share our models (and code) for research, evaluation, learning, and personal projects.
+- Please credit us when your released model is trained or improved using our Outputs (“Built with Nexa” or “Improved using Nexa”).
+- Business/paid use isn’t covered — reach out for a commercial license.
+- No warranties; use at your own risk.
+
+0. Purpose
+This License is designed to be developer-friendly. It enables non-commercial research, experimentation, benchmarking, education, and **personal** use of the Nexa Materials. Commercial use requires a separate written agreement with Nexa.
+
+1. Definitions
+1.1 “Nexa Materials” means any models, model weights, parameters (including optimizer states), software, algorithms, documentation, and related artifacts that Nexa makes available under this License.
+1.2 “Output” means any result produced by the Nexa Materials (e.g., text, code, images, embeddings, logits, metrics).
+1.3 “Source Form” means the preferred form for making modifications (e.g., source code, documentation source, configuration files).
+1.4 “Object Form” means any form resulting from mechanical transformation, compilation, or conversion of a Source Form (e.g., binaries, converted weights).
+1.5 “Non-Commercial Use” means use that is not intended for direct or indirect commercial advantage or monetary compensation and is not integrated into any paid, revenue-generating, or production service or product.
+1.6 “Personal Use” means Non-Commercial Use by a natural person, not acting on behalf of a company or organization. Examples include classwork, portfolio projects, hackathon prototypes, research notebooks, open-source demos, and personal apps that are distributed for free and without monetization.
+
+2. License Grant (Non-Commercial + Personal)
+Subject to this License, Nexa grants you a non-exclusive, worldwide, non-transferable, royalty-free license to:
+(a) use, reproduce, and modify the Nexa Materials; and
+(b) distribute the Nexa Materials and your modifications and derivative works,
+in each case solely for Non-Commercial Use (including Personal Use).
+
+Commercial Use is not permitted under this License. To engage in any Commercial Use, please contact Nexa for a separate license.
+
+3. Redistribution Conditions
+You may distribute the Nexa Materials or your derivative works for Non-Commercial Use in Source Form or Object Form if you:
+3.1 Include a copy of this License with each distribution;
+3.2 Clearly mark files you modified, including a brief description and date of change; and
+3.3 Retain the following attribution in a NOTICE or equivalent file:
+    “Includes or is derived from Nexa Materials licensed under the Nexa AI Research. © Nexa AI, Inc. All rights reserved.”
+You may apply additional or different license terms to your own modifications or derivative works as a whole, provided your terms do not grant rights that exceed or conflict with this License with respect to the Nexa Materials.
+
+4. Attribution for Models Built Using Outputs
+If you use Outputs to train, fine-tune, distill, or otherwise improve a model that you distribute or make available (free or paid), include a clear credit in documentation (e.g., README, model card, product docs) as:
+    “Built with Nexa” or “Improved using Nexa.”
+For interactive products, a reasonable placement is an About/Credits page or documentation site.
+
+5. Intellectual Property; No Patent License
+5.1 Nexa retains all intellectual property rights in the Nexa Materials and in derivatives created by or for Nexa.
+5.2 You retain rights in your original modifications and derivative works you create, subject to this License.
+5.3 No patent rights are granted by this License, whether by implication, estoppel, or otherwise.
+
+6. Trademarks
+This License does not grant permission to use Nexa’s names, logos, or trademarks, except for the limited factual attributions required by Section 4 or customary references to the origin of the Nexa Materials.
+
+7. Legal Compliance and Export Controls
+You must comply with all applicable laws and regulations, including U.S. export control and sanctions laws and any local equivalents, in connection with your access to and use of the Nexa Materials and Outputs.
+
+8. Acceptable Use and Responsibility
+You are solely responsible for your use of the Nexa Materials and Outputs, including datasets and prompts you supply, and for ensuring compliance with privacy, data protection, security, and other applicable laws. Do not use the Nexa Materials or Outputs to violate rights of others or applicable law.
+
+9. Indemnification
+To the maximum extent permitted by law, you agree to defend, indemnify, and hold harmless Nexa and its affiliates from and against any claims, liabilities, damages, losses, and expenses (including reasonable attorneys’ fees) arising out of or related to your use, modification, or distribution of the Nexa Materials or Outputs.
+
+10. Warranty Disclaimer
+THE NEXA MATERIALS AND OUTPUTS ARE PROVIDED “AS IS” AND “AS AVAILABLE.” NEXA DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, ACCURACY, OR PERFORMANCE. YOU BEAR ALL RISK AS TO THE RESULTS AND PERFORMANCE OF THE NEXA MATERIALS AND OUTPUTS.
+
+11. Limitation of Liability
+TO THE MAXIMUM EXTENT PERMITTED BY LAW, NEXA SHALL NOT BE LIABLE FOR ANY INDIRECT, INCIDENTAL, SPECIAL, CONSEQUENTIAL, EXEMPLARY, OR PUNITIVE DAMAGES, OR FOR ANY LOSS OF PROFITS, REVENUE, DATA, OR GOODWILL, ARISING OUT OF OR RELATING TO THIS LICENSE OR THE NEXA MATERIALS OR OUTPUTS, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. NEXA’S TOTAL AGGREGATE LIABILITY FOR DIRECT DAMAGES SHALL NOT EXCEED ONE HUNDRED U.S. DOLLARS (US$100).
+
+12. Defensive Termination
+If you or any entity you control initiates any legal claim alleging that the Nexa Materials or Outputs infringe your intellectual property rights, this License automatically terminates as of the filing date of such claim.
+
+13. Term and Termination
+13.1 This License takes effect when you access the Nexa Materials and continues unless terminated as provided herein.
+13.2 Nexa may terminate this License if you breach any of its terms. Upon termination, you must cease all use and distribution of the Nexa Materials and delete all copies under your possession or control.
+13.3 Sections 5–16 survive termination.
+
+14. Updates
+Nexa may provide updates to the Nexa Materials or publish new versions of this License. Updated Nexa Materials will be governed by the license accompanying them. Continued use of updated Nexa Materials constitutes acceptance of the then-current license.
+
+15. Governing Law and Venue
+This License is governed by the laws of the State of Delaware, excluding its conflicts-of-law rules.
+
+16. Contact
+Questions about this License or commercial use? Email dev@nexa.ai (or your usual Nexa contact).
+
+Attribution Snippet (for NOTICE)
+“Nexa Materials are licensed under the Nexa AI Research © Nexa AI, Inc., Cupertino, CA, USA. All rights reserved.”

README.md

@@ -0,0 +1,188 @@
+---
+tags:
+- multimodal
+- NPU
+- On-device
+- Snapdragon PC
+- Android
+license: cc-by-4.0
+license_name: nexa-research
+license_link: LICENSE
+pipeline_tag: any-to-any
+---
+<p align="center">
+  <img alt="omnineural" src="https://cdn-uploads.huggingface.co/production/uploads/6618e0424dbef6bd3c72f89a/zRUnoWmw43fl9hrXHg0pE.png">
+</p>
+
+# **OmniNeural** — World’s First NPU-aware Multimodal Model
+
+
+## **Overview**  
+**OmniNeural** is the first fully multimodal model designed specifically for Neural Processing Units (NPUs). It natively understands **text, images, and audio**, and runs across PCs, mobile devices, automobile, IoT, and robotics.  
+
+## Demos
+
+### 📱 Mobile Phone NPU - Demo on Samsung S25 Ultra
+The first-ever fully local, multimodal, and conversational AI assistant that hears you and sees what you see, running **natively on Snapdragon NPU** for long battery life and low latency.
+
+<video controls width="720" preload="metadata"
+  src="https://huggingface.co/NexaAI/OmniNeural-4B/resolve/main/assets/MOBILE_50MB.mp4"
+  type="video/mp4"></video>
+
+---
+
+## ✨ PC NPU - Capabilities Highlights
+
+<table>
+<tr>
+<td width="33%">
+<video controls width="100%" preload="metadata"
+  src="https://huggingface.co/NexaAI/OmniNeural-4B/resolve/main/assets/PC_demo_2_image.mov"></video>
+<p align="center"><b>🖼️ Multi-Image Reasoning</b><br>Spot the difference across two images in multi-round dialogue.</p>
+</td>
+
+<td width="33%">
+<video controls width="100%" preload="metadata"
+  src="https://huggingface.co/NexaAI/OmniNeural-4B/resolve/main/assets/PC_Demo_Agent.mov"></video>
+<p align="center"><b>🤖 Image + Text → Function Call</b><br>Snap a poster, add a text instruction, and AI agent creates a calendar event.</p>
+</td>
+
+<td width="33%">
+<video controls width="100%" preload="metadata"
+  src="https://huggingface.co/NexaAI/OmniNeural-4B/resolve/main/assets/PC_Demo_Audio.mov"></video>
+<p align="center"><b>🎶 Multi-Audio Comparison</b><br>Tell the difference between two music clips locally.</p>
+</td>
+</tr>
+</table>
+
+
+
+---
+
+## **Key Features**  
+- **Multimodal Intelligence** – Processes **text, image, and audio** in a unified model for richer reasoning and perception.  
+- **NPU-Optimized Architecture** – Uses ReLU ops, sparse tensors, convolutional layers, and static graph execution for maximum throughput — **20% faster than non-NPU-aware models** .  
+- **Hardware-Aware Attention** – Attention patterns tuned for NPU, lowering compute and memory demand .  
+- **Native Static Graph** – Supports variable-length multimodal inputs with stable, predictable latency .  
+- **Performance Gains** – **9× faster audio processing** and **3.5× faster image processing** on NPUs compared to baseline encoders .  
+- **Privacy-First Inference** – All computation stays local: private, offline-capable, and cost-efficient.
+
+---
+
+## **Performance / Benchmarks**  
+### Human Evaluation (vs baselines)   
+- **Vision**: Wins/ties in ~75% of prompts against Apple Foundation, Gemma-3n-E4B, Qwen2.5-Omni-3B.  
+- **Audio**: Clear lead over baselines, much better than Gemma3n and Apple foundation model.  
+- **Text**: Matches or outperforms leading multimodal baselines.  
+
+
+<p align="center">
+  <img src="https://cdn-uploads.huggingface.co/production/uploads/6618e0424dbef6bd3c72f89a/vsrg43GxTOSAj7q_SI60o.png" width="1560" alt="Human eval chart" />
+</p>
+
+### Nexa Attention Speedups   
+- **9× faster** audio encoding (vs Whisper encoder).  
+- **3.5× faster** image encoding (vs SigLIP encoder).  
+
+
+<p align="center">
+  <img src="https://cdn-uploads.huggingface.co/production/uploads/6618e0424dbef6bd3c72f89a/1039SN5JBQkS04z4YnoIi.png" width="400" alt="Human eval chart" />
+</p>
+
+---
+
+## **Architecture Overview**  
+OmniNeural’s design is tightly coupled with NPU hardware:  
+- **NPU-friendly ops** (ReLU > GELU/SILU).  
+- **Sparse + small tensor multiplications** for efficiency.  
+- **Convolutional layers** favored over linear for better NPU parallelization.  
+- **Hardware-aware attention** patterns to cut compute cost.  
+- **Static graph execution** for predictable latency.
+
+
+![image/png](https://cdn-uploads.huggingface.co/production/uploads/6618e0424dbef6bd3c72f89a/oINYbgXILJgTuKxKc1aO_.png)
+
+---
+
+## **Production Use Cases**  
+
+- **PC & Mobile** – On-device AI agents combine **voice, vision, and text** for natural, accurate responses.  
+   - Examples: Summarize slides into an email (PC)*, *extract action items from chat (mobile).  
+   - Benefits: Private, offline, battery-efficient.  
+
+- **Automotive** – In-car assistants handle **voice control, cabin safety, and environment awareness**.  
+   - Examples: Detects risks (child unbuckled, pet left, loose objects) and road conditions (fog, construction).  
+   - Benefits: Decisions run locally in milliseconds.  
+
+- **IoT & Robotics** – Multimodal sensing for **factories, AR/VR, drones, and robots**.  
+   - Examples: Defect detection, technician overlays, hazard spotting mid-flight, natural robot interaction.  
+   - Benefits: Works without network connectivity.  
+
+---
+
+## How to use
+
+> ⚠️ **Hardware requirement:** OmniNeural-4B currently runs **only on Qualcomm NPUs** (e.g., Snapdragon-powered AIPC).  
+> Apple NPU support is planned next.
+
+### 1) Install Nexa-SDK
+
+- Download and follow the steps under "Deploy Section" Nexa's model page:  [Download Windows arm64 SDK](https://sdk.nexa.ai/model/OmniNeural-4B)
+- (Other platforms coming soon)
+
+### 2) Get an access token
+Create a token in the Model Hub, then log in:
+
+```bash
+nexa config set license '<access_token>'
+```
+
+### 3) Run the model
+Running:
+
+```bash
+nexa infer NexaAI/OmniNeural-4B
+```
+
+/mic mode. Once the model is running, you can type below to record your voice directly in terminal
+```bash
+> /mic
+```
+
+For images and audio, simply drag your files into the command line. Remember to leave space between file paths.
+
+---
+
+## Links & Community
+
+[![Discord](https://img.shields.io/badge/Discord-Join-5865F2?logo=discord&logoColor=white)](https://discord.com/invite/nexa-ai)
+
+[![X (Twitter) Follow](https://img.shields.io/badge/Follow-@nexa_ai-111?logo=x&logoColor=white)](https://x.com/nexa_ai)
+
+[![Website](https://img.shields.io/badge/Website-nexa.ai-0A84FF)](https://nexa.ai)
+
+- **Issues / Feedback:** Use the **HF Discussions** tab or submit an issue in our discord or nexa-sdk github. 
+- **Roadmap & updates:** Follow us on X and Discord.
+
+> If you want to see more **NPU-first, multimodal** releases on HF, please give our model a like ❤️.
+
+## Limitation
+The current model is mainly optimized for English. We will optimize other language as the next step. 
+
+---
+
+## **Citation**  
+
+```bibtex
+@misc{
+      title={OmniNeural: World’s First NPU-aware Multimodal Model}, 
+      author={Nexa AI},
+      year={2025},
+      url={https://huggingface.co/NexaAI/OmniNeural-4B}, 
+}
+```
+
+## License  
+This model is released under the **Creative Commons Attribution–NonCommercial 4.0 (CC BY-NC 4.0)** license.  
+Non-commercial use, modification, and redistribution are permitted with attribution.  
+For commercial licensing, please contact **dev@nexa.ai**.

assets/MOBILE_50MB.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:856a7eab6ec4b12236b44e4264637774081242423b5575cb84121f38e82c179e
+size 40200022

assets/PC_Demo_Agent.mov

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5054bb51bf3701f004f5df710231fc86c32cf97be5590ee6198218ba30d5f3aa
+size 22748433

assets/PC_Demo_Audio.mov

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84c353c9361c8225c4518bbb0e0ffff4921ff86b3b7d4bdf75d8e47936564310
+size 29321731

assets/PC_demo_2_image.mov

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5a1aafb5a39e291887a6eb4671c22586b327497b485c7e2b37ffa1b6158a91e
+size 23245220

audio/htp_backend_ext_config.json

@@ -0,0 +1,31 @@
+{
+    "graphs": [
+        {
+            "O": 3.0,
+            "vtcm_mb": 8,
+            "graph_names": [
+                "vit"
+            ],
+            "fp16_relaxed_precision": 0
+        }
+    ],
+    "devices": [
+        {
+            "soc_id": 60,
+            "dsp_arch": "v73",
+            "cores": [
+                {
+                    "perf_profile": "burst",
+                    "rpc_control_latency": 100
+                }
+            ],
+            "pd_session": "unsigned"
+        }
+    ],
+    "context": {
+        "weight_sharing_enabled": false
+    },
+    "memory": {
+        "mem_type": "shared_buffer"
+    }
+}

files-1-1.nexa

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c14310cd665fa61505bf6859233afd6b72dd836b28b81e19ca7b751a76e29e39
+size 12179329

llm/htp_backend_ext_config.json

@@ -0,0 +1,21 @@
+{
+    "devices": [
+        {
+            "soc_model": 60,
+            "dsp_arch": "v73",
+            "cores": [
+                {
+                    "core_id": 0,
+                    "perf_profile": "burst",
+                    "rpc_control_latency": 100
+                }
+            ]
+        }
+    ],
+    "memory": {
+        "mem_type": "shared_buffer"
+    },
+    "context": {
+        "weight_sharing_enabled": true
+    }
+}

nexa.manifest

@@ -0,0 +1,5 @@
+{
+    "ModelName": "omni-neural",
+    "ModelType": "vlm",
+    "PluginId": "npu"
+}

vit/htp_backend_ext_config.json

@@ -0,0 +1,31 @@
+{
+    "graphs": [
+        {
+            "O": 3.0,
+            "vtcm_mb": 8,
+            "graph_names": [
+                "vit"
+            ],
+            "fp16_relaxed_precision": 0
+        }
+    ],
+    "devices": [
+        {
+            "soc_id": 60,
+            "dsp_arch": "v73",
+            "cores": [
+                {
+                    "perf_profile": "burst",
+                    "rpc_control_latency": 100
+                }
+            ],
+            "pd_session": "unsigned"
+        }
+    ],
+    "context": {
+        "weight_sharing_enabled": false
+    },
+    "memory": {
+        "mem_type": "shared_buffer"
+    }
+}

vlm/htp_backend_ext_config.json

@@ -0,0 +1,31 @@
+{
+    "graphs": [
+        {
+            "O": 3.0,
+            "vtcm_mb": 8,
+            "graph_names": [
+                "vit"
+            ],
+            "fp16_relaxed_precision": 0
+        }
+    ],
+    "devices": [
+        {
+            "soc_id": 60,
+            "dsp_arch": "v73",
+            "cores": [
+                {
+                    "perf_profile": "burst",
+                    "rpc_control_latency": 100
+                }
+            ],
+            "pd_session": "unsigned"
+        }
+    ],
+    "context": {
+        "weight_sharing_enabled": false
+    },
+    "memory": {
+        "mem_type": "shared_buffer"
+    }
+}

weights-1-8.nexa

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08cdf336224b5798ee876a47d760a3ae1e1569a11d931e7117822b64ac68ed37
+size 770293308

weights-2-8.nexa

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4611b0bdfaf1d15a680f59c38c1e286adf0e5fa3ee653a599a7fd64e7461347
+size 1165936148

weights-3-8.nexa

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d0ded0791faad29bafc06f18d5f352a4e297f14f3e5b4f771ffb50912343c1e
+size 4572692

weights-4-8.nexa

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b111649ac3fc3dc7cc86099029ccc821f68d674a36beb70834f02301846ebac5
+size 908198932

weights-5-8.nexa

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddf26428c50efbbb048d0492a3b57ea7b93406556e8998757e5b0d46ca96db05
+size 14281108

weights-6-8.nexa

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:743fdd8011df0e2b5eda56fdca8d8290b6af754209b7e8599f2011384d7a0683
+size 5588636

weights-7-8.nexa

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:188fb10f97a41ef6e83d2025d96143080e5c3bc41aff7f3e50d43a4c86c24534
+size 649296044

weights-8-8.nexa

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4910ef47adb345d915ab6750f42bdcfd1b13294f01c8b7104eefdd72e6d9eb1b
+size 1244659876