Duplicate from PeiqingYang/MatAnyone2

Co-authored-by: pq-yang <PeiqingYang@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,74 @@
+---
+library_name: matanyone2
+tags:
+- model_hub_mixin
+- pytorch_model_hub_mixin
+---
+
+<div align="center">
+<img src="https://github.com/pq-yang/MatAnyone2/blob/main/assets/matanyone2_logo.png?raw=true" alt="MatAnyone Logo" style="height: 52px;">
+<div style="text-align: center">
+    <h2>Scaling Video Matting via a Learned Quality Evaluator</h2>
+</div>
+
+<div>
+    <a href='https://pq-yang.github.io/' target='_blank'>Peiqing Yang</a><sup>1</sup>&emsp;
+    <a href='https://shangchenzhou.com/' target='_blank'>Shangchen Zhou</a><sup>1†</sup>&emsp;
+    <a href="https://www.linkedin.com/in/kai-hao-794321382/" target='_blank'>Kai Hao</a><sup>1</sup>&emsp;
+    <a href="https://scholar.google.com.sg/citations?user=fMXnSGMAAAAJ&hl=en/" target='_blank'>Qingyi Tao</a><sup>2</sup>&emsp;
+</div>
+<div>
+    <sup>1</sup>S-Lab, Nanyang Technological University&emsp; 
+    <sup>2</sup>SenseTime Research, Singapore&emsp; 
+    <br>
+    <sup>†</sup>Project lead
+</div>
+
+
+<div>
+    <h4 align="center">
+        <a href="https://pq-yang.github.io/projects/MatAnyone2/" target='_blank'>
+        <img src="https://img.shields.io/badge/😈-Project%20Page-blue">
+        </a>
+        <a href="https://arxiv.org/abs/2512.11782" target='_blank'>
+        <img src="https://img.shields.io/badge/arXiv-2501.14677-b31b1b.svg">
+        </a>
+        <a href="https://www.youtube.com/watch?v=tyi8CNyjOhc&lc=Ugw1OS7z5QbW29RZCFZ4AaABAg" target='_blank'>
+        <img src="https://img.shields.io/badge/Demo%20Video-%23FF0000.svg?logo=YouTube&logoColor=white">
+        </a>
+        <a href="https://huggingface.co/spaces/PeiqingYang/MatAnyone" target='_blank'>
+        <img src="https://img.shields.io/badge/Demo-%F0%9F%A4%97%20Hugging%20Face-blue">
+        </a>
+        <a href="https://colab.research.google.com/drive/1NYW_CUDf7jnzxir7tOOlY7wRRalVOifD?usp=sharing" target='_blank'>
+        <img src="https://colab.research.google.com/assets/colab-badge.svg">
+        </a>
+    </h4>
+</div>
+
+<strong>MatAnyone 2 is a practical human video matting framework that preserves fine details by avoiding segmentation-like boundaries, while also shows enhanced robustness under challenging real-world conditions.</strong>
+
+<div style="width: 100%; text-align: center; margin:auto;">
+    <img style="width:100%" src="https://github.com/pq-yang/MatAnyone2/blob/main/assets/teaser.jpg?raw=true">
+</div>
+
+🎥 For more visual results, go checkout our <a href="https://pq-yang.github.io/projects/MatAnyone2/" target="_blank">project page</a>
+
+</div>
+
+---
+
+## How to use
+you can run the following commands to get started and start working with the model
+```
+pip install -qqU git+https://github.com/pq-yang/MatAnyone2.git
+```
+
+```python
+from matanyone2 import MatAnyone2, InferenceCore
+model = MatAnyone2.from_pretrained("PeiqingYang/MatAnyone2")
+processor = InferenceCore(model,device="cuda:0")
+# inference
+processor.process_video(input_path="inputs/video/test-sample2.mp4",
+                        mask_path="inputs/mask/test-sample2.png",
+                        output_path="results")
+```

config.json

@@ -0,0 +1,137 @@
+{
+  "cfg": {
+    "amp": false,
+    "chunk_size": -1,
+    "flip_aug": false,
+    "long_term": {
+      "buffer_tokens": 2000,
+      "count_usage": true,
+      "max_mem_frames": 10,
+      "max_num_tokens": 10000,
+      "min_mem_frames": 5,
+      "num_prototypes": 128
+    },
+    "max_internal_size": -1,
+    "max_mem_frames": 5,
+    "mem_every": 5,
+    "model": {
+      "aux_loss": {
+        "query": {
+          "enabled": true,
+          "weight": 0.01
+        },
+        "sensory": {
+          "enabled": true,
+          "weight": 0.01
+        }
+      },
+      "embed_dim": 256,
+      "key_dim": 64,
+      "mask_decoder": {
+        "up_dims": [
+          256,
+          128,
+          128,
+          64,
+          16
+        ]
+      },
+      "mask_encoder": {
+        "final_dim": 256,
+        "type": "resnet18"
+      },
+      "object_summarizer": {
+        "add_pe": true,
+        "embed_dim": "${model.object_transformer.embed_dim}",
+        "num_summaries": "${model.object_transformer.num_queries}"
+      },
+      "object_transformer": {
+        "embed_dim": "${model.embed_dim}",
+        "ff_dim": 2048,
+        "num_blocks": 3,
+        "num_heads": 8,
+        "num_queries": 16,
+        "pixel_self_attention": {
+          "add_pe_to_qkv": [
+            true,
+            true,
+            false
+          ]
+        },
+        "query_self_attention": {
+          "add_pe_to_qkv": [
+            true,
+            true,
+            false
+          ]
+        },
+        "read_from_memory": {
+          "add_pe_to_qkv": [
+            true,
+            true,
+            false
+          ]
+        },
+        "read_from_past": {
+          "add_pe_to_qkv": [
+            true,
+            true,
+            false
+          ]
+        },
+        "read_from_pixel": {
+          "add_pe_to_qkv": [
+            true,
+            true,
+            false
+          ],
+          "input_add_pe": false,
+          "input_norm": false
+        },
+        "read_from_query": {
+          "add_pe_to_qkv": [
+            true,
+            true,
+            false
+          ],
+          "output_norm": false
+        }
+      },
+      "pixel_dim": 256,
+      "pixel_encoder": {
+        "ms_dims": [
+          1024,
+          512,
+          256,
+          64,
+          3
+        ],
+        "type": "resnet50"
+      },
+      "pixel_mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "pixel_pe_scale": 32,
+      "pixel_pe_temperature": 128,
+      "pixel_std": [
+        0.229,
+        0.224,
+        0.225
+      ],
+      "sensory_dim": 256,
+      "value_dim": 256
+    },
+    "output_dir": null,
+    "save_all": true,
+    "save_aux": false,
+    "save_scores": false,
+    "stagger_updates": 5,
+    "top_k": 30,
+    "use_all_masks": false,
+    "use_long_term": false,
+    "visualize": false
+  },
+  "single_object": true
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70d3bf1d85d0aaf2020f9ef3577239f4f83b77c2ba47fca1eebaaf872f9ad40f
+size 141199924