Initial commit

.gitattributes

@@ -0,0 +1,184 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/docs/teaser.jpg filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_camel/00.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_camel/01.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_camel/02.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_camel/03.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_camel/04.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_camel/05.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_camel/06.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_camel/07.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_camel/08.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_camel/09.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_camel/10.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_camel/11.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_camel/12.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_camel/13.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_camel/14.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_camel/15.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_camel.gif filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_flamingo/00.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_flamingo/01.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_flamingo/02.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_flamingo/03.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_flamingo/04.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_flamingo/05.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_flamingo/06.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_flamingo/07.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_flamingo/08.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_flamingo/09.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_flamingo/10.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_flamingo/11.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_flamingo/12.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_flamingo.gif filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_horse/00.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_horse/01.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_horse/02.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_horse/03.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_horse/04.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_horse/05.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_horse/06.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_horse/07.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_horse/08.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_horse/09.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_horse/10.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_horse/11.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_horse/12.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_horse/13.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_horse/14.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_horse/15.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/davis_horse.gif filter=lfs diff=lfs merge=lfs -text
+assets/examples/kangaroo/00.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/kangaroo/01.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/kangaroo/02.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/kangaroo/03.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/kangaroo/04.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/kangaroo/05.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/kangaroo/06.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/kangaroo/07.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/kangaroo/08.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/kangaroo/09.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/kangaroo/10.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/kangaroo/11.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/kangaroo/12.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/kangaroo/13.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/kangaroo/14.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/kangaroo/15.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/kangaroo.gif filter=lfs diff=lfs merge=lfs -text
+assets/examples/spring/00.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/spring/01.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/spring/02.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/spring/03.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/spring/04.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/spring/05.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/spring/06.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/spring/07.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/spring/08.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/spring/09.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/spring/10.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/spring/11.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/spring/12.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/spring/13.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/spring/14.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/spring/15.png filter=lfs diff=lfs merge=lfs -text
+assets/examples/spring.gif filter=lfs diff=lfs merge=lfs -text
+assets/camel_demo.glb filter=lfs diff=lfs merge=lfs -text
+assets/davis_camel/00.png filter=lfs diff=lfs merge=lfs -text
+assets/davis_camel/01.png filter=lfs diff=lfs merge=lfs -text
+assets/davis_camel/02.png filter=lfs diff=lfs merge=lfs -text
+assets/davis_camel/03.png filter=lfs diff=lfs merge=lfs -text
+assets/davis_camel/04.png filter=lfs diff=lfs merge=lfs -text
+assets/davis_camel/05.png filter=lfs diff=lfs merge=lfs -text
+assets/davis_camel/06.png filter=lfs diff=lfs merge=lfs -text
+assets/davis_camel/07.png filter=lfs diff=lfs merge=lfs -text
+assets/davis_camel/08.png filter=lfs diff=lfs merge=lfs -text
+assets/davis_camel/09.png filter=lfs diff=lfs merge=lfs -text
+assets/davis_camel/10.png filter=lfs diff=lfs merge=lfs -text
+assets/davis_camel/11.png filter=lfs diff=lfs merge=lfs -text
+assets/davis_camel/12.png filter=lfs diff=lfs merge=lfs -text
+assets/davis_camel/13.png filter=lfs diff=lfs merge=lfs -text
+assets/davis_camel/14.png filter=lfs diff=lfs merge=lfs -text
+assets/davis_camel/15.png filter=lfs diff=lfs merge=lfs -text
+assets/davis_camel.gif filter=lfs diff=lfs merge=lfs -text
+assets/davis_flamingo/00.png filter=lfs diff=lfs merge=lfs -text
+assets/davis_flamingo/01.png filter=lfs diff=lfs merge=lfs -text
+assets/davis_flamingo/02.png filter=lfs diff=lfs merge=lfs -text
+assets/davis_flamingo/03.png filter=lfs diff=lfs merge=lfs -text
+assets/davis_flamingo/04.png filter=lfs diff=lfs merge=lfs -text
+assets/davis_flamingo/05.png filter=lfs diff=lfs merge=lfs -text
+assets/davis_flamingo/06.png filter=lfs diff=lfs merge=lfs -text
+assets/davis_flamingo/07.png filter=lfs diff=lfs merge=lfs -text
+assets/davis_flamingo/08.png filter=lfs diff=lfs merge=lfs -text
+assets/davis_flamingo/09.png filter=lfs diff=lfs merge=lfs -text
+assets/davis_flamingo/10.png filter=lfs diff=lfs merge=lfs -text
+assets/davis_flamingo/11.png filter=lfs diff=lfs merge=lfs -text
+assets/davis_flamingo/12.png filter=lfs diff=lfs merge=lfs -text
+assets/davis_flamingo.gif filter=lfs diff=lfs merge=lfs -text
+assets/kangaroo/00.png filter=lfs diff=lfs merge=lfs -text
+assets/kangaroo/01.png filter=lfs diff=lfs merge=lfs -text
+assets/kangaroo/02.png filter=lfs diff=lfs merge=lfs -text
+assets/kangaroo/03.png filter=lfs diff=lfs merge=lfs -text
+assets/kangaroo/04.png filter=lfs diff=lfs merge=lfs -text
+assets/kangaroo/05.png filter=lfs diff=lfs merge=lfs -text
+assets/kangaroo/06.png filter=lfs diff=lfs merge=lfs -text
+assets/kangaroo/07.png filter=lfs diff=lfs merge=lfs -text
+assets/kangaroo/08.png filter=lfs diff=lfs merge=lfs -text
+assets/kangaroo/09.png filter=lfs diff=lfs merge=lfs -text
+assets/kangaroo/10.png filter=lfs diff=lfs merge=lfs -text
+assets/kangaroo/11.png filter=lfs diff=lfs merge=lfs -text
+assets/kangaroo/12.png filter=lfs diff=lfs merge=lfs -text
+assets/kangaroo/13.png filter=lfs diff=lfs merge=lfs -text
+assets/kangaroo/14.png filter=lfs diff=lfs merge=lfs -text
+assets/kangaroo/15.png filter=lfs diff=lfs merge=lfs -text
+assets/kangaroo.gif filter=lfs diff=lfs merge=lfs -text
+assets/spring/00.png filter=lfs diff=lfs merge=lfs -text
+assets/spring/01.png filter=lfs diff=lfs merge=lfs -text
+assets/spring/02.png filter=lfs diff=lfs merge=lfs -text
+assets/spring/03.png filter=lfs diff=lfs merge=lfs -text
+assets/spring/04.png filter=lfs diff=lfs merge=lfs -text
+assets/spring/05.png filter=lfs diff=lfs merge=lfs -text
+assets/spring/06.png filter=lfs diff=lfs merge=lfs -text
+assets/spring/07.png filter=lfs diff=lfs merge=lfs -text
+assets/spring/08.png filter=lfs diff=lfs merge=lfs -text
+assets/spring/09.png filter=lfs diff=lfs merge=lfs -text
+assets/spring/10.png filter=lfs diff=lfs merge=lfs -text
+assets/spring/11.png filter=lfs diff=lfs merge=lfs -text
+assets/spring/12.png filter=lfs diff=lfs merge=lfs -text
+assets/spring/13.png filter=lfs diff=lfs merge=lfs -text
+assets/spring/14.png filter=lfs diff=lfs merge=lfs -text
+assets/spring/15.png filter=lfs diff=lfs merge=lfs -text
+assets/spring.gif filter=lfs diff=lfs merge=lfs -text

LICENSE.txt

@@ -0,0 +1,133 @@
+INSTRUCTIONS FOR USE  - NOT FOR INCLUSION IN REPO
+This license should not be modified.  If you wish to change terms in the license (e.g., remove the noncommercial provision), please work with your product counsel to determine whether a bespoke, model-specific license is appropriate.  Any modified version of this license should have a distinct license name to avoid confusion.
+Confirm with your product counsel (i.e., the legal POC assigned to your RR/OS review) that this license is appropriate for your use-case. This license should be applied where a FAIR team seeks to release a generative  model (including Llama derivatives) or model that otherwise requires more robust terms and an AUP (per case-by-case assessment from PXFN).
+Before the license goes live, please add the responsible STOs to the “List of Projects” tab in the FAIR License Tracker.  Note, to the extent there is a report of a violation of the FAIR License AUP, the STOs responsible for the model release will be responsible for any required investigation or action.
+Update the “Last Updated” date with the date of the model release.
+*Any release of data, code, or model weights requires documented legal approval.
+
+* * *
+
+FAIR Noncommercial Research License
+v1 Last Updated: August 18, 2025
+
+“Acceptable Use Policy” means the FAIR Acceptable Use Policy, applicable to Research Materials, that is incorporated into this Agreement.
+
+“Agreement” means the terms and conditions for use, reproduction, distribution and modification of the Research Materials set forth herein.
+
+
+“Documentation” means the specifications, manuals and documentation accompanying
+Research Materials distributed by Meta.
+
+
+“Licensee” or “you” means you, or your employer or any other person or entity (if you are entering into this Agreement on such person or entity’s behalf), of the age required under applicable laws, rules or regulations to provide legal consent and that has legal authority to bind your employer or such other person or entity if you are entering in this Agreement on their behalf.
+
+
+“Meta” or “we” means Meta Platforms Ireland Limited (if you are located in or, if you are an entity, your principal place of business is in the EEA or Switzerland) and Meta Platforms, Inc. (if you are located outside of the EEA or Switzerland).
+
+“Noncommercial Research Uses” means noncommercial research use cases related to research, development, education, processing, or analysis and in each case, is not primarily intended for commercial advantage or monetary compensation to you or others.
+
+“Research Materials” means, collectively, Documentation and the models, software and algorithms, including machine-learning model code, trained model weights, inference-enabling code, training-enabling code, fine-tuning enabling code, demonstration materials and other elements of the foregoing distributed by Meta and made available under this Agreement.
+
+By clicking “I Accept” below or by using or distributing any portion or element of the Research Materials, you agree to be bound by this Agreement.
+
+
+1. License Rights and Redistribution.
+
+
+a. Grant of Rights. You are granted a non-exclusive, worldwide, non-transferable and royalty-free limited license under Meta’s intellectual property or other rights owned by Meta embodied in the Research Materials to use, reproduce, distribute, copy, create derivative works of, and make modifications to the Research Materials.
+
+b. Redistribution and Use.
+ i. You will not use the Research Materials or any outputs or results of the Research Materials in connection with any commercial uses or for any uses other than Noncommercial Research Uses;
+
+
+ii. Distribution of Research Materials, and any derivative works thereof, are subject to the terms of this Agreement. If you distribute or make the Research Materials, or any derivative works thereof, available to a third party, you may only do so under the terms of this Agreement. You shall also provide a copy of this Agreement to such third party.
+
+
+iii.  If you submit for publication the results of research you perform on, using, or otherwise in connection with Research Materials, you must acknowledge the use of Research Materials in your publication.
+
+
+iv. Your use of the Research Materials must comply with applicable laws and regulations (including Trade Control Laws) and adhere to the FAIR Acceptable Use Policy, which is hereby incorporated by reference into this Agreement.
+2. User Support. Your Noncommercial Research Use of the Research Materials is done at your own discretion; Meta does not process any information nor provide any service in relation to such use.  Meta is under no obligation to provide any support services for the Research Materials. Any support provided is “as is”, “with all faults”, and without warranty of any kind.
+
+
+3. Disclaimer of Warranty. UNLESS REQUIRED BY APPLICABLE LAW, THE RESEARCH MATERIALS AND ANY OUTPUT AND RESULTS THEREFROM ARE PROVIDED ON AN “AS IS” BASIS, WITHOUT WARRANTIES OF ANY KIND, AND META DISCLAIMS ALL WARRANTIES OF ANY KIND, BOTH EXPRESS AND IMPLIED, INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. YOU ARE SOLELY RESPONSIBLE FOR DETERMINING THE APPROPRIATENESS OF USING OR REDISTRIBUTING THE RESEARCH MATERIALS AND ASSUME ANY RISKS ASSOCIATED WITH YOUR USE OF THE RESEARCH MATERIALS AND ANY OUTPUT AND RESULTS.
+
+4. Limitation of Liability. IN NO EVENT WILL META OR ITS AFFILIATES BE LIABLE UNDER ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, TORT, NEGLIGENCE, PRODUCTS LIABILITY, OR OTHERWISE, ARISING OUT OF THIS AGREEMENT, FOR ANY LOST PROFITS OR ANY DIRECT OR INDIRECT, SPECIAL, CONSEQUENTIAL, INCIDENTAL, EXEMPLARY OR PUNITIVE DAMAGES, EVEN IF META OR ITS AFFILIATES HAVE BEEN ADVISED OF THE POSSIBILITY OF ANY OF THE FOREGOING.
+
+5. Intellectual Property.
+
+
+a. Subject to Meta’s ownership of Research Materials and derivatives made by or for Meta, with respect to any derivative works and modifications of the Research Materials that are made by you, as between you and Meta, you are and will be the owner of such derivative works and modifications.
+
+b. If you institute litigation or other proceedings against Meta or any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Research Materials, outputs or results, or any portion of any of the foregoing, constitutes infringement of intellectual property or other rights owned or licensable by you, then any licenses granted to you under this Agreement shall terminate as of the date such litigation or claim is filed or instituted. You will indemnify and hold harmless Meta from and against any claim by any third party arising out of or related to your use or distribution of the Research Materials.
+
+6. Term and Termination. The term of this Agreement will commence upon your acceptance of this Agreement or access to the Research Materials and will continue in full force and effect until terminated in accordance with the terms and conditions herein. Meta may terminate this Agreement if you are in breach of any term or condition of this Agreement. Upon termination of this Agreement, you shall delete and cease use of the Research Materials. Sections 3, 4 and 7 shall survive the termination of this Agreement.
+
+7. Governing Law and Jurisdiction. This Agreement will be governed and construed under the laws of the State of California without regard to choice of law principles, and the UN Convention on Contracts for the International Sale of Goods does not apply to this Agreement. The courts of California shall have exclusive jurisdiction of any dispute arising out of this Agreement.
+
+
+8. Modifications and Amendments. Meta may modify this Agreement from time to time; provided that they are similar in spirit to the current version of the Agreement, but may differ in detail to address new problems or concerns. All such changes will be effective immediately. Your continued use of the Research Materials after any modification to this Agreement constitutes your agreement to such modification. Except as provided in this Agreement, no modification or addition to any provision of this Agreement will be binding unless it is in writing and signed by an authorized representative of both you and Meta.
+
+
+FAIR Acceptable Use Policy
+
+The Fundamental AI Research (FAIR) team at Meta seeks to further understanding of new and existing research domains with the mission of advancing the state-of-the-art in artificial intelligence through open research for the benefit of all.
+
+As part of this mission, Meta makes certain research materials available for noncommercial research use. Meta is committed to promoting the safe and responsible use of such research materials.
+
+Prohibited Uses
+
+You agree you will not use, or allow others to use, Research Materials to:
+
+ Violate the law or others’ rights, including to:
+Engage in, promote, generate, contribute to, encourage, plan, incite, or further illegal or unlawful activity or content, such as:
+Violence or terrorism
+Exploitation or harm to children, including the solicitation, creation, acquisition, or dissemination of child exploitative content or failure to report Child Sexual Abuse Material
+Human trafficking, exploitation, and sexual violence
+The illegal distribution of information or materials to minors, including obscene materials, or failure to employ legally required age-gating in connection with such information or materials.
+Sexual solicitation
+Any other criminal activity
+
+Engage in, promote, incite, or facilitate the harassment, abuse, threatening, or bullying of individuals or groups of individuals
+
+Engage in, promote, incite, or facilitate discrimination or other unlawful or harmful conduct in the provision of employment, employment benefits, credit, housing, other economic benefits, or other essential goods and services
+
+Engage in the unauthorized or unlicensed practice of any profession including, but not limited to, financial, legal, medical/health, or related professional practices
+
+Collect, process, disclose, generate, or infer health, demographic, or other sensitive personal or private information about individuals without rights and consents required by applicable laws
+
+Engage in or facilitate any action or generate any content that infringes, misappropriates, or otherwise violates any third-party rights, including the outputs or results of any technology using FAIR research materials
+
+Create, generate, or facilitate the creation of malicious code, malware, computer viruses or do anything else that could disable, overburden, interfere with or impair the proper working, integrity, operation or appearance of a website or computer system
+
+2. Engage in, promote, incite, facilitate, or assist in the planning or development of activities that present a risk of death or bodily harm to individuals, including use of research artifacts related to the following:
+
+Military, warfare, nuclear industries or applications, espionage, use for materials or activities that are subject to the International Traffic Arms Regulations (ITAR) maintained by the United States Department of State
+
+Guns and illegal weapons (including weapon development)
+
+Illegal drugs and regulated/controlled substances
+
+Operation of critical infrastructure, transportation technologies, or heavy machinery
+
+Self-harm or harm to others, including suicide, cutting, and eating disorders
+
+Any content intended to incite or promote violence, abuse, or any infliction of bodily harm to an individual
+
+3. Intentionally deceive or mislead others, including use of FAIR Research Materials related to the following:
+
+ Generating, promoting, or furthering fraud or the creation or promotion of disinformation
+
+ Generating, promoting, or furthering defamatory content, including the creation of defamatory statements, images, or other content
+
+Generating, promoting, or further distributing spam
+
+ Impersonating another individual without consent, authorization, or legal right
+
+Representing that outputs of FAIR research materials or outputs from technology using FAIR research materials are human-generated
+
+Generating or facilitating false online engagement, including fake reviews and other means of fake online engagement
+
+4. Fail to appropriately disclose to end users any known dangers of your Research Materials.
+
+Please report any violation of this Policy or other problems that could lead to a violation of this Policy by submitting a report here [https://docs.google.com/forms/d/e/1FAIpQLSeb11cryAopJ7LNrC4nxEUXrHY26hfkXQMf_uH-oFgA3WlYZQ/viewform].

README.md

@@ -0,0 +1,11 @@
+---
+title: ActionMesh
+emoji: 🎬
+colorFrom: purple
+colorTo: pink
+sdk: gradio
+sdk_version: 6.3.0
+app_file: app.py
+pinned: false
+license: other
+---

app.py

@@ -0,0 +1,676 @@
+"""
+ActionMesh Gradio Demo
+
+A complete demo for video-to-4D mesh generation using ActionMesh.
+Input: Video file or list of images
+Output: Animated GLB mesh with shape key animation
+"""
+
+import glob
+import logging
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+import gradio as gr
+import spaces
+import torch
+
+# Configure logging for actionmesh modules
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+)
+logger = logging.getLogger(__name__)
+
+
+# Path to examples directory
+EXAMPLES_DIR = Path(__file__).parent / "assets"
+
+
+# --- Setup functions ---
+def setup_blender() -> Path:
+    """
+    Download and setup Blender 3.5.1 for Linux x64.
+
+    Downloads Blender from the official release page if not already present,
+    extracts it, and returns the path to the blender executable.
+
+    Returns:
+        Path to the blender executable.
+    """
+    import tarfile
+    import urllib.request
+
+    # Define paths
+    repo_dir = Path(__file__).parent.parent
+    third_party_dir = repo_dir / "third_party"
+    blender_archive = third_party_dir / "blender-3.5.1-linux-x64.tar.xz"
+    blender_dir = third_party_dir / "blender-3.5.1-linux-x64"
+    blender_executable = blender_dir / "blender"
+
+    # Create third_party directory if it doesn't exist
+    third_party_dir.mkdir(parents=True, exist_ok=True)
+
+    # Check if Blender is already installed
+    if blender_executable.exists():
+        print(f"Blender already installed at {blender_executable}")
+        return blender_executable
+
+    # Download URL
+    blender_url = (
+        "https://download.blender.org/release/Blender3.5/"
+        "blender-3.5.1-linux-x64.tar.xz"
+    )
+
+    # Download Blender if archive doesn't exist
+    if not blender_archive.exists():
+        print(f"Downloading Blender from {blender_url}...")
+        try:
+            urllib.request.urlretrieve(blender_url, blender_archive)
+            print("Blender downloaded successfully.")
+        except Exception as e:
+            raise RuntimeError(f"Failed to download Blender: {e}")
+
+    # Extract the archive
+    print(f"Extracting Blender to {third_party_dir}...")
+    try:
+        with tarfile.open(blender_archive, "r:xz") as tar:
+            tar.extractall(path=third_party_dir)
+        print("Blender extracted successfully.")
+    except Exception as e:
+        # Clean up partial extraction
+        if blender_dir.exists():
+            shutil.rmtree(blender_dir)
+        raise RuntimeError(f"Failed to extract Blender: {e}")
+
+    # Optionally remove the archive to save space
+    if blender_archive.exists():
+        blender_archive.unlink()
+        print("Removed Blender archive to save space.")
+
+    # Verify installation
+    if not blender_executable.exists():
+        raise RuntimeError(
+            f"Blender executable not found at expected path: " f"{blender_executable}"
+        )
+
+    print(f"Blender installed successfully at {blender_executable}")
+    return blender_executable
+
+
+def setup_actionmesh():
+    """Clone and install ActionMesh if not already installed."""
+    cache_dir = Path.home() / ".cache" / "actionmesh"
+
+    try:
+        import actionmesh
+
+        print("ActionMesh already installed.")
+        # Still need to add paths for current process
+        actionmesh_path = str(cache_dir.resolve())
+        if actionmesh_path not in sys.path:
+            sys.path.insert(0, actionmesh_path)
+        triposg_path = str((cache_dir / "third_party" / "TripoSG").resolve())
+        if triposg_path not in sys.path:
+            sys.path.insert(0, triposg_path)
+        return cache_dir
+    except ImportError:
+        pass
+
+    print("Cloning ActionMesh...")
+    if cache_dir.exists():
+        shutil.rmtree(cache_dir)
+    cache_dir.parent.mkdir(parents=True, exist_ok=True)
+
+    subprocess.run(
+        [
+            "git",
+            "clone",
+            "https://github.com/facebookresearch/actionmesh.git",
+            str(cache_dir),
+        ],
+        check=True,
+    )
+    print("ActionMesh cloned successfully.")
+
+    # Configure git to use HTTPS instead of SSH (for submodules)
+    subprocess.run(
+        [
+            "git",
+            "config",
+            "--global",
+            "url.https://github.com/.insteadOf",
+            "git@github.com:",
+        ],
+        check=True,
+    )
+
+    # Initialize submodules
+    print("Initializing submodules...")
+    subprocess.run(
+        ["git", "submodule", "update", "--init", "--recursive"],
+        cwd=cache_dir,
+        check=True,
+    )
+    print("Submodules initialized successfully.")
+
+    # Install actionmesh in editable mode (ignore Python version requirement)
+    print("Installing ActionMesh...")
+    subprocess.run(
+        [sys.executable, "-m", "pip", "install", "-e", ".", "--ignore-requires-python"],
+        cwd=cache_dir,
+        check=True,
+    )
+    print("ActionMesh installed successfully.")
+
+    # Add actionmesh to Python path for current process
+    actionmesh_path = str(cache_dir.resolve())
+    if actionmesh_path not in sys.path:
+        sys.path.insert(0, actionmesh_path)
+
+    # Add TripoSG (submodule) to Python path for current process
+    triposg_path = str((cache_dir / "third_party" / "TripoSG").resolve())
+    if triposg_path not in sys.path:
+        sys.path.insert(0, triposg_path)
+
+    return cache_dir
+
+
+def setup_environment():
+    """Setup the complete environment for ActionMesh."""
+    print("=" * 50)
+    print("Setting up ActionMesh environment...")
+    print("=" * 50)
+
+    # Clone and install ActionMesh if needed
+    setup_actionmesh()
+    blender_path = setup_blender()
+
+    print("=" * 50)
+    print("Environment setup complete!")
+    print("=" * 50)
+    return blender_path
+
+
+# Run setup on import
+blender_path = setup_environment()
+
+
+from actionmesh.io.glb_export import create_animated_glb
+from actionmesh.io.mesh_io import save_deformation
+
+# --- Import ActionMesh modules after setup ---
+from actionmesh.io.video_input import load_frames
+from actionmesh.pipeline import ActionMeshPipeline
+from actionmesh.render.utils import save_rgba_video
+
+# Global pipeline instance (loaded on CPU at startup)
+pipeline: ActionMeshPipeline | None = None
+
+
+def get_available_examples() -> list[tuple[str, str]]:
+    """
+    Get available examples from the assets directory.
+
+    Returns:
+        List of tuples (display_name, example_dir_path) for each example.
+    """
+    examples = []
+    if EXAMPLES_DIR.exists():
+        for example_dir in sorted(EXAMPLES_DIR.iterdir()):
+            if example_dir.is_dir():
+                # Get the first image as a thumbnail
+                images = sorted(glob.glob(str(example_dir / "*.png")))
+                if images:
+                    display_name = example_dir.name.replace("_", " ").title()
+                    examples.append((display_name, str(example_dir)))
+    return examples
+
+
+def get_example_thumbnails() -> list[str]:
+    """
+    Get thumbnail images/GIFs for all available examples.
+
+    Looks for a GIF file named "{folder_name}.gif" in the same parent directory
+    as the example folder. Falls back to the first PNG image if no GIF is found.
+
+    Returns:
+        List of paths to the GIF or first image of each example.
+    """
+    thumbnails = []
+    if EXAMPLES_DIR.exists():
+        for example_dir in sorted(EXAMPLES_DIR.iterdir()):
+            if example_dir.is_dir():
+                # Try to find a GIF with the same name as the folder
+                gif_path = example_dir.parent / f"{example_dir.name}.gif"
+                if gif_path.exists():
+                    thumbnails.append(str(gif_path))
+                else:
+                    # Fall back to first PNG image
+                    images = sorted(glob.glob(str(example_dir / "*.png")))
+                    if images:
+                        thumbnails.append(images[0])
+    return thumbnails
+
+
+def load_example_images(evt: gr.SelectData) -> list[str]:
+    """
+    Load images from the selected example.
+
+    Args:
+        evt: Gradio SelectData event containing the selected index.
+
+    Returns:
+        List of image paths from the selected example.
+    """
+    examples = get_available_examples()
+    if evt.index < len(examples):
+        _, example_dir = examples[evt.index]
+        images = sorted(glob.glob(os.path.join(example_dir, "*.png")))
+        return images
+    return []
+
+
+def load_pipeline_cpu() -> ActionMeshPipeline:
+    """Load the ActionMesh pipeline on CPU (called once at module load)."""
+    global pipeline
+    if pipeline is None:
+        print("Loading ActionMesh pipeline on CPU...")
+        # Get config path from actionmesh cache directory
+        cache_dir = Path.home() / ".cache" / "actionmesh"
+        config_dir = str(cache_dir / "actionmesh" / "configs")
+        pipeline = ActionMeshPipeline(
+            config_name="actionmesh.yaml",
+            config_dir=config_dir,
+        )
+        print("Pipeline loaded on CPU successfully.")
+    return pipeline
+
+
+# Initialize pipeline on CPU at module load (outside GPU time)
+print("Initializing pipeline on CPU...")
+load_pipeline_cpu()
+print("Pipeline ready (on CPU).")
+
+
+def _run_actionmesh_impl(
+    video_input: str | None,
+    image_files: list[str] | None,
+    seed: int,
+    reference_frame: int,
+    quality_mode: str,
+    progress: gr.Progress = gr.Progress(),
+) -> tuple[str | None, str | None, str | None, str]:
+    """
+    Internal implementation of ActionMesh pipeline.
+
+    Args:
+        video_input: Path to input video file.
+        image_files: List of paths to input image files.
+        seed: Random seed for generation.
+        reference_frame: Reference frame index (1-indexed).
+        quality_mode: Quality mode string.
+        progress: Gradio progress tracker.
+
+    Returns:
+        Tuple of (animated_glb_path, animated_glb_path, input_video_path, status_message)
+    """
+    # Create temporary output directory
+    output_dir = tempfile.mkdtemp(prefix="actionmesh_")
+
+    try:
+        # Determine input source
+        progress(0.1, desc="Loading input...")
+
+        if video_input is not None:
+            input_path = video_input
+        elif image_files is not None and len(image_files) > 0:
+            # Create temp directory with images
+            img_dir = os.path.join(output_dir, "input_images")
+            os.makedirs(img_dir, exist_ok=True)
+            for i, img_path in enumerate(image_files):
+                ext = Path(img_path).suffix
+                shutil.copy(img_path, os.path.join(img_dir, f"{i:04d}{ext}"))
+            input_path = img_dir
+        else:
+            return None, None, None, "Error: Please provide a video or images."
+
+        # Load input
+        input_data = load_frames(path=input_path, max_frames=16)
+
+        if input_data.n_frames < 16:
+            return None, None, None, "Error: At least 16 frames are required."
+
+        # Get pipeline and move to GPU
+        progress(0.2, desc="Moving pipeline to GPU...")
+        pipe = load_pipeline_cpu()
+        pipe.to("cuda")
+
+        # Clear GPU cache before inference
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+
+        # Run inference
+        progress(0.3, desc="Running ActionMesh inference...")
+
+        # Set steps based on quality mode
+        if quality_mode == "⚡ Fast":
+            stage_0_steps = 50
+            stage_1_steps = 15
+        else:  # High Quality
+            stage_0_steps = 100
+            stage_1_steps = 30
+
+        meshes = pipe(
+            input=input_data,
+            anchor_idx=reference_frame - 1,  # Convert from 1-indexed UI to 0-indexed
+            stage_0_steps=stage_0_steps,
+            stage_1_steps=stage_1_steps,
+            seed=seed,
+        )
+
+        # Save input video
+        input_video_path = f"{output_dir}/input_video.mp4"
+        save_rgba_video(input_data.frames, output_path=input_video_path)
+
+        if not meshes:
+            return None, None, None, "Error: No meshes generated."
+
+        # Save deformations and create animated GLB
+        progress(0.9, desc="Creating animated GLB...")
+
+        vertices_path, faces_path = save_deformation(
+            meshes, path=f"{output_dir}/deformations"
+        )
+        animated_glb_path = f"{output_dir}/animated_mesh.glb"
+        create_animated_glb(
+            blender_path=blender_path,
+            vertices_npy=vertices_path,
+            faces_npy=faces_path,
+            output_glb=animated_glb_path,
+            fps=8,
+        )
+
+        progress(1.0, desc="Done!")
+        status = f"Success! Generated animated mesh with {len(meshes)} frames."
+
+        return animated_glb_path, animated_glb_path, input_video_path, status
+
+    except Exception as e:
+        return None, None, None, f"Error: {str(e)}"
+
+
+@spaces.GPU(duration=120)
+@torch.no_grad()
+def _run_actionmesh_fast(
+    video_input: str | None,
+    image_files: list[str] | None,
+    seed: int,
+    reference_frame: int,
+    quality_mode: str,
+    progress: gr.Progress = gr.Progress(),
+) -> tuple[str | None, str | None, str | None, str]:
+    """Fast mode wrapper with 120s GPU duration."""
+    return _run_actionmesh_impl(
+        video_input, image_files, seed, reference_frame, quality_mode, progress
+    )
+
+
+@spaces.GPU(duration=240)
+@torch.no_grad()
+def _run_actionmesh_hq(
+    video_input: str | None,
+    image_files: list[str] | None,
+    seed: int,
+    reference_frame: int,
+    quality_mode: str,
+    progress: gr.Progress = gr.Progress(),
+) -> tuple[str | None, str | None, str | None, str]:
+    """High quality mode wrapper with 260s GPU duration."""
+    return _run_actionmesh_impl(
+        video_input, image_files, seed, reference_frame, quality_mode, progress
+    )
+
+
+def run_actionmesh(
+    video_input: str | None,
+    image_files: list[str] | None,
+    seed: int,
+    reference_frame: int,
+    quality_mode: str,
+    progress: gr.Progress = gr.Progress(),
+) -> tuple[str | None, str | None, str | None, str]:
+    """
+    Run ActionMesh pipeline on input video or images.
+
+    Dispatches to the appropriate GPU-decorated function based on quality mode.
+
+    Args:
+        video_input: Path to input video file.
+        image_files: List of paths to input image files.
+        seed: Random seed for generation.
+        reference_frame: Reference frame index (1-indexed).
+        quality_mode: Quality mode string.
+        progress: Gradio progress tracker.
+
+    Returns:
+        Tuple of (animated_glb_path, animated_glb_path, input_video_path, status_message)
+    """
+    if quality_mode == "⚡ Fast":
+        return _run_actionmesh_fast(
+            video_input, image_files, seed, reference_frame, quality_mode, progress
+        )
+    else:
+        return _run_actionmesh_hq(
+            video_input, image_files, seed, reference_frame, quality_mode, progress
+        )
+
+
+def create_demo() -> gr.Blocks:
+    """Create the Gradio demo interface."""
+
+    with gr.Blocks(
+        title="ActionMesh - Video to 4D Mesh",
+        theme=gr.themes.Soft(),
+    ) as demo:
+
+        gr.Markdown(
+            """
+            # 🎬 ActionMesh: Video to Animated 3D Mesh
+
+            [**Project Page**](https://remysabathier.github.io/actionmesh/) · [**GitHub**](https://github.com/facebookresearch/ActionMesh)
+            [Remy Sabathier](https://www.linkedin.com/in/r%C3%A9my-sabathier-97b264179/), [David Novotny](https://d-novotny.github.io/), [Niloy J. Mitra](http://www0.cs.ucl.ac.uk/staff/n.mitra/), [Tom Monnier](https://tmonnier.com/)
+            **[Meta Reality Labs](https://ai.facebook.com/research/)**  · **[SpAItial](https://www.spaitial.ai/)** · **[University College London](https://geometry.cs.ucl.ac.uk/)**
+
+            Generate animated 3D meshes from video input using ActionMesh.
+
+            **Instructions:**
+            1. Upload a video OR multiple images ⚠️ *Input is limited to exactly 16 frames. Extra frames will be discarded.*
+            2. Click "Generate"
+            3. View the animated 4D mesh in the viewer
+            4. Download the animated GLB mesh (ready for Blender)
+
+            ⏱️ **Performance:** Inference on HuggingFace Space (ZeroGPU) is 2x slower than running locally.
+            We recommend **Fast mode** (90s). For faster inference, run [locally via GitHub](https://github.com/facebookresearch/ActionMesh).
+            """
+        )
+
+        with gr.Row():
+            with gr.Column(scale=1):
+                gr.Markdown("### Input")
+
+                gr.Markdown(
+                    """
+                    ℹ️ **Input should have a uniform background**.
+                    See our [SAM2 tutorial](https://github.com/facebookresearch/actionmesh/blob/main/assets/docs/sam2_extraction_guide.md) to preprocess any video with background removal.
+                    """
+                )
+
+                with gr.Tab("Video"):
+                    video_input = gr.Video(
+                        label="Upload Video",
+                        sources=["upload"],
+                    )
+
+                with gr.Tab("Images"):
+                    image_input = gr.File(
+                        label="Upload Images (multiple frames)",
+                        file_count="multiple",
+                        file_types=["image"],
+                    )
+
+                # Examples gallery
+                example_thumbnails = get_example_thumbnails()
+                if example_thumbnails:
+                    gr.Markdown("### 📁 Example videos")
+                    gr.Markdown("*Click a video example to load it*")
+                    example_labels = [e[0] for e in get_available_examples()]
+                    examples_gallery = gr.Gallery(
+                        value=[
+                            (thumb, label)
+                            for thumb, label in zip(example_thumbnails, example_labels)
+                        ],
+                        columns=3,
+                        rows=2,
+                        height=350,
+                        allow_preview=False,
+                        object_fit="cover",
+                    )
+
+                gr.Markdown("### Parameters")
+
+                quality_mode = gr.Radio(
+                    label="Generation Mode",
+                    choices=["⚡ Fast", "✨ High Quality"],
+                    value="⚡ Fast",
+                    interactive=True,
+                    info="⚡ Fast: ~90s, ✨ High Quality: ~3min30s",
+                )
+
+                reference_frame = gr.Slider(
+                    minimum=1,
+                    maximum=16,
+                    value=1,
+                    step=1,
+                    label="Reference Frame",
+                    info="Frame used as reference for 3D generation (1 recommended)",
+                )
+
+                seed = gr.Slider(
+                    minimum=0,
+                    maximum=100,
+                    value=44,
+                    step=1,
+                    label="Random Seed",
+                )
+
+                generate_btn = gr.Button("🎬 Generate", variant="primary", size="lg")
+
+            with gr.Column(scale=2):
+                gr.Markdown("### Output")
+
+                status_text = gr.Textbox(
+                    label="Status",
+                    interactive=False,
+                    value="Ready",
+                    lines=2,
+                )
+
+                gr.Markdown("### 4D Viewer")
+
+                # Toggle between input video and 4D mesh viewer
+                viewer_toggle = gr.Radio(
+                    label="Display Mode",
+                    choices=["4D Mesh Viewer", "Input Video"],
+                    value="4D Mesh Viewer",
+                    interactive=True,
+                )
+
+                # 4D mesh display showing animated GLB
+                mesh_display = gr.Model3D(
+                    label="4D Mesh Viewer",
+                    clear_color=[0.9, 0.9, 0.9, 1.0],
+                    height=500,
+                    visible=True,
+                )
+
+                # Input video display
+                input_video_display = gr.Video(
+                    label="Input Video",
+                    height=500,
+                    visible=False,
+                    interactive=False,
+                )
+
+                # Interaction legend for 3D viewer
+                gr.Markdown(
+                    """
+                    <div style="background: #2d3748; padding: 8px 12px; border-radius: 6px; font-size: 0.85em; color: #e2e8f0;">
+                    🖱️ <b>Drag</b> to rotate · <b>Scroll</b> to zoom · <b>Right-click drag</b> to pan
+                    </div>
+                    """,
+                    visible=True,
+                )
+
+                # Download button for the animated GLB
+                download_glb = gr.DownloadButton(
+                    label="Download Animated GLB",
+                    visible=True,
+                )
+
+                # State to store input video path
+                input_video_state = gr.State(value=None)
+
+        # Toggle handler to switch between mesh viewer and input video
+        def toggle_display(choice: str, video_path: str | None):
+            if choice == "4D Mesh Viewer":
+                return gr.update(visible=True), gr.update(visible=False)
+            else:
+                return gr.update(visible=False), gr.update(
+                    visible=True, value=video_path
+                )
+
+        viewer_toggle.change(
+            fn=toggle_display,
+            inputs=[viewer_toggle, input_video_state],
+            outputs=[mesh_display, input_video_display],
+        )
+
+        # Generate button click - runs pipeline and shows animated GLB
+        generate_btn.click(
+            fn=run_actionmesh,
+            inputs=[video_input, image_input, seed, reference_frame, quality_mode],
+            outputs=[
+                mesh_display,
+                download_glb,
+                input_video_state,
+                status_text,
+            ],
+        )
+
+        # Example gallery click - loads example images into the image input
+        if example_thumbnails:
+            examples_gallery.select(
+                fn=load_example_images,
+                inputs=None,
+                outputs=image_input,
+            )
+
+        gr.Markdown(
+            """
+            ---
+            **Note:** This demo requires a GPU with sufficient VRAM.
+            """
+        )
+
+    return demo
+
+
+if __name__ == "__main__":
+    demo = create_demo()
+    demo.queue()
+    demo.launch(share=True)

assets/camel_demo.glb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a40f4afc9375854179f8816ef5cc1af654e3e100d68da671cb3caaabe98d8660
+size 4066872