pipeline.py

# Copyright 2026 Echo Team and The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Echo-Memory community pipeline for official Wan 2.1 Diffusers weights.

Loads `Wan-AI/Wan2.1-T2V-1.3B-Diffusers`, then overlays the released
`context_k1` row from `Echo-Team/Echo-Memory` after remapping original
DiffSynth / Wan keys onto the Diffusers transformer.

This is a community overlay, not a new official Wan checkpoint. Extra
action-MLP / SSM slots stay in the Echo-Memory research stack.

Paper: https://arxiv.org/abs/2606.09803
Code: https://github.com/Echo-Team-Joy-Future-Academy-JD/Echo-Memory
"""

from typing import Dict, Iterable, List, Optional, Tuple

import torch
from huggingface_hub import hf_hub_download
from safetensors.torch import load_file

from diffusers import WanPipeline
from diffusers.utils import logging


logger = logging.get_logger(__name__)  # pylint: disable=invalid-name

DEFAULT_BASE_MODEL = "Wan-AI/Wan2.1-T2V-1.3B-Diffusers"
DEFAULT_REPO_ID = "Echo-Team/Echo-Memory"
DEFAULT_FILENAME = "context_k1/epoch-0.safetensors"
DEFAULT_CONVERTED_REPO_ID = "Wayne-King/echo-memory-diffusers"
DEFAULT_CONVERTED_FILENAME = "context_k1-diffusers/diffusion_pytorch_model.safetensors"

SKIP_SUBSTRINGS = (
    "action_mlp",
    "self_attn_with_action",
    "block_wise_ssm",
    "videossm_hybrid",
    "spatial_memory_module",
)

# Same mapping as `scripts/convert_wan_to_diffusers.py` for Wan 2.1 T2V.
# Duplicated here because that script is not an importable package.
TRANSFORMER_KEYS_RENAME_DICT = {
    "time_embedding.0": "condition_embedder.time_embedder.linear_1",
    "time_embedding.2": "condition_embedder.time_embedder.linear_2",
    "text_embedding.0": "condition_embedder.text_embedder.linear_1",
    "text_embedding.2": "condition_embedder.text_embedder.linear_2",
    "time_projection.1": "condition_embedder.time_proj",
    "head.modulation": "scale_shift_table",
    "head.head": "proj_out",
    "modulation": "scale_shift_table",
    "ffn.0": "ffn.net.0.proj",
    "ffn.2": "ffn.net.2",
    # The original model names norms as norm1, norm3, norm2.
    # Diffusers uses norm1, norm2, norm3.
    "norm2": "norm__placeholder",
    "norm3": "norm2",
    "norm__placeholder": "norm3",
    "self_attn.q": "attn1.to_q",
    "self_attn.k": "attn1.to_k",
    "self_attn.v": "attn1.to_v",
    "self_attn.o": "attn1.to_out.0",
    "self_attn.norm_q": "attn1.norm_q",
    "self_attn.norm_k": "attn1.norm_k",
    "cross_attn.q": "attn2.to_q",
    "cross_attn.k": "attn2.to_k",
    "cross_attn.v": "attn2.to_v",
    "cross_attn.o": "attn2.to_out.0",
    "cross_attn.norm_q": "attn2.norm_q",
    "cross_attn.norm_k": "attn2.norm_k",
}


def is_diffusers_transformer_state_dict(keys: Iterable[str]) -> bool:
    keys = list(keys)
    return any(key.startswith("condition_embedder.") or ".attn1." in key for key in keys)


def convert_echo_memory_transformer_state_dict(
    state_dict: Dict[str, torch.Tensor],
    skip_substrings: Iterable[str] = SKIP_SUBSTRINGS,
) -> Tuple[Dict[str, torch.Tensor], List[str]]:
    """Convert original Echo-Memory / DiffSynth Wan keys to Diffusers names."""
    skip_substrings = tuple(skip_substrings)
    if is_diffusers_transformer_state_dict(state_dict):
        converted = {
            key: value
            for key, value in state_dict.items()
            if not any(token in key for token in skip_substrings)
        }
        skipped = [key for key in state_dict if key not in converted]
        return converted, skipped

    converted = {}
    skipped = []
    for key, value in state_dict.items():
        if any(token in key for token in skip_substrings):
            skipped.append(key)
            continue
        new_key = key
        for replace_key, rename_key in TRANSFORMER_KEYS_RENAME_DICT.items():
            new_key = new_key.replace(replace_key, rename_key)
        converted[new_key] = value
    return converted, skipped


class EchoMemoryPipeline(WanPipeline):
    """Wan 2.1 T2V pipeline with an Echo-Memory `context_k1` overlay.

    `load_echo_memory_weights` replaces `self.transformer` parameters in place.
    Call it once after `from_pretrained`, before generation.
    """

    def load_echo_memory_weights(
        self,
        repo_id: str = DEFAULT_REPO_ID,
        filename: str = DEFAULT_FILENAME,
        local_path: Optional[str] = None,
        strict: bool = False,
    ):
        """Download one Echo-Memory row and overlay it on `self.transformer`."""
        if getattr(self, "transformer", None) is None:
            raise ValueError("pipeline.transformer is empty; load Wan 2.1 1.3B before overlaying Echo-Memory.")

        ckpt_path = local_path or hf_hub_download(repo_id=repo_id, filename=filename)
        raw = load_file(ckpt_path)
        converted, skipped = convert_echo_memory_transformer_state_dict(raw)
        missing, unexpected = self.transformer.load_state_dict(converted, strict=strict)
        logger.info(
            "Overlaid %s/%s transformer keys from %s (skipped=%s, missing=%s, unexpected=%s)",
            len(converted),
            len(raw),
            ckpt_path,
            len(skipped),
            len(missing),
            len(unexpected),
        )
        return missing, unexpected, skipped

    def load_converted_echo_memory_weights(
        self,
        repo_id: str = DEFAULT_CONVERTED_REPO_ID,
        filename: str = DEFAULT_CONVERTED_FILENAME,
        local_path: Optional[str] = None,
        strict: bool = False,
    ):
        """Overlay the already-remapped `context_k1` transformer weights."""
        return self.load_echo_memory_weights(
            repo_id=repo_id,
            filename=filename,
            local_path=local_path,
            strict=strict,
        )

    @classmethod
    def from_echo_memory(
        cls,
        pretrained_model_name_or_path: str = DEFAULT_BASE_MODEL,
        echo_memory_repo: str = DEFAULT_REPO_ID,
        echo_memory_filename: str = DEFAULT_FILENAME,
        **kwargs,
    ):
        pipe = cls.from_pretrained(pretrained_model_name_or_path, **kwargs)
        pipe.load_echo_memory_weights(repo_id=echo_memory_repo, filename=echo_memory_filename)
        return pipe