b.sh

#!/usr/bin/env bash
# 将 Modal 镜像内的安装步骤转换为 Bash；在 Debian/Ubuntu 容器或主机中以 root 运行。
# 直接使用当前环境中的 python（找不到时使用 python3），不安装 Python、不创建虚拟环境。
# 项目用 git clone；底模和 LoRA 实体始终在 /marimo 下，项目内仅放目录软链接。
set -Eeuo pipefail

if [[ "$EUID" -ne 0 ]]; then
    echo '请以 root 身份运行：bash build_forge_neo_sdk_marimo.sh' >&2
    exit 1
fi

if [[ -z "${PYTHON_BIN:-}" ]]; then
    if command -v python >/dev/null 2>&1; then
        PYTHON_BIN=python
    else
        PYTHON_BIN=python3
    fi
fi
if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then
    echo "未找到可用的 Python：$PYTHON_BIN" >&2
    exit 1
fi
"$PYTHON_BIN" --version
"$PYTHON_BIN" -m pip --version

if [[ -z "${HF_TOKEN:-}" ]]; then
    read -r -s -p '请输入可访问 ACCC1380/private-model 的 HF_TOKEN：' HF_TOKEN
    printf '\n'
fi
: "${HF_TOKEN:?HF_TOKEN 不能为空}"
export HF_TOKEN
export DEBIAN_FRONTEND=noninteractive

WEBUI_DIR=/marimo/sd-webui-forge-neo
CHECKPOINT_DIR=/marimo/Stable-diffusion
LORA_DIR=/marimo/Lora
FORGE_REPO=https://github.com/Haoming02/sd-webui-forge-classic.git

# 基础工具及常用网络工具
apt-get update
apt-get install -y --no-install-recommends \
    ca-certificates aria2 sshpass openssh-client unzip git \
    procps sudo gnupg psmisc p7zip-full \
    iputils-ping iproute2 dnsutils traceroute net-tools

# PyTorch CUDA 13.0 环境
"$PYTHON_BIN" -m pip install torch torchvision torchaudio \
    --index-url https://download.pytorch.org/whl/cu130

# 保留原来的依赖和版本
"$PYTHON_BIN" -m pip install \
    jupyterlab \
    'GitPython==3.1.46' \
    'Pillow==12.2.0' \
    'accelerate==1.13.0' \
    'audioop-lts==0.2.2; python_version >= "3.13"' \
    'av==17.0.0' \
    'comfy-kitchen==0.2.10' \
    'diffusers==0.37.1' \
    'diskcache==5.6.3' \
    'einops==0.8.2' \
    'facexlib==0.3.0' \
    'fastapi==0.127.1' \
    'httpx==0.24.1' \
    'huggingface-hub==0.36.2' \
    'inflection==0.5.1' \
    'joblib==1.5.3' \
    'kornia==0.6.12' \
    'lark==1.2.2' \
    'numpy==2.3.5' \
    'omegaconf==2.2.3' \
    'opencv-python==4.10.0.84' \
    'peft==0.17.1' \
    'piexif==1.1.3' \
    'pillow-heif==1.3.0' \
    'pillow-jxl-plugin==1.3.7' \
    'protobuf==4.25.9' \
    'psutil==6.1.1' \
    'pydantic-core==2.27.2' \
    'pydantic==2.10.6' \
    'pyyaml==6.0.3' \
    'rich==14.3.3' \
    'safetensors==0.7.0' \
    'scikit-image==0.25.2' \
    'setuptools==69.5.1' \
    'spandrel-extra-arches==0.2.0' \
    'spandrel==0.4.2' \
    'torchdiffeq==0.2.5' \
    'torchsde==0.2.6' \
    'tqdm==4.67.3' \
    'transformers==4.56.2' \
    modelscope

# 使用 Hugging Face 官方 SDK；区分模型仓库和数据集仓库。
# SDK 未完成文件保留供下次续传；每 5 秒打印实际落盘大小和速度。
hf_download() {
    "$PYTHON_BIN" -u - "$1" "$2" <<'HF_PY'
import os
import sys
import threading
import time
from pathlib import Path, PurePosixPath

from huggingface_hub import hf_hub_download
from huggingface_hub.utils import HfHubHTTPError
from requests.exceptions import ConnectionError as RequestConnectionError, Timeout

source_uri, directory = sys.argv[1:]
if source_uri.startswith("hf://datasets/"):
    repo_type, repo_path = "dataset", source_uri[len("hf://datasets/"):]
elif source_uri.startswith("hf://"):
    repo_type, repo_path = "model", source_uri[len("hf://"):]
else:
    sys.exit(f"不支持的 HF 地址：{source_uri}")
parts = repo_path.split("/", 2)
if len(parts) != 3 or not all(parts):
    sys.exit(f"HF 地址缺少仓库名或文件路径：{source_uri}")
owner, repo_name, filename = parts
remote_path = PurePosixPath(filename)
if remote_path.is_absolute() or ".." in remote_path.parts or filename.endswith("/"):
    sys.exit(f"无效的文件路径：{filename}")

local_dir = Path(directory)
local_dir.mkdir(parents=True, exist_ok=True)
target = local_dir / remote_path.name
if target.is_file() and target.stat().st_size > 0:
    print(f"已存在，跳过下载：{target}", flush=True)
    sys.exit(0)

# 下载暂存和最终文件在同一模型目录中，完成后移动到目录顶层。
staging = local_dir / ".hf-downloads" / f"{repo_type}--{owner}--{repo_name}"
staging.mkdir(parents=True, exist_ok=True)
staged_file = staging / filename
finished = threading.Event()

def disk_bytes():
    total = 0
    for path in [staged_file, *staging.rglob("*.incomplete")]:
        try:
            info = path.stat()
            total += info.st_blocks * 512
        except FileNotFoundError:
            pass
    return total

def monitor():
    previous_size, previous_time = disk_bytes(), time.monotonic()
    while not finished.wait(5):
        size, now = disk_bytes(), time.monotonic()
        speed = max(0, size - previous_size) / max(now - previous_time, 0.001)
        print(f"[{target.name}] 已落盘 {size / 1048576:.2f} MiB，"
              f"落盘速度 {speed / 1048576:.2f} MiB/s", flush=True)
        previous_size, previous_time = size, now

print(f"\n开始下载：{source_uri}", flush=True)
watcher = threading.Thread(target=monitor, daemon=True)
watcher.start()
try:
    for attempt in range(6):
        try:
            downloaded = Path(hf_hub_download(
                repo_id=f"{owner}/{repo_name}",
                filename=filename,
                repo_type=repo_type,
                revision="main",
                token=os.environ.get("HF_TOKEN") or None,
                local_dir=str(staging),
            ))
            break
        except (HfHubHTTPError, RequestConnectionError, Timeout) as exc:
            if isinstance(exc, HfHubHTTPError):
                status = getattr(exc.response, "status_code", None)
                if status not in (408, 429) and not (status is not None and status >= 500):
                    raise
            if attempt == 5:
                raise
            print(f"下载暂时失败（{type(exc).__name__}），5 秒后重试 {attempt + 1}/5。", flush=True)
            time.sleep(5)
    if not downloaded.is_file() or downloaded.stat().st_size == 0:
        raise RuntimeError("SDK 未返回有效的非空文件")
    os.replace(downloaded, target)
except Exception as exc:
    print(f"下载失败：{source_uri}\n{type(exc).__name__}: {exc}\n"
          f"SDK 未完成文件保留在：{staging}", file=sys.stderr, flush=True)
    sys.exit(1)
finally:
    finished.set()
    watcher.join()
print(f"下载完成：{target}（{target.stat().st_size} 字节）", flush=True)
HF_PY
}

# 将旧项目内的模型移出。递归合并目录；同名文件保留编号备份，不删除模型。
merge_model_dir() (
    set -Eeuo pipefail
    src_dir="$1"
    dst_dir="$2"
    mkdir -p "$dst_dir"
    if [[ "$src_dir" -ef "$dst_dir" ]]; then
        exit 0
    fi
    shopt -s dotglob nullglob
    for entry in "$src_dir"/*; do
        dest="$dst_dir/${entry##*/}"
        if [[ -d "$entry" && ! -L "$entry" && -d "$dest" && ! -L "$dest" ]]; then
            merge_model_dir "$entry" "$dest"
        else
            mv -T --backup=numbered -- "$entry" "$dest"
        fi
    done
    rmdir -- "$src_dir"
)

# 整个模型目录使用软链接，之后放入 /marimo 的新模型也会直接被项目看到。
link_model_dir() {
    local storage_dir="$1" link_path="$2"
    mkdir -p "$storage_dir" "$(dirname "$link_path")"
    if [[ -L "$link_path" ]]; then
        if [[ "$link_path" -ef "$storage_dir" ]]; then
            return 0
        fi
        # 只替换软链接本身，不删除其原目标目录。
        rm -- "$link_path"
    elif [[ -d "$link_path" ]]; then
        merge_model_dir "$link_path" "$storage_dir"
    elif [[ -e "$link_path" ]]; then
        printf '模型目录位置被普通文件占用：%s\n' "$link_path" >&2
        return 1
    fi
    ln -s -- "$storage_dir" "$link_path"
}

prepare_forge() {
    local kind storage_dir old_dir origin branch backup_dir
    mkdir -p /marimo "$CHECKPOINT_DIR" "$LORA_DIR"
    # 兼容前几版脚本：先把项目内已有的底模和 LoRA 移到项目外。
    for kind in Stable-diffusion Lora; do
        if [[ "$kind" == Lora ]]; then
            storage_dir="$LORA_DIR"
        else
            storage_dir="$CHECKPOINT_DIR"
        fi
        old_dir="$WEBUI_DIR/models/$kind"
        if [[ -d "$old_dir" && ! -L "$old_dir" ]]; then
            merge_model_dir "$old_dir" "$storage_dir"
        fi
    done

    origin=''
    branch=''
    if [[ -d "$WEBUI_DIR/.git" ]]; then
        origin=$(git -C "$WEBUI_DIR" remote get-url origin 2>/dev/null || true)
        branch=$(git -C "$WEBUI_DIR" branch --show-current 2>/dev/null || true)
    fi
    if [[ "${origin%.git}" == "${FORGE_REPO%.git}" && "$branch" == neo ]]; then
        printf '已有 Forge Neo 的 neo 分支，继续使用：%s\n' "$WEBUI_DIR"
    else
        # git clone 不能写入旧的非空目录；旧项目保留在 /marimo 下。
        if [[ -e "$WEBUI_DIR" || -L "$WEBUI_DIR" ]]; then
            backup_dir=$(mktemp -d /marimo/forge-neo-before-git.XXXXXX)
            mv -T -- "$WEBUI_DIR" "$backup_dir/project"
            printf '旧项目已保留：%s/project\n' "$backup_dir"
        fi
        git clone --depth 1 --single-branch --branch neo \
            "$FORGE_REPO" "$WEBUI_DIR"
    fi

    link_model_dir "$CHECKPOINT_DIR" "$WEBUI_DIR/models/Stable-diffusion"
    link_model_dir "$LORA_DIR" "$WEBUI_DIR/models/Lora"
}

df -h
"$PYTHON_BIN" - <<'IP_PY'
import sys
from urllib.request import urlopen

try:
    with urlopen("https://ipinfo.io", timeout=30) as response:
        print(response.read().decode("utf-8"))
except Exception as exc:
    print(f"IP 信息查询失败，继续安装：{exc}", file=sys.stderr)
IP_PY
printf '\n'
mkdir -p ~/.ssh
chmod 700 ~/.ssh
if ! grep -qxF 'StrictHostKeyChecking no' ~/.ssh/config 2>/dev/null; then
    echo 'StrictHostKeyChecking no' >> ~/.ssh/config
fi
chmod 600 ~/.ssh/config

# 用官方仓库 neo 分支替换 sd-neo.tar 的下载和解压。
prepare_forge

# 三个底模：均直接保存到 /marimo/Stable-diffusion，不保留 HF 的多层子目录。
hf_download \
    'hf://circlestone-labs/Anima/split_files/diffusion_models/anima-base-v1.0.safetensors' \
    "$CHECKPOINT_DIR"
hf_download \
    'hf://datasets/ACCC1380/private-model/upload/BSSANIRLANIMASemi_v20.safetensors' \
    "$CHECKPOINT_DIR"
hf_download \
    'hf://datasets/ACCC1380/private-model/upload/Checkpoints/miaomiaoRealskin_anima11.safetensors' \
    "$CHECKPOINT_DIR"

# 保留原来的 HF LoRA 下载，实体放在 /marimo/Lora。
hf_download \
    'hf://datasets/ACCC1380/private-model/upload/betanon_lora_000004000.safetensors' \
    "$LORA_DIR"

# 保留原来的 ModelScope LoRA 和缩略图还原。
PYTHON_SCRIPTS_DIR=$("$PYTHON_BIN" -c 'import sysconfig; print(sysconfig.get_path("scripts"))')
"$PYTHON_SCRIPTS_DIR/modelscope" download \
    --model ACCC1380/Anima_in_real_update_v2_epoch_20_20260908_083702 \
    --local_dir "$LORA_DIR"

7z t -pNeko1102202 "$LORA_DIR/lora_thumbnails_encrypted.zip"
7z x -y -pNeko1102202 "$LORA_DIR/lora_thumbnails_encrypted.zip" "-o$LORA_DIR"
rm -f -- "$LORA_DIR/lora_thumbnails_encrypted.zip"

# 确认三个底模存在，项目里的两个模型目录确实是指向项目外的软链接。
for model_name in \
    anima-base-v1.0.safetensors \
    BSSANIRLANIMASemi_v20.safetensors \
    miaomiaoRealskin_anima11.safetensors; do
    test -s "$CHECKPOINT_DIR/$model_name"
done
test -L "$WEBUI_DIR/models/Stable-diffusion"
test -L "$WEBUI_DIR/models/Lora"
test "$WEBUI_DIR/models/Stable-diffusion" -ef "$CHECKPOINT_DIR"
test "$WEBUI_DIR/models/Lora" -ef "$LORA_DIR"
df -h
ls -ld "$WEBUI_DIR/models/Stable-diffusion" "$WEBUI_DIR/models/Lora"
ls -lh "$CHECKPOINT_DIR" "$LORA_DIR"
printf '\n配置完成。项目：%s\n底模：%s\nLoRA：%s\n' "$WEBUI_DIR" "$CHECKPOINT_DIR" "$LORA_DIR"