File size: 3,436 Bytes
76b7b25 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 | from __future__ import annotations
import json
import os
import time
from pathlib import Path
from typing import Any
import yaml
PROXY_ENV_VARS = (
"http_proxy",
"https_proxy",
"HTTP_PROXY",
"HTTPS_PROXY",
"all_proxy",
"ALL_PROXY",
)
def load_config(path: str | Path) -> dict[str, Any]:
config_path = Path(path).expanduser()
with config_path.open(encoding="utf-8") as f:
config = yaml.safe_load(f)
if not isinstance(config, dict):
raise RuntimeError(f"Config must be a mapping: {config_path}")
config["_config_path"] = str(config_path)
config["_config_dir"] = str(config_path.parent)
return config
def as_path(value: str | Path, base_dir: str | Path | None = None) -> Path:
path = Path(value).expanduser()
if not path.is_absolute() and base_dir is not None:
path = Path(base_dir) / path
return path
def cfg_path(config: dict[str, Any], key: str, *, required: bool = True) -> Path | None:
value = config.get("paths", {}).get(key)
if value is None:
if required:
raise KeyError(f"Missing paths.{key}")
return None
base = config.get("_config_dir")
return as_path(value, base)
def unset_proxy_env(env: dict[str, str] | None = None) -> dict[str, str]:
copied = dict(os.environ if env is None else env)
for key in PROXY_ENV_VARS:
copied.pop(key, None)
return copied
def model_by_name(config: dict[str, Any]) -> dict[str, dict[str, Any]]:
models = config.get("models", [])
if not isinstance(models, list):
raise RuntimeError("Config field 'models' must be a list")
return {str(model["name"]): model for model in models}
def select_models(
config: dict[str, Any],
only: str | None = None,
roles: str | None = None,
) -> list[dict[str, Any]]:
models = list(config.get("models", []))
if only:
names = [item.strip() for item in only.split(",") if item.strip()]
by_name = model_by_name(config)
missing = [name for name in names if name not in by_name]
if missing:
raise RuntimeError(f"Unknown model names: {missing}")
models = [by_name[name] for name in names]
if roles:
wanted = {item.strip() for item in roles.split(",") if item.strip()}
models = [model for model in models if str(model.get("role", "")) in wanted]
return models
def utc_now() -> str:
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
def write_json(path: str | Path, payload: Any) -> None:
out = Path(path)
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
def count_generated_media(output_dir: Path) -> dict[str, int]:
return {
"mp4": len(list(output_dir.glob("sample_*.mp4"))) if output_dir.exists() else 0,
"wav": len(list(output_dir.glob("sample_*.wav"))) if output_dir.exists() else 0,
"json": len(list(output_dir.glob("sample_*.json"))) if output_dir.exists() else 0,
}
def media_dir_for_model(config: dict[str, Any], model: dict[str, Any]) -> Path:
if model.get("media_dir"):
return as_path(model["media_dir"], config.get("_config_dir"))
return cfg_path(config, "diversity_root") / str(model["name"])
def model_output_dir(config: dict[str, Any], model: dict[str, Any]) -> Path:
return media_dir_for_model(config, model)
|