from __future__ import annotations from pathlib import Path from typing import Any import pytest from kneiff.config import StorageConfig from kneiff.datasets.export.config import load_export_config from kneiff.datasets.export.types import ( ImageResizeConfig, TrainingExportResult, ) from kneiff.datasets.export.writer import ( export_training_dataset as _export_training_dataset, ) from kneiff.project import ProjectContextError from tests.captions._helpers import load_rook_vocabulary from tests.dataset._export_helpers import ( _manifest_row, _write_config, _write_image, _write_manifest, ) def export_training_dataset(*args: Any, **kwargs: Any) -> TrainingExportResult: """Run config-loading export checks with the Rook fixture vocabulary.""" kwargs.setdefault("vocabulary", load_rook_vocabulary()) return _export_training_dataset(*args, **kwargs) def test_export_config_workers_ignore_process_and_project_dotenv( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: source_root = tmp_path / "SOURCE" export_root = tmp_path / "HF" / "ready" source_root.mkdir() (source_root / ".env.shared").write_text("KNF_WORKERS=4\n", encoding="utf-8") monkeypatch.setenv("KNF_WORKERS", "3") config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) _write_config(config_path, source_root, export_root) config = load_export_config(config_path) assert config.workers == 8 def test_export_workers_allow_explicit_orchestration_override( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: source_root = tmp_path / "SOURCE" export_root = tmp_path / "HF" / "ready" source_root.mkdir() monkeypatch.setenv("KNF_WORKERS", "2") config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) _write_config(config_path, source_root, export_root) config = load_export_config(config_path, workers_override=4) assert config.workers == 4 def test_export_workers_reject_invalid_override(tmp_path: Path) -> None: source_root = tmp_path / "SOURCE" export_root = tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) _write_config(config_path, source_root, export_root) with pytest.raises(ValueError, match="KNF_WORKERS"): load_export_config(config_path, workers_override=0) @pytest.mark.parametrize( "config_path", [ Path("ready.knf.yaml"), Path("configs/ready.yaml"), Path("configs/nested/ready.knf.yaml"), Path("configs/.knf.yaml"), Path("configs/..knf.yaml"), Path("configs/ ready.knf.yaml"), ], ) def test_export_config_requires_direct_project_config_path( tmp_path: Path, config_path: Path, ) -> None: invalid_path = tmp_path / config_path invalid_path.parent.mkdir(parents=True, exist_ok=True) invalid_path.write_text("mappings: {fullbody: [0-FULLBODY]}\n", encoding="utf-8") with pytest.raises(ValueError, match="config|Config"): load_export_config(invalid_path) def test_storage_config_owns_project_worker_setting( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setenv("KNF_STORAGE", str(tmp_path)) monkeypatch.setenv("KNF_WORKERS", "4") config = StorageConfig() assert config.root == tmp_path assert config.workers == 4 def test_storage_config_rejects_non_positive_worker_setting( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setenv("KNF_STORAGE", str(tmp_path)) monkeypatch.setenv("KNF_WORKERS", "0") with pytest.raises(ValueError, match="KNF_WORKERS"): StorageConfig() def test_export_config_defaults_mirrored_extra_to_false(tmp_path: Path) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" """, encoding="utf-8", ) config = load_export_config(config_path) assert config.augmentations.mirrored_extra is False assert config.augmentations.mirrored_transform == "augmented" assert config.augmentations.seed == 12345 def test_export_config_defaults_missing_mirrored_extra_to_false( tmp_path: Path, ) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" augmentations: seed: 12345 """, encoding="utf-8", ) config = load_export_config(config_path) assert config.augmentations.mirrored_extra is False assert config.augmentations.mirrored_transform == "augmented" def test_export_config_loads_flip_only_mirrored_transform(tmp_path: Path) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" augmentations: mirrored_extra: true mirrored_transform: flip_only seed: 12345 """, encoding="utf-8", ) config = load_export_config(config_path) assert config.augmentations.mirrored_extra is True assert config.augmentations.mirrored_transform == "flip_only" def test_export_config_rejects_string_mirrored_extra(tmp_path: Path) -> None: source_root = tmp_path / "SOURCE" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" augmentations: mirrored_extra: "false" """, encoding="utf-8", ) with pytest.raises( ValueError, match="augmentations.mirrored_extra must be true or false" ): load_export_config(config_path) def test_export_config_rejects_coerced_augmentation_seed(tmp_path: Path) -> None: source_root = tmp_path / "SOURCE" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" augmentations: seed: "12345" """, encoding="utf-8", ) with pytest.raises(ValueError, match="augmentations.seed must be an integer"): load_export_config(config_path) @pytest.mark.parametrize( ("extra_config", "error"), [ ("mappingz: {}", "Dataset export config contains unknown fields: mappingz"), ("augmentations:\n sed: 1", "augmentations contains unknown fields: sed"), ( "caption:\n add_domain_to_tags: true", "caption contains unknown fields: add_domain_to_tags", ), ( "image_resize:\n min_pixel_area: 512\n max_pixels: 1024", "image_resize contains unknown fields: max_pixels", ), ( "publishing:\n hugginface: {}", "publishing contains unknown fields: hugginface", ), ( "publishing:\n huggingface:\n reop_id: account/data", "publishing.huggingface contains unknown fields: reop_id", ), ( "caption_outputs_overrides:\n fullbody:\n formtas: [tag]", "caption_outputs_overrides.fullbody contains unknown fields: formtas", ), ], ) def test_export_config_rejects_unknown_fields( tmp_path: Path, extra_config: str, error: str, ) -> None: source_root = tmp_path / "SOURCE" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( "mappings:\n fullbody:\n - 0-FULLBODY\n" + extra_config + "\n", encoding="utf-8", ) with pytest.raises(ValueError, match=error): load_export_config(config_path) def test_export_config_rejects_invalid_mirrored_transform(tmp_path: Path) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" augmentations: mirrored_transform: sideways """, encoding="utf-8", ) with pytest.raises(ValueError, match="augmentations.mirrored_transform"): load_export_config(config_path) def test_export_config_derives_fixed_project_paths(tmp_path: Path) -> None: source_root = tmp_path / "SOURCE" export_root = tmp_path / "HF" / "ready" _write_image(source_root / "0-FULLBODY" / "scene.png") _write_manifest( source_root, [_manifest_row("0-FULLBODY/scene.png")], workbook_name="MANIFEST.knf.xlsx", ) config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" augmentations: mirrored_extra: false seed: 12345 """, encoding="utf-8", ) config = load_export_config(config_path) result = export_training_dataset(config, dry_run=True) assert config.source_root == source_root assert config.manifest_path == tmp_path / "MANIFEST.knf.xlsx" assert config.export_root == export_root assert result.source_image_count == 1 def test_export_config_does_not_fallback_to_lowercase_knf_manifest( tmp_path: Path, ) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" _write_image(source_root / "0-FULLBODY" / "scene.png") _write_manifest( source_root, [_manifest_row("0-FULLBODY/scene.png")], workbook_name="MANIFEST.knf.xlsx", ) (tmp_path / "MANIFEST.knf.xlsx").rename(tmp_path / "manifest.knf.xlsx") config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" augmentations: mirrored_extra: false seed: 12345 """, encoding="utf-8", ) config = load_export_config(config_path) assert config.manifest_path == tmp_path / "MANIFEST.knf.xlsx" with pytest.raises(ProjectContextError) as exc_info: export_training_dataset(config, dry_run=True) assert "MANIFEST.knf.xlsx" in str(exc_info.value) def test_export_config_rejects_legacy_densities_key(tmp_path: Path) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" densities: [short] """, encoding="utf-8", ) with pytest.raises(ValueError, match="densities is no longer supported"): load_export_config(config_path) def test_export_config_defaults_sfw_subset_export_to_false(tmp_path: Path) -> None: source_root = tmp_path / "SOURCE" export_root = tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) _write_config(config_path, source_root, export_root) config = load_export_config(config_path) assert config.export_sfw_subset is False def test_export_config_defaults_image_resize_to_none(tmp_path: Path) -> None: source_root = tmp_path / "SOURCE" export_root = tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) _write_config(config_path, source_root, export_root) config = load_export_config(config_path) assert config.image_resize is None def test_export_config_loads_image_resize_square_edges(tmp_path: Path) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" image_resize: min_pixel_area: 768 max_pixel_area: 1536 """, encoding="utf-8", ) config = load_export_config(config_path) assert config.image_resize == ImageResizeConfig( min_pixel_area=768, max_pixel_area=1536, ) def test_export_config_rejects_invalid_image_resize_type(tmp_path: Path) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" image_resize: yes please """, encoding="utf-8", ) with pytest.raises(ValueError, match="image_resize"): load_export_config(config_path) def test_export_config_rejects_empty_image_resize(tmp_path: Path) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" image_resize: {} """, encoding="utf-8", ) with pytest.raises(ValueError, match="min_pixel_area or max_pixel_area"): load_export_config(config_path) def test_export_config_rejects_non_positive_image_resize_value( tmp_path: Path, ) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" image_resize: max_pixel_area: 0 """, encoding="utf-8", ) with pytest.raises(ValueError, match="image_resize.max_pixel_area"): load_export_config(config_path) def test_export_config_rejects_reversed_image_resize_bounds( tmp_path: Path, ) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" image_resize: min_pixel_area: 1536 max_pixel_area: 768 """, encoding="utf-8", ) with pytest.raises(ValueError, match="min_pixel_area"): load_export_config(config_path) def test_export_config_rejects_invalid_sfw_subset_toggle(tmp_path: Path) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" export_sfw_subset: yes please """, encoding="utf-8", ) with pytest.raises(ValueError, match="export_sfw_subset"): load_export_config(config_path) def test_export_config_rejects_generated_sfw_subset_name_clash( tmp_path: Path, ) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: sfw: - "0-FULLBODY" export_sfw_subset: true """, encoding="utf-8", ) with pytest.raises(ValueError, match="mappings already defines an sfw subset"): load_export_config(config_path) def test_export_config_rejects_old_source_to_subset_mappings(tmp_path: Path) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: "0-FULLBODY": "fullbody" """, encoding="utf-8", ) with pytest.raises(ValueError, match="subset names as keys"): load_export_config(config_path) def test_export_config_rejects_invalid_subject_sex(tmp_path: Path) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" caption: subject_sex: unknown """, encoding="utf-8", ) with pytest.raises(ValueError, match="caption.subject_sex"): load_export_config(config_path) def test_export_config_rejects_removed_caption_domain_toggle(tmp_path: Path) -> None: source_root = tmp_path / "SOURCE" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" caption: add_domains_to_tags: true """, encoding="utf-8", ) with pytest.raises(ValueError, match="caption contains unknown fields"): load_export_config(config_path) @pytest.mark.parametrize("format_name", ["tag", "prose"]) def test_export_config_rejects_removed_caption_format_aliases( tmp_path: Path, format_name: str, ) -> None: source_root = tmp_path / "SOURCE" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( f""" mappings: fullbody: - "0-FULLBODY" caption_outputs: formats: [{format_name}] """, encoding="utf-8", ) with pytest.raises(ValueError, match="must contain only"): load_export_config(config_path) def test_export_config_rejects_unknown_caption_output_subset(tmp_path: Path) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" caption_outputs_overrides: missing: mode: hybrid_txt formats: [tags] """, encoding="utf-8", ) with pytest.raises( ValueError, match="caption_outputs_overrides references unknown subset: missing", ): load_export_config(config_path) def test_export_config_rejects_invalid_caption_output_mode(tmp_path: Path) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" caption_outputs: mode: yaml_txt formats: [tags] """, encoding="utf-8", ) with pytest.raises(ValueError, match="caption_outputs.mode"): load_export_config(config_path) def test_export_config_rejects_invalid_caption_output_format(tmp_path: Path) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" caption_outputs: mode: hybrid_txt formats: [tags, yaml] """, encoding="utf-8", ) with pytest.raises(ValueError, match="caption_outputs.formats"): load_export_config(config_path) def test_export_config_rejects_chroma_mixed_in_hybrid_caption( tmp_path: Path, ) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" caption_outputs: mode: hybrid_txt formats: [tags, chroma] """, encoding="utf-8", ) with pytest.raises(ValueError, match="cannot combine chroma"): load_export_config(config_path) def test_export_config_loads_global_caption_output_default(tmp_path: Path) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" caption_outputs: mode: hybrid_txt formats: [chroma] tag_scope: all """, encoding="utf-8", ) config = load_export_config(config_path) assert config.caption_output_default.mode == "hybrid_txt" assert config.caption_output_default.formats == ("chroma",) assert config.caption_output_default.tag_scope == "all" assert config.caption_outputs == {} @pytest.mark.parametrize( "format_name", ["flux2", "z-image", "Z-Image", "z_image", "zimage"], ) def test_export_config_rejects_removed_nlg_caption_aliases( tmp_path: Path, format_name: str, ) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( f""" mappings: fullbody: - "0-FULLBODY" caption_outputs: formats: [{format_name}] """, encoding="utf-8", ) with pytest.raises(ValueError, match="must contain only"): load_export_config(config_path) def test_export_config_caption_output_override_inherits_global_formats( tmp_path: Path, ) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" caption_outputs: mode: hybrid_txt formats: [chroma] tag_scope: all caption_outputs_overrides: fullbody: mode: separate_txt """, encoding="utf-8", ) config = load_export_config(config_path) assert config.caption_outputs["fullbody"].mode == "separate_txt" assert config.caption_outputs["fullbody"].formats == ("chroma",) assert config.caption_outputs["fullbody"].tag_scope == "all" def test_export_config_caption_output_override_inherits_global_mode( tmp_path: Path, ) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" caption_outputs: mode: separate_txt formats: [tags, natural] tag_scope: all caption_outputs_overrides: fullbody: formats: [json] """, encoding="utf-8", ) config = load_export_config(config_path) assert config.caption_outputs["fullbody"].mode == "separate_txt" assert config.caption_outputs["fullbody"].formats == ("json",) assert config.caption_outputs["fullbody"].tag_scope == "all" def test_export_config_caption_output_override_can_change_tag_scope( tmp_path: Path, ) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" caption_outputs: mode: hybrid_txt formats: [chroma] caption_outputs_overrides: fullbody: tag_scope: all """, encoding="utf-8", ) config = load_export_config(config_path) assert config.caption_outputs["fullbody"].mode == "hybrid_txt" assert config.caption_outputs["fullbody"].formats == ("chroma",) assert config.caption_outputs["fullbody"].tag_scope == "all" def test_export_config_rejects_invalid_global_caption_tag_scope( tmp_path: Path, ) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" caption_outputs: mode: hybrid_txt formats: [chroma] tag_scope: noisy """, encoding="utf-8", ) with pytest.raises(ValueError, match="caption_outputs.tag_scope"): load_export_config(config_path) def test_export_config_rejects_invalid_override_caption_tag_scope( tmp_path: Path, ) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" caption_outputs_overrides: fullbody: tag_scope: noisy """, encoding="utf-8", ) with pytest.raises( ValueError, match="caption_outputs_overrides.fullbody.tag_scope", ): load_export_config(config_path) def test_export_config_rejects_unknown_caption_output_override_subset( tmp_path: Path, ) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" caption_outputs: mode: hybrid_txt formats: [tags] caption_outputs_overrides: missing: formats: [natural] """, encoding="utf-8", ) with pytest.raises(ValueError, match="caption_outputs_overrides.*missing"): load_export_config(config_path) def test_export_config_rejects_non_string_caption_output_override_subset( tmp_path: Path, ) -> None: config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True) config_path.write_text( """ mappings: fullbody: [0-FULLBODY] caption_outputs_overrides: 1: formats: [tags] """, encoding="utf-8", ) with pytest.raises(ValueError, match="non-empty exact string"): load_export_config(config_path) def test_export_config_accepts_sfw_caption_override_when_generated_subset_enabled( tmp_path: Path, ) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" export_sfw_subset: true caption_outputs_overrides: sfw: formats: [json] """, encoding="utf-8", ) config = load_export_config(config_path) assert config.caption_outputs["sfw"].formats == ("json",) def test_export_config_rejects_sfw_caption_override_without_sfw_subset( tmp_path: Path, ) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" caption_outputs_overrides: sfw: formats: [json] """, encoding="utf-8", ) with pytest.raises(ValueError, match="caption_outputs_overrides.*sfw"): load_export_config(config_path) def test_export_config_rejects_invalid_global_caption_output_mode( tmp_path: Path, ) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" caption_outputs: mode: yaml_txt formats: [tags] """, encoding="utf-8", ) with pytest.raises(ValueError, match="caption_outputs.mode"): load_export_config(config_path) def test_export_config_rejects_invalid_override_caption_output_format( tmp_path: Path, ) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" caption_outputs: mode: hybrid_txt formats: [tags] caption_outputs_overrides: fullbody: formats: [yaml] """, encoding="utf-8", ) with pytest.raises( ValueError, match="caption_outputs_overrides.fullbody.formats", ): load_export_config(config_path) def test_export_config_rejects_subset_entries_in_global_caption_outputs( tmp_path: Path, ) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" caption_outputs: mode: hybrid_txt formats: [chroma] fullbody: formats: [tags] """, encoding="utf-8", ) with pytest.raises(ValueError, match="caption_outputs_overrides"): load_export_config(config_path) def test_export_config_rejects_chroma_mixed_in_global_hybrid_caption( tmp_path: Path, ) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" caption_outputs: mode: hybrid_txt formats: [tags, chroma] """, encoding="utf-8", ) with pytest.raises(ValueError, match="caption_outputs.formats"): load_export_config(config_path) def test_export_config_rejects_flux2_mixed_in_global_hybrid_caption( tmp_path: Path, ) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" caption_outputs: mode: hybrid_txt formats: [tags, nlg] """, encoding="utf-8", ) with pytest.raises(ValueError, match="caption_outputs.formats"): load_export_config(config_path) def test_export_config_rejects_chroma_mixed_in_override_hybrid_caption( tmp_path: Path, ) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" caption_outputs: mode: separate_txt formats: [tags, chroma] caption_outputs_overrides: fullbody: mode: hybrid_txt """, encoding="utf-8", ) with pytest.raises( ValueError, match="caption_outputs_overrides.fullbody.formats", ): load_export_config(config_path) def test_export_config_accepts_flux2_with_other_separate_outputs( tmp_path: Path, ) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" caption_outputs: mode: separate_txt formats: [tags, natural, json, chroma, nlg] """, encoding="utf-8", ) config = load_export_config(config_path) assert config.caption_output_default.mode == "separate_txt" assert config.caption_output_default.formats == ( "tags", "natural", "json", "chroma", "nlg", ) def test_export_config_rejects_nested_subset(tmp_path: Path) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" _write_image(source_root / "0-FULLBODY" / "scene.png") _write_manifest(source_root, [_manifest_row("0-FULLBODY/scene.png")]) config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: "nested/fullbody": - "0-FULLBODY" """, encoding="utf-8", ) with pytest.raises(ValueError, match="mapping subset"): load_export_config(config_path) @pytest.mark.parametrize("subset_key", ["1", "' fullbody '"]) def test_export_config_rejects_coerced_mapping_subset_names( tmp_path: Path, subset_key: str, ) -> None: config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True) config_path.write_text( f"mappings:\n {subset_key}: [0-FULLBODY]\n", encoding="utf-8", ) with pytest.raises(ValueError, match="non-empty exact string"): load_export_config(config_path) def test_export_config_rejects_duplicate_normalized_mapping_sources( tmp_path: Path, ) -> None: config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True) config_path.write_text( """ mappings: fullbody: - identity\\closeups - identity/closeups """, encoding="utf-8", ) with pytest.raises(ValueError, match="repeats source directory"): load_export_config(config_path) @pytest.mark.parametrize( "source_dir", [ "/absolute/source", r"C:\absolute\source", r"\\server\share\source", "../outside", "nested/../outside", "./nested", "nested/./child", ], ) def test_export_config_rejects_unsafe_mapping_source_directories( tmp_path: Path, source_dir: str, ) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( f""" mappings: fullbody: - '{source_dir}' """, encoding="utf-8", ) with pytest.raises(ValueError, match="relative path|path components"): load_export_config(config_path) def test_export_config_normalizes_nested_windows_mapping_source_directory( tmp_path: Path, ) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - 'identity\\closeups' """, encoding="utf-8", ) config = load_export_config(config_path) assert config.mappings == {"fullbody": ("identity/closeups",)} def test_export_config_rejects_backslash_nested_subset(tmp_path: Path) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" _write_image(source_root / "0-FULLBODY" / "scene.png") _write_manifest(source_root, [_manifest_row("0-FULLBODY/scene.png")]) config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: 'nested\\fullbody': - "0-FULLBODY" """, encoding="utf-8", ) with pytest.raises(ValueError, match="mapping subset"): load_export_config(config_path) @pytest.mark.parametrize("subset_name", ["_TRAINING", "training"]) def test_export_config_accepts_training_named_subset( tmp_path: Path, subset_name: str, ) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" _write_image(source_root / "0-FULLBODY" / "scene.png") _write_manifest(source_root, [_manifest_row("0-FULLBODY/scene.png")]) config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( f""" mappings: {subset_name}: - "0-FULLBODY" """, encoding="utf-8", ) config = load_export_config(config_path) assert config.mappings == {subset_name: ("0-FULLBODY",)} def test_export_config_loads_huggingface_publish_settings(tmp_path: Path) -> None: source_root = tmp_path / "SOURCE" tmp_path / "HF" / "ready" source_root.mkdir() config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( """ mappings: fullbody: - "0-FULLBODY" publishing: huggingface: enabled: false repo_id: example/demo-dataset pretty_name: Demo Dataset version: v5.1 optimized_for_model: Chroma1-HD license: creativeml-openrail-m tags: [chroma, lora] provenance: Local source set. adult_content: true notes: Needs review. """, encoding="utf-8", ) config = load_export_config(config_path) assert config.huggingface.enabled is False assert config.huggingface.repo_id == "example/demo-dataset" assert config.huggingface.pretty_name == "Demo Dataset" assert config.huggingface.version == "v5.1" assert config.huggingface.optimized_for_model == "Chroma1-HD" assert config.huggingface.license == "creativeml-openrail-m" assert config.huggingface.tags == ("chroma", "lora") assert config.huggingface.provenance == "Local source set." assert config.huggingface.adult_content is True assert config.huggingface.notes == "Needs review." @pytest.mark.parametrize( ("key", "value"), [ ("source_root", "somewhere"), ("manifest_path", "somewhere.xlsx"), ("export_root", "somewhere"), ("allow_export_inside_source", "true"), ], ) def test_export_config_rejects_project_owned_path_settings( tmp_path: Path, key: str, value: str, ) -> None: config_path = tmp_path / "configs" / "ready.knf.yaml" config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text( f""" {key}: {value} mappings: fullbody: - "0-FULLBODY" """, encoding="utf-8", ) with pytest.raises(ValueError, match=f"remove these key\\(s\\): {key}"): load_export_config(config_path)