| [ |
| { |
| "id": "moss-sft-anyflow-cc12m-long-512", |
| "type": "webshart", |
| "dataset_type": "image", |
| "source": "laion/conceptual-captions-12m-webdataset", |
| "subfolder": "data", |
| "metadata": "webshart/cc12m-structured-captions", |
| "caption_strategy": "webshart", |
| "metadata_backend": "webshart", |
| "crop": true, |
| "crop_aspect": "square", |
| "crop_style": "random", |
| "resolution": 512, |
| "resolution_type": "pixel_area", |
| "minimum_image_size": 256, |
| "maximum_image_size": 512, |
| "target_downsample_size": 512, |
| "cache_dir_vae": "cache/vae/moss-sft-anyflow-cc12m-long-512", |
| "webshart": { |
| "buffer_size": 64, |
| "parallel_downloads": 8, |
| "shard_cache_gb": 50, |
| "max_file_size": 536870912, |
| "cache_dir": "cache/webshart/moss-sft-anyflow-cc12m", |
| "metadata_cache_dir": "cache/webshart/moss-sft-anyflow-cc12m/metadata", |
| "shard_cache_dir": "cache/webshart/moss-sft-anyflow-cc12m/shards", |
| "caption_key": "long_caption", |
| "optimize_captions": false |
| }, |
| "max_num_samples": 65536, |
| "text_embeds": "text", |
| "repeats": 1, |
| "probability": 0.25, |
| "disable_multiline_split": true |
| }, |
| { |
| "id": "moss-sft-anyflow-e621-512", |
| "type": "webshart", |
| "dataset_type": "image", |
| "source": "NebulaeWis/e621-2024-webp-4Mpixel", |
| "subfolder": "original", |
| "metadata": "RareConcepts/e621-2024-webp-4Mpixel-webshart-indices", |
| "caption_strategy": "webshart", |
| "metadata_backend": "webshart", |
| "crop": true, |
| "crop_aspect": "square", |
| "crop_style": "random", |
| "resolution": 512, |
| "resolution_type": "pixel_area", |
| "minimum_image_size": 256, |
| "maximum_image_size": 512, |
| "target_downsample_size": 512, |
| "cache_dir_vae": "cache/vae/moss-sft-anyflow-512", |
| "webshart": { |
| "buffer_size": 64, |
| "parallel_downloads": 8, |
| "shard_cache_gb": 50, |
| "max_file_size": 536870912, |
| "cache_dir": "cache/webshart/moss-sft-anyflow", |
| "metadata_cache_dir": "cache/webshart/moss-sft-anyflow/metadata", |
| "shard_cache_dir": "cache/webshart/moss-sft-anyflow/shards" |
| }, |
| "max_num_samples": 65536, |
| "text_embeds": "text", |
| "repeats": 1, |
| "probability": 0.25, |
| "disable_multiline_split": true |
| }, |
| { |
| "id": "moss-sft-anyflow-cc12m-long-1024-uncropped", |
| "type": "webshart", |
| "dataset_type": "image", |
| "source": "laion/conceptual-captions-12m-webdataset", |
| "subfolder": "data", |
| "metadata": "webshart/cc12m-structured-captions", |
| "caption_strategy": "webshart", |
| "metadata_backend": "webshart", |
| "crop": false, |
| "resolution": 1024, |
| "resolution_type": "pixel_area", |
| "minimum_image_size": 1024, |
| "maximum_image_size": 1024, |
| "target_downsample_size": 1024, |
| "cache_dir_vae": "cache/vae/moss-sft-anyflow-cc12m-long-1024-uncropped", |
| "webshart": { |
| "buffer_size": 64, |
| "parallel_downloads": 8, |
| "shard_cache_gb": 50, |
| "max_file_size": 536870912, |
| "cache_dir": "cache/webshart/moss-sft-anyflow-cc12m", |
| "metadata_cache_dir": "cache/webshart/moss-sft-anyflow-cc12m/metadata", |
| "shard_cache_dir": "cache/webshart/moss-sft-anyflow-cc12m/shards", |
| "caption_key": "long_caption", |
| "optimize_captions": false |
| }, |
| "max_num_samples": 65536, |
| "text_embeds": "text", |
| "repeats": 0, |
| "probability": 0.25, |
| "disable_multiline_split": true |
| }, |
| { |
| "id": "moss-sft-anyflow-e621-1024-uncropped", |
| "type": "webshart", |
| "dataset_type": "image", |
| "source": "NebulaeWis/e621-2024-webp-4Mpixel", |
| "subfolder": "original", |
| "metadata": "RareConcepts/e621-2024-webp-4Mpixel-webshart-indices", |
| "caption_strategy": "webshart", |
| "metadata_backend": "webshart", |
| "crop": false, |
| "resolution": 1024, |
| "resolution_type": "pixel_area", |
| "minimum_image_size": 1024, |
| "maximum_image_size": 1024, |
| "target_downsample_size": 1024, |
| "cache_dir_vae": "cache/vae/moss-sft-anyflow-1024-uncropped", |
| "webshart": { |
| "buffer_size": 64, |
| "parallel_downloads": 8, |
| "shard_cache_gb": 50, |
| "max_file_size": 536870912, |
| "cache_dir": "cache/webshart/moss-sft-anyflow", |
| "metadata_cache_dir": "cache/webshart/moss-sft-anyflow/metadata", |
| "shard_cache_dir": "cache/webshart/moss-sft-anyflow/shards" |
| }, |
| "max_num_samples": 65536, |
| "text_embeds": "text", |
| "repeats": 0, |
| "probability": 0.25, |
| "disable_multiline_split": true |
| }, |
| { |
| "id": "text", |
| "type": "local", |
| "dataset_type": "text_embeds", |
| "default": true, |
| "cache_dir": "cache/text/moss-sft-anyflow" |
| } |
| ] |
|
|
