File size: 918 Bytes
be7e4b7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 | #!/usr/bin/env bash
# Encode a dataset.jsonl into VAE latents + text-encoder embeddings for training.
# See data/README.md for the JSONL schema.
#
# Usage:
# scripts/preprocess_dataset.sh data/dataset.jsonl "1920x1024x233"
# scripts/preprocess_dataset.sh data/dataset_image_only.jsonl "1920x1024x233"
set -euo pipefail
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
DATASET="${1:?usage: preprocess_dataset.sh <dataset.jsonl> [resolution-buckets]}"
BUCKETS="${2:-1920x1024x233}"
DATASET_ABS="$DATASET"
if [[ "$DATASET" != /* ]]; then DATASET_ABS="$REPO_ROOT/$DATASET"; fi
cd "$REPO_ROOT/packages/ltx-trainer"
python scripts/process_dataset.py "$DATASET_ABS" \
--resolution-buckets "$BUCKETS" \
--model-path "$REPO_ROOT/weights/ltx-2.3/ltx-2.3-22b-dev.safetensors" \
--text-encoder-path "$REPO_ROOT/weights/gemma-3-12b-it-qat-q4_0-unquantized" \
--output-dir "$REPO_ROOT/data/preprocessed"
|