scripts/preprocess_dataset.sh

#!/usr/bin/env bash
# Encode a dataset.jsonl into VAE latents + text-encoder embeddings for training.
# See data/README.md for the JSONL schema.
#
# Usage:
#   scripts/preprocess_dataset.sh data/dataset.jsonl "1920x1024x233"
#   scripts/preprocess_dataset.sh data/dataset_image_only.jsonl "1920x1024x233"
set -euo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"

DATASET="${1:?usage: preprocess_dataset.sh <dataset.jsonl> [resolution-buckets]}"
BUCKETS="${2:-1920x1024x233}"

DATASET_ABS="$DATASET"
if [[ "$DATASET" != /* ]]; then DATASET_ABS="$REPO_ROOT/$DATASET"; fi

cd "$REPO_ROOT/packages/ltx-trainer"
python scripts/process_dataset.py "$DATASET_ABS" \
  --resolution-buckets "$BUCKETS" \
  --model-path "$REPO_ROOT/weights/ltx-2.3/ltx-2.3-22b-dev.safetensors" \
  --text-encoder-path "$REPO_ROOT/weights/gemma-3-12b-it-qat-q4_0-unquantized" \
  --output-dir "$REPO_ROOT/data/preprocessed"