fairseq/examples/wav2vec/scripts/binarize_manifest.sh

#!/usr/bin/env bash

# usage: bash binarize_manifest <dest_dir> <train_split> <valid_split>

DEST_DIR=$1
TRAIN_SPLIT=$2
VALID_SPLIT=$3
FAIRSEQ_ROOT=$4

mkdir -p $DEST_DIR

# split file path and lengths into separate files
cut -f1 $TRAIN_SPLIT.tsv > $DEST_DIR/train_fnames.txt
cut -f1 $VALID_SPLIT.tsv > $DEST_DIR/valid_fnames.txt
cut -f2 $TRAIN_SPLIT.tsv > $DEST_DIR/train.lengths
cut -f2 $VALID_SPLIT.tsv > $DEST_DIR/valid.lengths

# copy root directory
head -1 $TRAIN_SPLIT.tsv > $DEST_DIR/train.root
head -1 $VALID_SPLIT.tsv > $DEST_DIR/valid.root

# remove root directory
sed -i '1d' $DEST_DIR/train_fnames.txt
sed -i '1d' $DEST_DIR/valid_fnames.txt
sed -i '1d' $DEST_DIR/train.lengths
sed -i '1d' $DEST_DIR/valid.lengths

# insert spaces between characters
sed -i -e 's/\(.\)/\1 /g' $DEST_DIR/train_fnames.txt
sed -i -e 's/\(.\)/\1 /g' $DEST_DIR/valid_fnames.txt

# run preprocessor
PYTHONPATH=$FAIRSEQ_ROOT python $FAIRSEQ_ROOT/fairseq_cli/preprocess.py --dataset-impl mmap --trainpref $DEST_DIR/train_fnames.txt --validpref $DEST_DIR/valid_fnames.txt --workers 60 --only-source --destdir $DEST_DIR