superbpe/scripts/upload_tokenizer_to_bytenas.sh

#!/bin/bash

source_dir="tokenizer_json"
dest_dir="/mnt/bn/tiktok-mm-5/aiic/users/linzheng/artifacts"

for file in "$source_dir"/*/tokenizer.json; do
    if [ -f "$file" ]; then
        tokenizer_name=$(basename $(dirname $file))
        new_name="${dest_dir}/superbpe_${tokenizer_name}.json"
        cp "$file" "$new_name"
        echo "Copy $file to $new_name"
    fi
done