#!/bin/bash

echo "Shuffling and cleaning the West Syriac corpus..."
shuf syriac_west_corpus.jsonl -o syriac_west_corpus_shuffled.jsonl
mv syriac_west_corpus_shuffled.jsonl syriac_west_corpus.jsonl
grep '\S' syriac_west_corpus.jsonl > syriac_west_clean_corpus.jsonl
sed -i 's/\xe2\x80\x8f//g' syriac_west_clean_corpus.jsonl
rm syriac_west_corpus.jsonl

echo "Shuffling and cleaning the East Syriac corpus..."
shuf syriac_east_corpus.jsonl -o syriac_east_corpus_shuffled.jsonl
mv syriac_east_corpus_shuffled.jsonl syriac_east_corpus.jsonl
grep '\S' syriac_east_corpus.jsonl > syriac_east_clean_corpus.jsonl
sed -i 's/\xe2\x80\x8f//g' syriac_east_clean_corpus.jsonl
rm syriac_east_corpus.jsonl

echo ""
echo "Generated corpora:"
echo "  - syriac_west_clean_corpus.jsonl (West Syriac / Serto)"
echo "  - syriac_east_clean_corpus.jsonl (East Syriac / Madnḥaya)"
echo ""
echo "Next step: Run train_tokeniser.py to train the tokeniser on the clean corpus."