| #!/bin/sh |
|
|
| wget https://lindat.mff.cuni.cz/repository/xmlui/bitstream/handle/11234/1-4611/ud-treebanks-v2.9.tgz |
| tar xf ud-treebanks-v2.9.tgz |
| rm ud-treebanks-v2.9.tgz |
| for tb in ud-treebanks-v2.9/*/*-ud-train.conllu; do |
| dir=`dirname $tb` |
| long_name=`basename $dir` |
| code=`basename $tb` |
| code=${code%%-*} |
|
|
| case $long_name in |
| UD_Arabic-NYUAD|UD_English-ESL|UD_English-GUMReddit|UD_French-FTB|UD_Hindi_English-HIENCS|UD_Japanese-BCCWJ|UD_Japanese-BCCWJLUW) echo Skipping treebank $long_name without forms; continue;; |
| esac |
|
|
| mkdir -p "$code" |
| cp "$dir"/* "$code" |
| if [ -f "$dir"/"$code"-ud-dev.conllu ]; then |
| cp "$dir"/"$code"-ud-train.conllu "$dir"/"$code"-ud-dev.conllu "$code" |
| else |
| perl conllu_split.pl "$code" "$code" <"$dir"/"$code"-ud-train.conllu |
| fi |
|
|
| train_words=$(grep -c '^[0-9]' $code/*-ud-train.conllu) |
| [ "$train_words" -lt 1000 ] && { echo Skipping treebank $code with $train_words training words. >&2; rm -r "$code/"; continue; } |
| echo $code $long_name | tee -a iso_names.txt |
|
|
| for conllu in "$code"/*.conllu; do |
| sed "1i# variant = $code" -i "$conllu" |
| case "$conllu" in |
| *train.conllu) rm "${conllu%.conllu}.txt";; |
| *) perl conllu_to_text.pl --language="$code" <"$conllu" >"${conllu%.conllu}.txt";; |
| esac |
| done |
| done |
| rm -r ud-treebanks-v2.9 |
|
|
| for code in $(for d in */; do echo ${d%%[_/]*}; done | sort | uniq -d); do |
| mkdir "${code}_all" |
| cat "$code"*/*train.conllu >"${code}_all/${code}_all-ud-train.conllu" |
| done |
|
|
