File size: 5,326 Bytes
613ce86 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 | #!/usr/bin/python3
from colorama import Fore, Style
import json
import subprocess
import argparse
import pathlib
import csv
import sys
import os
from subprocess import PIPE
def clear_frase_fonetica(input_string: str) -> str:
input_string = input_string.lower() # Transform input to lowercase
symbols_to_remove = ["-", "^"]
for symbol in symbols_to_remove:
input_string = input_string.replace(symbol, "")
return input_string
def aggregate_json_field(json, json_field):
concatenated_values = ""
first_value = True
for item in json:
if not first_value:
concatenated_values += " \n"
concatenated_values += item.get(json_field, "").strip()
first_value = False
return concatenated_values
# Default filenames
eval_file = 'eval_gal_voice.csv'
exec_file = 'cotovia'
# Global run stats
passed = 0
failed = 0
# Colors
GREEN = Fore.GREEN
YELLOW = Fore.YELLOW
RED = Fore.RED
RESET = Style.RESET_ALL
# Get default paths
p = pathlib.PurePath(os.path.abspath(__file__))
data_path = p.parents[1].joinpath('data').joinpath(eval_file)
exec_path = p.parents[2].joinpath('build/cotovia').joinpath(exec_file)
# Command line arguments
parser = argparse.ArgumentParser(
description='Evaluate cotovia transcription against annotated files')
parser.add_argument('--exec', help='Override cotovia executable path')
parser.add_argument('--eval', help='Override cotovia evaluation file path')
parser.add_argument('--pmode', type=int, choices=[0, 1], default=0,
help='Select processing mode (default 0): 0 => Voice Synthesis, 1 => Speech Recognition')
parser.add_argument('--delimiter', type=str, default=',',
help='CSV eval files default delimiter')
parser.add_argument('--line', default=False,
action='store_true', help='Enables treating each line as an expression, ignoring other criteria.')
parser.add_argument('--compact', default=False,
action='store_true', help='Report only failed tests')
parser.add_argument('--json_field', type=str, help='Json field of the cotovia output to match', default="frase_procesada")
args = parser.parse_args()
if args.exec:
exec_path = pathlib.PurePath(os.path.abspath(args.exec))
if args.eval:
data_path = pathlib.PurePath(os.path.abspath(args.eval))
process_mode: int = args.pmode
json_field = args.json_field
print("=====================================================")
print(f"Eval data path: {data_path}")
print(f"Exec path: {exec_path}")
if process_mode == 0:
print("Process mode: Voice Synthesis")
elif process_mode == 1:
print("Process mode: Speech Recognition")
paths = pathlib.Path(data_path.parent).glob(data_path.name)
for path in paths:
with open(path, newline='') as eval_data:
print("////////////////////////START////////////////////////")
print(f"Evaluating file: {path}")
input_reader = csv.reader(eval_data, delimiter=args.delimiter)
for entry in input_reader:
# data to send to cotovia
cotovia_input = entry[0].strip()
expected_output = entry[1].strip()
# prepare process
command = [str(exec_path), f"-p{process_mode}", "-S", "-L", "--json"]
if args.line:
command.append("-n")
# Prepare process
cotovia_proc = subprocess.Popen(command, stdin=PIPE, stdout=PIPE, stderr=PIPE)
# Write UTF-8 data on input steam
cotovia_proc.stdin.write(bytes(cotovia_input, 'utf-8'))
# Wait for process response
(output, errs) = cotovia_proc.communicate()
# Convert results to UTF-8
output = output.decode('utf-8').strip()
errs = errs.decode('iso8859-1').strip()
cotovia_proc.wait()
data = json.loads(output)
output = aggregate_json_field(data, json_field)
if json_field == "frase_fonetica":
output = clear_frase_fonetica(output)
expected_output = clear_frase_fonetica(expected_output)
is_failure = output != expected_output
if not args.compact or is_failure:
print("=====================================================")
print(f"Testing input: {cotovia_input}")
if is_failure:
print(YELLOW + "Output Mismatch!")
print(YELLOW + f"Expected: {expected_output}")
print(YELLOW + f"Got: {output}" + RESET)
failed += 1
else:
if not args.compact:
print("OK!")
passed += 1
print("\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\")
if failed > 0:
print(RED + "=======================SUMMARY=======================")
print(RED + f"Passed {passed} tests")
print(RED + f"Failed {failed} tests" + RESET)
else:
print(GREEN + "=======================SUMMARY=======================")
print(GREEN + f"Passed {passed} tests")
print(GREEN + f"Failed {failed} tests" + RESET)
sys.stdout.flush()
sys.stderr.flush()
if failed > 0:
print(RED + "Failed to pass tests!" + RESET, file=sys.stderr )
sys.stderr.flush()
sys.exit(1)
|