#!/usr/bin/python3 from colorama import Fore, Style import json import subprocess import argparse import pathlib import csv import sys import os from subprocess import PIPE def clear_frase_fonetica(input_string: str) -> str: input_string = input_string.lower() # Transform input to lowercase symbols_to_remove = ["-", "^"] for symbol in symbols_to_remove: input_string = input_string.replace(symbol, "") return input_string def aggregate_json_field(json, json_field): concatenated_values = "" first_value = True for item in json: if not first_value: concatenated_values += " \n" concatenated_values += item.get(json_field, "").strip() first_value = False return concatenated_values # Default filenames eval_file = 'eval_gal_voice.csv' exec_file = 'cotovia' # Global run stats passed = 0 failed = 0 # Colors GREEN = Fore.GREEN YELLOW = Fore.YELLOW RED = Fore.RED RESET = Style.RESET_ALL # Get default paths p = pathlib.PurePath(os.path.abspath(__file__)) data_path = p.parents[1].joinpath('data').joinpath(eval_file) exec_path = p.parents[2].joinpath('build/cotovia').joinpath(exec_file) # Command line arguments parser = argparse.ArgumentParser( description='Evaluate cotovia transcription against annotated files') parser.add_argument('--exec', help='Override cotovia executable path') parser.add_argument('--eval', help='Override cotovia evaluation file path') parser.add_argument('--pmode', type=int, choices=[0, 1], default=0, help='Select processing mode (default 0): 0 => Voice Synthesis, 1 => Speech Recognition') parser.add_argument('--delimiter', type=str, default=',', help='CSV eval files default delimiter') parser.add_argument('--line', default=False, action='store_true', help='Enables treating each line as an expression, ignoring other criteria.') parser.add_argument('--compact', default=False, action='store_true', help='Report only failed tests') parser.add_argument('--json_field', type=str, help='Json field of the cotovia output to match', default="frase_procesada") args = parser.parse_args() if args.exec: exec_path = pathlib.PurePath(os.path.abspath(args.exec)) if args.eval: data_path = pathlib.PurePath(os.path.abspath(args.eval)) process_mode: int = args.pmode json_field = args.json_field print("=====================================================") print(f"Eval data path: {data_path}") print(f"Exec path: {exec_path}") if process_mode == 0: print("Process mode: Voice Synthesis") elif process_mode == 1: print("Process mode: Speech Recognition") paths = pathlib.Path(data_path.parent).glob(data_path.name) for path in paths: with open(path, newline='') as eval_data: print("////////////////////////START////////////////////////") print(f"Evaluating file: {path}") input_reader = csv.reader(eval_data, delimiter=args.delimiter) for entry in input_reader: # data to send to cotovia cotovia_input = entry[0].strip() expected_output = entry[1].strip() # prepare process command = [str(exec_path), f"-p{process_mode}", "-S", "-L", "--json"] if args.line: command.append("-n") # Prepare process cotovia_proc = subprocess.Popen(command, stdin=PIPE, stdout=PIPE, stderr=PIPE) # Write UTF-8 data on input steam cotovia_proc.stdin.write(bytes(cotovia_input, 'utf-8')) # Wait for process response (output, errs) = cotovia_proc.communicate() # Convert results to UTF-8 output = output.decode('utf-8').strip() errs = errs.decode('iso8859-1').strip() cotovia_proc.wait() data = json.loads(output) output = aggregate_json_field(data, json_field) if json_field == "frase_fonetica": output = clear_frase_fonetica(output) expected_output = clear_frase_fonetica(expected_output) is_failure = output != expected_output if not args.compact or is_failure: print("=====================================================") print(f"Testing input: {cotovia_input}") if is_failure: print(YELLOW + "Output Mismatch!") print(YELLOW + f"Expected: {expected_output}") print(YELLOW + f"Got: {output}" + RESET) failed += 1 else: if not args.compact: print("OK!") passed += 1 print("\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\") if failed > 0: print(RED + "=======================SUMMARY=======================") print(RED + f"Passed {passed} tests") print(RED + f"Failed {failed} tests" + RESET) else: print(GREEN + "=======================SUMMARY=======================") print(GREEN + f"Passed {passed} tests") print(GREEN + f"Failed {failed} tests" + RESET) sys.stdout.flush() sys.stderr.flush() if failed > 0: print(RED + "Failed to pass tests!" + RESET, file=sys.stderr ) sys.stderr.flush() sys.exit(1)