Utils/cotovia/tests/scripts/eval.py

#!/usr/bin/python3

from colorama import Fore, Style
import json
import subprocess
import argparse
import pathlib
import csv
import sys
import os

from subprocess import PIPE

def clear_frase_fonetica(input_string: str) -> str:
    input_string = input_string.lower()  # Transform input to lowercase
    symbols_to_remove = ["-", "^"]
    for symbol in symbols_to_remove:
        input_string = input_string.replace(symbol, "")
    return input_string

def aggregate_json_field(json, json_field):
    concatenated_values = ""
    first_value = True

    for item in json:
        if not first_value:
            concatenated_values += " \n"
        concatenated_values += item.get(json_field, "").strip()
        first_value = False

    return concatenated_values

# Default filenames
eval_file = 'eval_gal_voice.csv'
exec_file = 'cotovia'

# Global run stats
passed = 0
failed = 0

# Colors
GREEN = Fore.GREEN
YELLOW = Fore.YELLOW
RED = Fore.RED
RESET = Style.RESET_ALL

# Get default paths
p = pathlib.PurePath(os.path.abspath(__file__))
data_path = p.parents[1].joinpath('data').joinpath(eval_file)
exec_path = p.parents[2].joinpath('build/cotovia').joinpath(exec_file)

# Command line arguments
parser = argparse.ArgumentParser(
    description='Evaluate cotovia transcription against annotated files')
parser.add_argument('--exec', help='Override cotovia executable path')
parser.add_argument('--eval', help='Override cotovia evaluation file path')
parser.add_argument('--pmode', type=int, choices=[0, 1], default=0,
                    help='Select processing mode (default 0): 0 => Voice Synthesis, 1 => Speech Recognition')
parser.add_argument('--delimiter', type=str, default=',',
                    help='CSV eval files default delimiter')
parser.add_argument('--line', default=False,
                    action='store_true', help='Enables treating each line as an expression, ignoring other criteria.')
parser.add_argument('--compact', default=False,
                    action='store_true', help='Report only failed tests')
parser.add_argument('--json_field', type=str, help='Json field of the cotovia output to match', default="frase_procesada")
args = parser.parse_args()

if args.exec:
    exec_path = pathlib.PurePath(os.path.abspath(args.exec))
if args.eval:
    data_path = pathlib.PurePath(os.path.abspath(args.eval))
process_mode: int = args.pmode
json_field = args.json_field

print("=====================================================")
print(f"Eval data path: {data_path}")
print(f"Exec path: {exec_path}")
if process_mode == 0:
    print("Process mode: Voice Synthesis")
elif process_mode == 1:
    print("Process mode: Speech Recognition")

paths = pathlib.Path(data_path.parent).glob(data_path.name)

for path in paths:
    with open(path, newline='') as eval_data:
        print("////////////////////////START////////////////////////")
        print(f"Evaluating file: {path}")
        input_reader = csv.reader(eval_data, delimiter=args.delimiter)
        for entry in input_reader:
            # data to send to cotovia
            cotovia_input = entry[0].strip()
            expected_output = entry[1].strip()
            # prepare process
            command = [str(exec_path), f"-p{process_mode}", "-S", "-L", "--json"]
            if args.line:
                command.append("-n")
            # Prepare process
            cotovia_proc = subprocess.Popen(command, stdin=PIPE, stdout=PIPE, stderr=PIPE)
            # Write UTF-8 data on input steam
            cotovia_proc.stdin.write(bytes(cotovia_input, 'utf-8'))
            # Wait for process response
            (output, errs) = cotovia_proc.communicate()
            # Convert results to UTF-8
            output = output.decode('utf-8').strip()
            errs = errs.decode('iso8859-1').strip()
            cotovia_proc.wait()

            data = json.loads(output)
            output = aggregate_json_field(data, json_field)

            if json_field == "frase_fonetica":
                output = clear_frase_fonetica(output)
                expected_output = clear_frase_fonetica(expected_output)

            is_failure = output != expected_output

            if not args.compact or is_failure:
                print("=====================================================")
                print(f"Testing input: {cotovia_input}")

            if is_failure:
                print(YELLOW + "Output Mismatch!")
                print(YELLOW + f"Expected: {expected_output}")
                print(YELLOW + f"Got:      {output}" + RESET)
                failed += 1
            else:
                if not args.compact:
                    print("OK!")
                passed += 1
        print("\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\")

if failed > 0:
    print(RED + "=======================SUMMARY=======================")
    print(RED + f"Passed {passed} tests")
    print(RED + f"Failed {failed} tests" + RESET)
else:
    print(GREEN + "=======================SUMMARY=======================")
    print(GREEN + f"Passed {passed} tests")
    print(GREEN + f"Failed {failed} tests" + RESET)

sys.stdout.flush()
sys.stderr.flush()

if failed > 0:
    print(RED + "Failed to pass tests!" + RESET, file=sys.stderr )
    sys.stderr.flush()
    sys.exit(1)