#include #include #include #include #include #include #include #include #include #include #include #include #include "ax_engine_api.h" #include "ax_sys_api.h" #include "src/engine_wrapper.hpp" #include "src/wav_reader.hpp" #ifndef AXERA_TARGET_NAME #define AXERA_TARGET_NAME "AXERA" #endif namespace { constexpr int kSampleRate = 16000; constexpr int kChunkSamples = 1280; constexpr int kHistorySamples = 480; constexpr int kFftSize = 512; constexpr int kSpectrumBins = 257; constexpr int kMelBins = 32; constexpr int kMelFrames = 8; constexpr int kEmbeddingFrames = 76; constexpr int kEmbeddingSize = 96; constexpr int kFeatureFrames = 34; using Clock = std::chrono::steady_clock; double ElapsedSeconds(Clock::time_point begin, Clock::time_point end) { return std::chrono::duration(end - begin).count(); } struct Args { std::string models_dir = "models"; std::string weights = "config/openwakeword_mel_weights.bin"; std::string audio = "audio/openwakeword/alexa_test.wav"; float threshold = 0.5f; }; void Usage(const char *program) { std::printf( "Usage: %s [--models-dir DIR] [--mel-weights FILE] [--audio WAV] " "[--threshold VALUE]\n", program); } Args ParseArgs(int argc, char **argv) { Args args; for (int i = 1; i < argc; ++i) { const std::string key = argv[i]; auto value = [&]() -> std::string { if (++i >= argc) throw std::runtime_error("Missing value for " + key); return argv[i]; }; if (key == "--models-dir") { args.models_dir = value(); } else if (key == "--mel-weights") { args.weights = value(); } else if (key == "--audio") { args.audio = value(); } else if (key == "--threshold") { args.threshold = std::stof(value()); } else if (key == "-h" || key == "--help") { Usage(argv[0]); std::exit(0); } else { throw std::runtime_error("Unknown argument: " + key); } } return args; } std::string Join(const std::string &left, const std::string &right) { return left.empty() || left.back() == '/' ? left + right : left + "/" + right; } class AxRuntime { public: AxRuntime() { if (AX_SYS_Init() != 0) throw std::runtime_error("AX_SYS_Init failed"); sys_initialized_ = true; AX_ENGINE_NPU_ATTR_T attr{}; if (AX_ENGINE_Init(&attr) != 0) { AX_SYS_Deinit(); sys_initialized_ = false; throw std::runtime_error("AX_ENGINE_Init failed"); } engine_initialized_ = true; } ~AxRuntime() { if (engine_initialized_) AX_ENGINE_Deinit(); if (sys_initialized_) AX_SYS_Deinit(); } private: bool sys_initialized_ = false; bool engine_initialized_ = false; }; template T ReadScalar(std::istream &input) { T value{}; input.read(reinterpret_cast(&value), sizeof(value)); if (!input) throw std::runtime_error("Truncated mel weight file"); return value; } struct MelWeights { std::vector real; std::vector imag; std::vector mel; float floor = 0.0f; static MelWeights Load(const std::string &path) { std::ifstream input(path, std::ios::binary); if (!input) throw std::runtime_error("Cannot open mel weights: " + path); char magic[8]{}; input.read(magic, 8); if (std::memcmp(magic, "OWWMEL1", 7) != 0 || ReadScalar(input) != 1) { throw std::runtime_error("Invalid openWakeWord mel weight file"); } const uint32_t real_rows = ReadScalar(input); const uint32_t real_cols = ReadScalar(input); const uint32_t imag_rows = ReadScalar(input); const uint32_t imag_cols = ReadScalar(input); const uint32_t mel_rows = ReadScalar(input); const uint32_t mel_cols = ReadScalar(input); MelWeights result; result.floor = ReadScalar(input); if (real_rows != kSpectrumBins || real_cols != kFftSize || imag_rows != kSpectrumBins || imag_cols != kFftSize || mel_rows != kSpectrumBins || mel_cols != kMelBins) { throw std::runtime_error("Unexpected openWakeWord mel weight shapes"); } result.real.resize(static_cast(real_rows) * real_cols); result.imag.resize(static_cast(imag_rows) * imag_cols); result.mel.resize(static_cast(mel_rows) * mel_cols); input.read(reinterpret_cast(result.real.data()), result.real.size() * sizeof(float)); input.read(reinterpret_cast(result.imag.data()), result.imag.size() * sizeof(float)); input.read(reinterpret_cast(result.mel.data()), result.mel.size() * sizeof(float)); if (!input) throw std::runtime_error("Truncated openWakeWord mel weights"); return result; } }; std::array ComputeMel( const std::array &samples, const MelWeights &weights) { std::array result{}; std::array power{}; float max_db = -std::numeric_limits::infinity(); for (int frame = 0; frame < kMelFrames; ++frame) { const float *frame_samples = samples.data() + frame * 160; for (int frequency = 0; frequency < kSpectrumBins; ++frequency) { const float *real = weights.real.data() + frequency * kFftSize; const float *imag = weights.imag.data() + frequency * kFftSize; float real_sum = 0.0f; float imag_sum = 0.0f; for (int n = 0; n < kFftSize; ++n) { real_sum += frame_samples[n] * real[n]; imag_sum += frame_samples[n] * imag[n]; } power[frequency] = real_sum * real_sum + imag_sum * imag_sum; } for (int bin = 0; bin < kMelBins; ++bin) { float value = 0.0f; for (int frequency = 0; frequency < kSpectrumBins; ++frequency) { value += power[frequency] * weights.mel[frequency * kMelBins + bin]; } value = std::max(value, weights.floor); const float db = std::log(value) * 10.0f / 2.3025851249694824f; result[frame * kMelBins + bin] = db; max_db = std::max(max_db, db); } } const float minimum = max_db - 80.0f; for (float &value : result) { value = std::max(value, minimum) / 10.0f + 2.0f; } return result; } struct Classifier { std::string name; int frames; EngineWrapper engine; std::vector maximum; }; void Run(const Args &args) { const PcmWav wav = ReadPcmWav(args.audio); if (wav.sample_rate != kSampleRate) { throw std::runtime_error("Input WAV must use 16 kHz sample rate"); } const double audio_seconds = static_cast(wav.samples.size()) / kSampleRate; if (audio_seconds <= 0.0) { throw std::runtime_error("Input WAV contains no samples"); } const MelWeights weights = MelWeights::Load(args.weights); const auto model_load_begin = Clock::now(); AxRuntime runtime; EngineWrapper embedding; if (embedding.Init(Join(args.models_dir, "openwakeword__embedding_model.axmodel")) != 0) { throw std::runtime_error("Failed to load embedding model"); } const std::array, 6> definitions{{ {"alexa_v0.1", 16}, {"hey_jarvis_v0.1", 16}, {"hey_mycroft_v0.1", 16}, {"hey_rhasspy_v0.1", 16}, {"timer_v0.1", 34}, {"weather_v0.1", 22}, }}; std::array classifiers; for (std::size_t classifier_index = 0; classifier_index < definitions.size(); ++classifier_index) { const auto &definition = definitions[classifier_index]; Classifier &classifier = classifiers[classifier_index]; classifier.name = definition.first; classifier.frames = definition.second; const std::string model = Join(args.models_dir, "openwakeword__" + classifier.name + ".axmodel"); if (classifier.engine.Init(model) != 0) { throw std::runtime_error("Failed to load classifier: " + classifier.name); } const int output_bytes = classifier.engine.GetOutputSizeByName( classifier.engine.OutputName(0)); if (output_bytes <= 0 || output_bytes % sizeof(float) != 0) { throw std::runtime_error("Unexpected classifier output: " + classifier.name); } classifier.maximum.assign(output_bytes / sizeof(float), -std::numeric_limits::infinity()); } const double model_load_seconds = ElapsedSeconds(model_load_begin, Clock::now()); std::vector padded = wav.samples; const std::size_t remainder = padded.size() % kChunkSamples; if (remainder != 0) padded.resize(padded.size() + kChunkSamples - remainder); std::array history{}; std::array mel_buffer{}; mel_buffer.fill(1.0f); std::array feature_buffer{}; int chunks = 0; double feature_seconds = 0.0; double npu_seconds = 0.0; const auto inference_begin = Clock::now(); for (std::size_t start = 0; start < padded.size(); start += kChunkSamples) { std::array mel_input{}; for (int i = 0; i < kHistorySamples; ++i) mel_input[i] = history[i]; for (int i = 0; i < kChunkSamples; ++i) { mel_input[kHistorySamples + i] = padded[start + i]; } for (int i = 0; i < kHistorySamples; ++i) { history[i] = padded[start + kChunkSamples - kHistorySamples + i]; } const auto feature_begin = Clock::now(); const auto mel = ComputeMel(mel_input, weights); feature_seconds += ElapsedSeconds(feature_begin, Clock::now()); std::memmove(mel_buffer.data(), mel_buffer.data() + kMelFrames * kMelBins, (kEmbeddingFrames - kMelFrames) * kMelBins * sizeof(float)); std::memcpy(mel_buffer.data() + (kEmbeddingFrames - kMelFrames) * kMelBins, mel.data(), mel.size() * sizeof(float)); const std::string &embedding_input = embedding.InputName(0); if (embedding.SetInputByName(embedding_input, mel_buffer.data(), mel_buffer.size() * sizeof(float)) != 0) { throw std::runtime_error("Failed to set embedding input"); } const auto embedding_begin = Clock::now(); const int embedding_ret = embedding.RunSync(); npu_seconds += ElapsedSeconds(embedding_begin, Clock::now()); if (embedding_ret != 0) { throw std::runtime_error("Embedding inference failed"); } std::array feature{}; if (embedding.GetOutputByName(embedding.OutputName(0), feature.data(), feature.size() * sizeof(float)) != 0) { throw std::runtime_error("Failed to read embedding output"); } std::memmove(feature_buffer.data(), feature_buffer.data() + kEmbeddingSize, (kFeatureFrames - 1) * kEmbeddingSize * sizeof(float)); std::memcpy(feature_buffer.data() + (kFeatureFrames - 1) * kEmbeddingSize, feature.data(), feature.size() * sizeof(float)); for (Classifier &classifier : classifiers) { const float *input = feature_buffer.data() + (kFeatureFrames - classifier.frames) * kEmbeddingSize; const std::string &input_name = classifier.engine.InputName(0); if (classifier.engine.SetInputByName( input_name, input, classifier.frames * kEmbeddingSize * sizeof(float)) != 0) { throw std::runtime_error("Failed to set classifier input: " + classifier.name); } const auto classifier_begin = Clock::now(); const int classifier_ret = classifier.engine.RunSync(); npu_seconds += ElapsedSeconds(classifier_begin, Clock::now()); if (classifier_ret != 0) { throw std::runtime_error("Classifier inference failed: " + classifier.name); } std::vector output(classifier.maximum.size()); if (classifier.engine.GetOutputByName( classifier.engine.OutputName(0), output.data(), output.size() * sizeof(float)) != 0) { throw std::runtime_error("Failed to read classifier output"); } for (std::size_t i = 0; i < output.size(); ++i) { classifier.maximum[i] = std::max(classifier.maximum[i], output[i]); } } ++chunks; } const double inference_seconds = ElapsedSeconds(inference_begin, Clock::now()); const double rtf = inference_seconds / audio_seconds; std::printf("\nopenWakeWord C++ inference complete\n"); std::printf("target: %s\naudio: %s\nchunks: %d\nthreshold: %.3f\n", AXERA_TARGET_NAME, args.audio.c_str(), chunks, args.threshold); bool detected = false; for (const Classifier &classifier : classifiers) { std::printf("%-22s", classifier.name.c_str()); float score = -std::numeric_limits::infinity(); for (std::size_t i = 0; i < classifier.maximum.size(); ++i) { const float value = classifier.maximum[i]; std::printf(" %.6f", value); if (classifier.name != "timer_v0.1" || i != 0) { score = std::max(score, value); } } if (score >= args.threshold) { std::printf(" WAKEUP"); detected = true; } std::printf("\n"); } std::printf("detected: %s\n", detected ? "true" : "false"); std::printf("audio_seconds: %.6f\n", audio_seconds); std::printf("feature_seconds: %.6f\n", feature_seconds); std::printf("npu_seconds: %.6f\n", npu_seconds); std::printf("model_load_seconds: %.6f\n", model_load_seconds); std::printf("inference_seconds: %.6f\n", inference_seconds); std::printf("rtf: %.6f\n", rtf); } } // namespace int main(int argc, char **argv) { try { const Args args = ParseArgs(argc, argv); Run(args); return 0; } catch (const std::exception &error) { std::fprintf(stderr, "ERROR: %s\n", error.what()); return 1; } }