// SPDX-License-Identifier: Apache-2.0 package com.laya import android.content.Context import android.os.Build import android.util.Log import java.io.File import kotlin.math.abs import kotlin.math.sqrt /** Debug-only device evidence collector. Numeric acceptance belongs to the supervising round. */ class LayaGateRunner(private val context: Context, private val engine: LayaEngine) { data class Report(val path: String, val status: String, val error: String? = null) fun run(backend: LayaEngine.Backend, window: Int): Report { check(BuildConfig.DEBUG) { "The fixture gate is available only in the debug build" } require(window == 256 || window == 512) { "Expected gate window 256 or 512" } val output = File(context.filesDir, "laya_gate_${backend.name.lowercase()}_$window.json") val report = linkedMapOf( "schema_version" to 1, "device" to "${Build.MANUFACTURER} ${Build.MODEL}", "android_release" to Build.VERSION.RELEASE, "android_sdk" to Build.VERSION.SDK_INT, "build_fingerprint" to Build.FINGERPRINT, "application_id" to BuildConfig.APPLICATION_ID, "version_name" to BuildConfig.VERSION_NAME, "debuggable" to BuildConfig.DEBUG, "litert_version" to LayaEngine.LITERT_VERSION, "accelerator" to backend.name.lowercase(), "gpu_precision" to if (backend == LayaEngine.Backend.GPU) "FP32" else null, "graph_storage" to engine.storage.argument, "main_model" to engine.mainFilename(window), "embedding_table_sha256" to engine.embeddingTableSha256, "window" to window, "temperature" to 1.0, "tokenizer_load_ms" to engine.tokenizerLoadMs, "embedding_map_ms" to engine.embeddingLoadMs, "numerical_acceptance" to "NOT_EVALUATED: supervisor supplies round-2 tolerances", "timing_definition" to "main and action write + run enqueue + output readback; cold first call excluded from warm summary", ) val results = mutableListOf>() report["rows"] = results var setupError: String? = null try { val fixtureFile = File(context.filesDir, "fixtures/gate_rows_s$window.json") check(fixtureFile.isFile) { "Missing ${fixtureFile.path}; run scripts/install_to_device.sh" } val fixture = LayaJson.asObject(LayaJson.parse(fixtureFile)) require((fixture["window"] as Number).toInt() == window) { "Fixture window mismatch" } val rows = LayaJson.asArray(fixture["rows"]).map { LayaJson.asObject(it) } check(rows.isNotEmpty()) { "Gate fixture has no rows" } report["expected_rows"] = rows.size val compileStarted = System.nanoTime() engine.initialize(backend, window) report["compile_ms"] = ms(System.nanoTime() - compileStarted) var firstCall = true rows.forEach { row -> val detail = linkedMapOf("row_id" to row["row_id"]) results += detail try { val question = LayaJson.asObject(row["question"]) val expectedIds = integers(row["sequence_ids"]) val expectedMarkers = integers(row["marker_positions"]) val preparedStarted = System.nanoTime() val sequence = engine.prepare(row["state"], question, window, row["question_id"].toString()) detail["prepare_ms"] = ms(System.nanoTime() - preparedStarted) detail["sequence_ids"] = sequence.ids.toList() detail["marker_positions"] = sequence.markers.toList() detail["ids_exact"] = sequence.ids.contentEquals(expectedIds) detail["markers_exact"] = sequence.markers.contentEquals(expectedMarkers) detail["first_id_difference"] = firstDifference(sequence.ids, expectedIds) detail["first_marker_difference"] = firstDifference(sequence.markers, expectedMarkers) detail["qtype"] = sequence.question.qtype detail["K"] = sequence.markers.size require(sequence.question.qtype == (row["qtype"] as Number).toInt()) { "qtype mismatch" } require(sequence.markers.size == (row["K"] as Number).toInt()) { "Option-count mismatch" } detail["cold_first_call"] = firstCall firstCall = false val raw = engine.runRaw(sequence, backend, window) detail["graph_run_completed"] = true detail["finite"] = raw.finite detail["token_logits"] = finiteList(raw.tokenLogits) detail["raw_logits"] = finiteList(raw.markerLogits) detail["raw_act_logits"] = finiteList(raw.actLogits) detail["act_features"] = finiteList(raw.actFeatures) detail["token_logit_scale"] = scale(raw.tokenLogits) detail["pooled_cls_scale"] = scale(raw.pooledCls) detail["marker_logit_difference"] = difference(raw.markerLogits, floats(row["raw_logits"])) detail["act_logit_difference"] = difference(raw.actLogits, floats(row["raw_act_logits"])) detail["main_timing"] = raw.mainTiming.toMap() detail["act_timing"] = raw.actTiming?.toMap() detail["embedding_lookup_ms"] = raw.embeddingLookupMs detail["feature_ms"] = raw.featureMs detail["write_run_read_ms"] = raw.graphMs detail["call_ms"] = raw.callMs if (raw.finite) { val decodeStarted = System.nanoTime() val actual = LayaDecoder.decode( raw.markerLogits, raw.actLogits, sequence.question, LayaCalibration.identity(), ) val official = LayaJson.asObject(row["official_answer"]) val probabilities = LayaDecoder.softmax(raw.markerLogits) val referenceProbabilities = floats(row["probabilities"]) detail["decode_ms"] = ms(System.nanoTime() - decodeStarted) detail["answer"] = actual detail["official_answer"] = official detail["official_dict_exact"] = jsonEqual(actual, official) detail["probabilities"] = probabilities.toList() detail["argmax"] = argmax(probabilities) detail["reference_argmax"] = argmax(referenceProbabilities) detail["argmax_exact"] = argmax(probabilities) == argmax(referenceProbabilities) detail["probability_max_abs_error"] = maxError(probabilities, referenceProbabilities) detail["answer_max_numeric_error"] = numericError(actual, official) } else { detail["error"] = "Nonfinite model values; action/decode may be skipped; see tensor scales" } } catch (failure: Exception) { detail["error"] = "${failure.javaClass.simpleName}: ${failure.message}" } catch (failure: LinkageError) { detail["error"] = "${failure.javaClass.simpleName}: ${failure.message}" } } } catch (failure: Exception) { setupError = "${failure.javaClass.simpleName}: ${failure.message}" } catch (failure: LinkageError) { setupError = "${failure.javaClass.simpleName}: ${failure.message}" } val summary = linkedMapOf( "rows_recorded" to results.size, "ids_exact" to results.count { it["ids_exact"] == true }, "markers_exact" to results.count { it["markers_exact"] == true }, "finite_rows" to results.count { it["finite"] == true }, "argmax_exact" to results.count { it["argmax_exact"] == true }, "official_dict_exact" to results.count { it["official_dict_exact"] == true }, "row_errors" to results.count { it["error"] != null }, "probability_max_abs_error" to results .mapNotNull { (it["probability_max_abs_error"] as? Number)?.toDouble() } .maxOrNull(), "cold_first_call" to results .firstOrNull { it["cold_first_call"] == true } ?.let { linkedMapOf( "row_id" to it["row_id"], "write_run_read_ms" to it["write_run_read_ms"], "call_ms" to it["call_ms"], "main_timing" to it["main_timing"], "act_timing" to it["act_timing"], ) }, "warm_write_run_read_ms" to timingSummary( results .filter { it["cold_first_call"] == false } .mapNotNull { (it["write_run_read_ms"] as? Number)?.toDouble() } ), ) val status = if ( setupError != null || results.isEmpty() || results.any { it["error"] != null || it["ids_exact"] != true || it["markers_exact"] != true } ) "FAIL" else "MEASURED" report["status"] = status report["summary"] = summary report["error"] = setupError output.writeText(LayaJson.stringify(report)) Log.i( TAG, "summary ${LayaJson.stringify(linkedMapOf( "status" to status, "accelerator" to backend.name.lowercase(), "window" to window, "graph_storage" to engine.storage.argument, "rows" to results.size, "ids_exact" to summary["ids_exact"], "markers_exact" to summary["markers_exact"], "finite_rows" to summary["finite_rows"], "argmax_exact" to summary["argmax_exact"], "max_delta_p" to summary["probability_max_abs_error"], "file" to output.name, "error" to setupError, ))}", ) return Report(output.absolutePath, status, setupError) } private fun integers(value: Any?) = LayaJson.asArray(value).map { (it as Number).toInt() }.toIntArray() private fun floats(value: Any?) = LayaJson.asArray(value).map { (it as Number).toFloat() }.toFloatArray() private fun finiteList(values: FloatArray): List = values.map { if (it.isFinite()) it else null } private fun ms(nanos: Long) = nanos / 1_000_000.0 private fun firstDifference(actual: IntArray, reference: IntArray): Int? { for (index in 0 until minOf(actual.size, reference.size)) { if (actual[index] != reference[index]) return index } return if (actual.size == reference.size) null else minOf(actual.size, reference.size) } private fun scale(values: FloatArray): Map { val finite = values.filter { it.isFinite() } return linkedMapOf( "count" to values.size, "nonfinite_count" to values.count { !it.isFinite() }, "minimum" to finite.minOrNull(), "maximum" to finite.maxOrNull(), "max_absolute" to finite.maxOfOrNull { abs(it.toDouble()) }, ) } private fun difference(actual: FloatArray, reference: FloatArray): Map { val valid = actual.size == reference.size && actual.all { it.isFinite() } && reference.all { it.isFinite() } val maxAbs = if (valid) maxError(actual, reference) else null val referenceScale = reference.maxOfOrNull { abs(it.toDouble()) } ?: 0.0 return linkedMapOf( "observed_scale" to scale(actual), "reference_scale" to scale(reference), "max_absolute_error" to maxAbs, "max_absolute_error_over_reference_max_absolute" to if (maxAbs != null && referenceScale != 0.0) maxAbs / referenceScale else null, "root_mean_square_error" to if (valid) sqrt( actual.indices.sumOf { val delta = actual[it].toDouble() - reference[it].toDouble() delta * delta } / actual.size ) else null, ) } private fun maxError(actual: FloatArray, reference: FloatArray): Double { require(actual.size == reference.size) { "Probability/vector length mismatch" } return actual.indices.maxOfOrNull { abs(actual[it].toDouble() - reference[it].toDouble()) } ?: 0.0 } private fun argmax(values: FloatArray): Int { require(values.isNotEmpty()) var best = 0 for (index in 1 until values.size) if (values[index] > values[best]) best = index return best } private fun jsonEqual(actual: Any?, reference: Any?): Boolean = when { actual is Number && reference is Number -> actual.toDouble() == reference.toDouble() actual is Map<*, *> && reference is Map<*, *> -> actual.keys == reference.keys && actual.keys.all { jsonEqual(actual[it], reference[it]) } actual is List<*> && reference is List<*> -> actual.size == reference.size && actual.indices.all { jsonEqual(actual[it], reference[it]) } else -> actual == reference } private fun numericError(actual: Any?, reference: Any?): Double = when { actual is Number && reference is Number -> abs(actual.toDouble() - reference.toDouble()) actual is Map<*, *> && reference is Map<*, *> -> actual.keys.maxOfOrNull { numericError(actual[it], reference[it]) } ?: 0.0 actual is List<*> && reference is List<*> -> actual.indices.maxOfOrNull { numericError(actual[it], reference.getOrNull(it)) } ?: 0.0 else -> 0.0 } private fun timingSummary(values: List): Map { val sorted = values.sorted() val median = if (sorted.isEmpty()) null else if (sorted.size % 2 == 0) { (sorted[sorted.size / 2 - 1] + sorted[sorted.size / 2]) / 2 } else sorted[sorted.size / 2] return linkedMapOf( "count" to values.size, "median" to median, "minimum" to sorted.firstOrNull(), "maximum" to sorted.lastOrNull(), ) } companion object { const val TAG = "LAYA_GATE" } }