using System.Collections.Generic; using System.Diagnostics; using Unity.InferenceEngine; using Unity.MLAgents.Actuators; using Unity.MLAgents.Inference; using Unity.MLAgents.Policies; using Unity.MLAgents.Sensors; using UnityEngine; #if MLA_UNITY_ANALYTICS_MODULE && ENABLE_CLOUD_SERVICES_ANALYTICS using UnityEngine.Analytics; #endif #if UNITY_EDITOR using UnityEditor; #if MLA_UNITY_ANALYTICS_MODULE using UnityEditor.Analytics; #endif // MLA_UNITY_ANALYTICS_MODULE #endif // UNITY_EDITOR namespace Unity.MLAgents.Analytics { internal class InferenceAnalytics { #if UNITY_EDITOR && MLA_UNITY_ANALYTICS_MODULE && ENABLE_CLOUD_SERVICES_ANALYTICS /// /// Models that we've already sent events for. /// private static HashSet s_SentModels; #endif static bool EnableAnalytics() { #if UNITY_EDITOR && MLA_UNITY_ANALYTICS_MODULE && ENABLE_CLOUD_SERVICES_ANALYTICS if (s_SentModels == null) { s_SentModels = new HashSet(); } return true; #else // no editor, no analytics return false; #endif } public static bool IsAnalyticsEnabled() { #if UNITY_EDITOR return EditorAnalytics.enabled; #else return false; #endif } /// /// Send an analytics event for the NNModel when it is set up for inference. /// No events will be sent if analytics are disabled, and at most one event /// will be sent per model instance. /// /// The NNModel being used for inference. /// The BehaviorName of the Agent using the model /// Whether inference is being performed on the CPU or GPU /// List of ISensors for the Agent. Used to generate information about the observation space. /// ActionSpec for the Agent. Used to generate information about the action space. /// List of IActuators for the Agent. Used to generate information about the action space. [Conditional("MLA_UNITY_ANALYTICS_MODULE")] public static void InferenceModelSet( ModelAsset nnModel, string behaviorName, InferenceDevice inferenceDevice, IList sensors, ActionSpec actionSpec, IList actuators ) { #if UNITY_EDITOR && MLA_UNITY_ANALYTICS_MODULE && ENABLE_CLOUD_SERVICES_ANALYTICS // The event shouldn't be able to report if this is disabled but if we know we're not going to report // Lets early out and not waste time gathering all the data if (!IsAnalyticsEnabled()) return; if (!EnableAnalytics()) return; var added = s_SentModels.Add(nnModel); if (!added) { // We previously added this model. Exit so we don't resend. return; } var data = GetEventForModel(nnModel, behaviorName, inferenceDevice, sensors, actionSpec, actuators); // Note - to debug, use JsonUtility.ToJson on the event. // Debug.Log(JsonUtility.ToJson(data, true)); if (AnalyticsUtils.s_SendEditorAnalytics) { EditorAnalytics.SendAnalytic(data); } #endif } /// /// Generate an InferenceEvent for the model. /// /// /// /// /// /// /// /// `InferenceEvent` from the input model. internal static InferenceEvent GetEventForModel( ModelAsset nnModel, string behaviorName, InferenceDevice inferenceDevice, IList sensors, ActionSpec actionSpec, IList actuators ) { var sentisModel = ModelLoader.Load(nnModel); using var sentisModelInfo = new SentisModelInfo(sentisModel); var inferenceEvent = new InferenceEvent(); // Hash the behavior name so that there's no concern about PII or "secret" data being leaked. inferenceEvent.BehaviorName = AnalyticsUtils.Hash(AnalyticsConstants.k_VendorKey, behaviorName); inferenceEvent.SentisModelVersion = sentisModelInfo.Version; inferenceEvent.SentisModelProducer = sentisModel.ProducerName; inferenceEvent.MemorySize = sentisModelInfo.MemorySize; inferenceEvent.InferenceDevice = (int)inferenceDevice; // TODO deprecate tensorflow conversion if (sentisModel.ProducerName == "Script") { // .nn files don't have these fields set correctly. Assign some placeholder values. inferenceEvent.SentisModelSource = "NN"; inferenceEvent.SentisModelProducer = "tensorflow_to_barracuda.py"; } #if UNITY_EDITOR var sentisPackageInfo = UnityEditor.PackageManager.PackageInfo.FindForAssembly(typeof(Tensor).Assembly); inferenceEvent.SentisPackageVersion = sentisPackageInfo.version; #else inferenceEvent.SentisPackageVersion = null; #endif inferenceEvent.ActionSpec = EventActionSpec.FromActionSpec(actionSpec); inferenceEvent.ObservationSpecs = new List(sensors.Count); foreach (var sensor in sensors) { inferenceEvent.ObservationSpecs.Add(EventObservationSpec.FromSensor(sensor)); } inferenceEvent.ActuatorInfos = new List(actuators.Count); foreach (var actuator in actuators) { inferenceEvent.ActuatorInfos.Add(EventActuatorInfo.FromActuator(actuator)); } inferenceEvent.TotalWeightSizeBytes = GetModelWeightSize(sentisModel); inferenceEvent.ModelHash = GetModelHash(sentisModel); return inferenceEvent; } /// /// Compute the total model weight size in bytes. /// This corresponds to the "Total weight size" display in the Sentis inspector, /// and the calculations are the same. /// /// /// The total model weight size in bytes. static long GetModelWeightSize(Model sentisModel) { long totalWeightsSizeInBytes = 0; for (var c = 0; c < sentisModel.constants.Count; c++) { totalWeightsSizeInBytes += sentisModel.constants[c].lengthBytes; } return totalWeightsSizeInBytes; } /// /// Wrapper around Hash128 that supports Append(float[], int, int) /// struct MLAgentsHash128 { private Hash128 m_Hash; public void Append(float[] values, int count) { if (values == null) { return; } // Pre-2020 versions of Unity don't have Hash128.Append() (can only hash strings and scalars) // For these versions, we'll hash element by element. #if UNITY_2020_1_OR_NEWER m_Hash.Append(values, 0, count); #else for (var i = 0; i < count; i++) { var tempHash = new Hash128(); HashUtilities.ComputeHash128(ref values[i], ref tempHash); HashUtilities.AppendHash(ref tempHash, ref m_Hash); } #endif } public void Append(string value) { var tempHash = Hash128.Compute(value); HashUtilities.AppendHash(ref tempHash, ref m_Hash); } public override string ToString() { return m_Hash.ToString(); } } /// /// Compute a hash of the model's layer data and return it as a string. /// A subset of the layer weights are used for performance. /// This increases the chance of a collision, but this should still be extremely rare. /// /// /// The hash of the model's layer data. static string GetModelHash(Model sentisModel) { var hash = new MLAgentsHash128(); foreach (var constant in sentisModel.constants) { hash.Append(constant.ToString()); } return hash.ToString(); } } }