using System; using System.Collections.Generic; using System.Linq; using Unity.InferenceEngine; using FailedCheck = Unity.MLAgents.Inference.SentisModelParamLoader.FailedCheck; namespace Unity.MLAgents.Inference { /// /// Sentis Model utility methods. /// internal class SentisModelInfo : IDisposable { public string[] InputNames; public string[] OutputNames; public int Version; public int NumVisualInputs; public int MemorySize; public bool HasContinuousOutputs; public bool HasDiscreteOutputs; public string ContinuousOutputName; public string DiscreteOutputName; public bool SupportsContinuousAndDiscrete; public int ContinuousOutputSize; public int DiscreteOutputSize; Worker m_Worker; Model m_Model; bool m_DeterministicInference; Dictionary m_ModelInputTensors; Dictionary m_ModelOutputTensors; /// /// Initializes a Sentis Model Info Object. This can be used to get information about the Sentis Model. /// /// The Sentis Model /// Whether to use deterministic inference. public SentisModelInfo(Model model, bool deterministicInference = false) { m_ModelOutputTensors = new Dictionary(); m_Model = model; m_DeterministicInference = deterministicInference; m_Worker = new Worker(m_Model, DeviceType.CPU); var inputTensors = GetInputTensors(); m_ModelInputTensors = PrepareInputs(inputTensors); foreach (var kv in m_ModelInputTensors) { m_Worker.SetInput(kv.Key, kv.Value); } m_Worker.Schedule(); CacheModelInfo(); } static Dictionary PrepareInputs(IReadOnlyList infInputs) { Dictionary inputs = new Dictionary(); inputs.Clear(); for (var i = 0; i < infInputs.Count; i++) { var inp = infInputs[i]; var newTensorShape = new TensorShape(inp.shape.Select(i => (int)i).ToArray()); inp.data = TensorUtils.CreateEmptyTensor(newTensorShape, inp.DType); TensorUtils.FillTensorBatch(inp, 0, 0f); inputs[inp.name] = inp.data; } return inputs; } /// /// Generates the Tensor inputs that are expected to be present in the Model. /// /// TensorProxy IEnumerable with the expected Tensor inputs. public IReadOnlyList GetInputTensors() { var tensors = new List(); if (m_Model == null) return tensors; foreach (var input in m_Model.inputs) { tensors.Add(new TensorProxy { name = input.name, valueType = TensorProxy.TensorType.FloatingPoint, data = null, shape = input.shape.ToArray() }); } tensors.Sort((el1, el2) => string.Compare(el1.name, el2.name, StringComparison.InvariantCulture)); return tensors; } /// /// Gets the Discrete Action Output Shape as a Tensor. /// /// `Tensor` representation of the discret Action Ouptut Shape. public Tensor GetDiscreteActionOutputShape() { return (Tensor)GetTensorByName(TensorNames.DiscreteActionOutputShape); } void CacheModelInfo() { CacheOutputTensors(); InputNames = GetInputNames(); Version = GetVersion(); NumVisualInputs = GetNumVisualInputs(); OutputNames = GetOutputNames(); MemorySize = GetMemorySize(); HasContinuousOutputs = CheckContinuousOutputs(); HasDiscreteOutputs = CheckDiscreteOutputs(); ContinuousOutputName = GetContinuousOutputName(); DiscreteOutputName = GetDiscreteOutputName(); SupportsContinuousAndDiscrete = CheckSupportsContinuousAndDiscrete(); ContinuousOutputSize = CheckContinuousOutputSize(); DiscreteOutputSize = CheckDiscreteOutputSize(); } void CacheOutputTensors() { foreach (var output in m_Model.outputs) { var outputName = output.name; Tensor outputTensor = null; m_Worker.CopyOutput(outputName, ref outputTensor); outputTensor.CompleteAllPendingOperations(); m_ModelOutputTensors.Add(outputName, outputTensor); } } Tensor GetTensorByName(string name) { try { return m_ModelOutputTensors[name]; } catch (KeyNotFoundException) { return null; } } string[] GetInputNames() { var names = new List(); if (m_Model == null) return names.ToArray(); foreach (var input in m_Model.inputs) { names.Add(input.name); } names.Sort(StringComparer.InvariantCulture); return names.ToArray(); } int GetVersion() { var version = GetTensorByNameAsInt(TensorNames.VersionNumber); return version > 0 ? version : -1; } int GetMemorySize() { return GetTensorByNameAsInt(TensorNames.MemorySize); } int GetTensorByNameAsInt(string name) { var tensor = GetTensorByName(name); var tensorAsInt = 0; if (tensor != null) tensorAsInt = (int)((Tensor)tensor)[0]; return tensorAsInt; } int GetNumVisualInputs() { var count = 0; if (m_Model == null) return count; foreach (var input in m_Model.inputs) { if (input.name.StartsWith(TensorNames.VisualObservationPlaceholderPrefix)) { count++; } } return count; } string[] GetOutputNames() { var names = new List(); if (m_Model == null) { return names.ToArray(); } if (CheckContinuousOutputs()) { names.Add(GetContinuousOutputName()); } if (CheckDiscreteOutputs()) { names.Add(GetDiscreteOutputName()); } var modelVersion = GetVersion(); var memory = GetMemorySize(); if (memory > 0) { names.Add(TensorNames.RecurrentOutput); } names.Sort(StringComparer.InvariantCulture); return names.ToArray(); } bool CheckContinuousOutputs() { if (m_Model == null) return false; if (!CheckSupportsContinuousAndDiscrete()) { return ((Tensor)GetTensorByName(TensorNames.IsContinuousControlDeprecated))[0] > 0; } bool hasStochasticOutput = !m_DeterministicInference && OutputsContainName(m_Model.outputs, TensorNames.ContinuousActionOutput); bool hasDeterministicOutput = m_DeterministicInference && OutputsContainName(m_Model.outputs, TensorNames.DeterministicContinuousActionOutput); return (hasStochasticOutput || hasDeterministicOutput) && GetTensorByNameAsInt(TensorNames.ContinuousActionOutputShape) > 0; } static bool OutputsContainName(List outputs, string name) { foreach (var output in outputs) { if (output.name.Contains(name)) { return true; } } return false; } int CheckContinuousOutputSize() { if (m_Model == null) return 0; if (!CheckSupportsContinuousAndDiscrete()) { return ((Tensor)GetTensorByName(TensorNames.IsContinuousControlDeprecated))[0] > 0 ? ((Tensor)GetTensorByName(TensorNames.ActionOutputShapeDeprecated))[0] : 0; } else { var continuousOutputShape = GetTensorByName(TensorNames.ContinuousActionOutputShape); return continuousOutputShape == null ? 0 : (int)((Tensor)continuousOutputShape)[0]; } } string GetContinuousOutputName() { if (m_Model == null) return null; if (!CheckSupportsContinuousAndDiscrete()) { return TensorNames.ActionOutputDeprecated; } return m_DeterministicInference ? TensorNames.DeterministicContinuousActionOutput : TensorNames.ContinuousActionOutput; } bool CheckDiscreteOutputs() { if (m_Model == null) return false; if (!CheckSupportsContinuousAndDiscrete()) { return ((Tensor)GetTensorByName(TensorNames.IsContinuousControlDeprecated))[0] == 0; } else { bool hasStochasticOutput = !m_DeterministicInference && OutputsContainName(m_Model.outputs, TensorNames.DiscreteActionOutput); bool hasDeterministicOutput = m_DeterministicInference && OutputsContainName(m_Model.outputs, TensorNames.DeterministicDiscreteActionOutput); return (hasStochasticOutput || hasDeterministicOutput) && CheckDiscreteOutputSize() > 0; } } int CheckDiscreteOutputSize() { if (m_Model == null) return 0; if (!CheckSupportsContinuousAndDiscrete()) { return ((Tensor)GetTensorByName(TensorNames.IsContinuousControlDeprecated))[0] > 0 ? 0 : ((Tensor)GetTensorByName(TensorNames.ActionOutputShapeDeprecated))[0]; } var discreteOutputShape = GetTensorByName(TensorNames.DiscreteActionOutputShape); if (discreteOutputShape == null) { return 0; } int result = 0; for (int i = 0; i < discreteOutputShape.Length(); i++) { result += (int)((Tensor)discreteOutputShape)[i]; } return result; } string GetDiscreteOutputName() { if (m_Model == null) return null; if (!CheckSupportsContinuousAndDiscrete()) { return TensorNames.ActionOutputDeprecated; } else { return m_DeterministicInference ? TensorNames.DeterministicDiscreteActionOutput : TensorNames.DiscreteActionOutput; } } bool CheckSupportsContinuousAndDiscrete() { return m_Model == null || OutputsContainName(m_Model.outputs, TensorNames.ContinuousActionOutput) || OutputsContainName(m_Model.outputs, TensorNames.DiscreteActionOutput); } /// /// Check if the model contains all the expected input/output tensors. /// /// Output list of failure messages /// True if the model contains all the expected tensors. /// TODO: add checks for deterministic actions /// TODO: add checks for deterministic actions public bool CheckExpectedTensors(List failedModelChecks) { // Check the presence of model version var modelApiVersionTensor = GetTensorByName(TensorNames.VersionNumber); if (modelApiVersionTensor == null) { failedModelChecks.Add( FailedCheck.Warning($"Required constant \"{TensorNames.VersionNumber}\" was not found in the model file.") ); return false; } // Check the presence of memory size var memorySizeTensor = GetTensorByName(TensorNames.MemorySize); if (memorySizeTensor == null) { failedModelChecks.Add( FailedCheck.Warning($"Required constant \"{TensorNames.MemorySize}\" was not found in the model file.") ); return false; } // Check the presence of action output tensor if (!OutputsContainName(m_Model.outputs, TensorNames.ActionOutputDeprecated) && !OutputsContainName(m_Model.outputs, TensorNames.ContinuousActionOutput) && !OutputsContainName(m_Model.outputs, TensorNames.DiscreteActionOutput) && !OutputsContainName(m_Model.outputs, TensorNames.DeterministicContinuousActionOutput) && !OutputsContainName(m_Model.outputs, TensorNames.DeterministicDiscreteActionOutput)) { failedModelChecks.Add( FailedCheck.Warning("The model does not contain any Action Output Node.") ); return false; } // Check the presence of action output shape tensor if (!CheckSupportsContinuousAndDiscrete()) { if (GetTensorByName(TensorNames.ActionOutputShapeDeprecated) == null) { failedModelChecks.Add( FailedCheck.Warning("The model does not contain any Action Output Shape Node.") ); return false; } if (GetTensorByName(TensorNames.IsContinuousControlDeprecated) == null) { failedModelChecks.Add( FailedCheck.Warning($"Required constant \"{TensorNames.IsContinuousControlDeprecated}\" was " + "not found in the model file. " + "This is only required for model that uses a deprecated model format.") ); return false; } } else { if (OutputsContainName(m_Model.outputs, TensorNames.ContinuousActionOutput)) { if (GetTensorByName(TensorNames.ContinuousActionOutputShape) == null) { failedModelChecks.Add( FailedCheck.Warning("The model uses continuous action but does not contain Continuous Action Output Shape Node.") ); return false; } else if (!CheckContinuousOutputs()) { var actionType = m_DeterministicInference ? "deterministic" : "stochastic"; var actionName = m_DeterministicInference ? "Deterministic" : ""; failedModelChecks.Add( FailedCheck.Warning($"The model uses {actionType} inference but does not contain {actionName} Continuous Action Output Tensor. Uncheck `Deterministic inference` flag..") ); return false; } } if (OutputsContainName(m_Model.outputs, TensorNames.DiscreteActionOutput)) { if (GetTensorByName(TensorNames.DiscreteActionOutputShape) == null) { failedModelChecks.Add( FailedCheck.Warning("The model uses discrete action but does not contain Discrete Action Output Shape Node.") ); return false; } else if (!CheckDiscreteOutputs()) { var actionType = m_DeterministicInference ? "deterministic" : "stochastic"; var actionName = m_DeterministicInference ? "Deterministic" : ""; failedModelChecks.Add( FailedCheck.Warning($"The model uses {actionType} inference but does not contain {actionName} Discrete Action Output Tensor. Uncheck `Deterministic inference` flag.") ); return false; } } } return true; } /// /// Disposes of the Sentis Model Info owned Tensors. /// public void Dispose() { m_Worker?.Dispose(); foreach (var key in m_ModelInputTensors.Keys) { m_ModelInputTensors[key].Dispose(); } m_ModelInputTensors.Clear(); foreach (var key in m_ModelOutputTensors.Keys) { m_ModelOutputTensors[key].Dispose(); } m_ModelOutputTensors.Clear(); } } }