using System; using System.Collections.Generic; using Unity.InferenceEngine; using Unity.MLAgents.Inference; using UnityEngine; using DeviceType = Unity.InferenceEngine.DeviceType; namespace Unity.MLAgents.Sensors { /// /// Allows sensors to write to both TensorProxy and float arrays/lists. /// public class ObservationWriter { IList m_Data; int m_Offset; TensorProxy m_Proxy; int m_Batch; int m_Capacity; TensorShape m_TensorShape; /// /// Initializes a new instance of the class. /// public ObservationWriter() { } /// /// Set the writer to write to an IList at the given channelOffset. /// /// Float array or list that will be written to. /// ObservationSpec of the observation to be written /// Offset from the start of the float data to write to. internal void SetTarget(IList data, ObservationSpec observationSpec, int offset) { SetTarget(data, observationSpec.Shape, offset); } /// /// Set the writer to write to an IList at the given channelOffset. /// /// Float array or list that will be written to. /// Shape of the observations to be written. /// Offset from the start of the float data to write to. internal void SetTarget(IList data, InplaceArray shape, int offset) { m_Data = data; m_Offset = offset; m_Proxy = null; m_Batch = 0; if (shape.Length == 1) { m_TensorShape = new TensorShape(m_Batch, shape[0]); } else if (shape.Length == 2) { m_TensorShape = new TensorShape(new[] { m_Batch, 1, shape[0], shape[1] }); } else { m_TensorShape = new TensorShape(m_Batch, shape[0], shape[1], shape[2]); } } /// /// Set the writer to write to a TensorProxy at the given batch and channel offset. /// /// Tensor proxy that will be written to. /// Batch index in the tensor proxy (i.e. the index of the Agent). /// Offset from the start of the channel to write to. internal void SetTarget(TensorProxy tensorProxy, int batchIndex, int channelOffset) { m_Proxy = tensorProxy; m_Batch = batchIndex; m_Offset = channelOffset; m_Data = null; m_TensorShape = m_Proxy.data.shape; m_Capacity = m_TensorShape.rank >= 2 ? m_TensorShape[1] : 0; } /// /// 1D write access at a specified index. Use AddList if possible instead. /// /// Index to write to. public float this[int index] { set { if (m_Data != null) { m_Data[index + m_Offset] = value; } else { if (index + m_Offset < 0 || index + m_Offset >= m_Capacity) return; m_Proxy.data.CompleteAllPendingOperations(); ((Tensor)m_Proxy.data)[m_Batch, index + m_Offset] = value; } } } /// /// Write access at the specified channel and width. /// /// Channels /// Width public float this[int ch, int w] { set { if (m_Data != null) { m_Data[ch * m_TensorShape[m_TensorShape.length - 1] + w] = value; } else { m_Proxy.data.CompleteAllPendingOperations(); ((Tensor)m_Proxy.data)[m_Batch, ch, w] = value; } } } /// /// 3D write access at the specified height, width, and channel. /// /// Height /// Width /// Channels public float this[int ch, int h, int w] { set { if (m_Data != null) { if (h < 0 || h >= m_TensorShape.Height()) { throw new IndexOutOfRangeException($"height value {h} must be in range [0, {m_TensorShape.Height() - 1}]"); } if (w < 0 || w >= m_TensorShape.Width()) { throw new IndexOutOfRangeException($"width value {w} must be in range [0, {m_TensorShape.Width() - 1}]"); } if (ch < 0 || ch >= m_TensorShape.Channels()) { throw new IndexOutOfRangeException($"channel value {ch} must be in range [0, {m_TensorShape.Channels() - 1}]"); } var index = m_TensorShape.Index(m_Batch, ch + m_Offset, h, w); m_Data[index] = value; } else { if (ch + m_Offset < 0 || ch + m_Offset >= m_TensorShape.Channels() || h < 0 || h >= m_TensorShape.Height() || w < 0 || w >= m_TensorShape.Width()) return; m_Proxy.data.CompleteAllPendingOperations(); ((Tensor)m_Proxy.data)[m_Batch, ch + m_Offset, h, w] = value; } } } /// /// Write the list of floats. /// /// The actual list of floats to write. /// Optional write offset to start writing from. public void AddList(IList data, int writeOffset = 0) { if (m_Data != null) { for (var index = 0; index < data.Count; index++) { var val = data[index]; m_Data[index + m_Offset + writeOffset] = val; } } else { m_Proxy.data.CompleteAllPendingOperations(); var maxCount = Math.Min(data.Count, Math.Max(0, m_Capacity - m_Offset - writeOffset)); for (var index = 0; index < maxCount; index++) { ((Tensor)m_Proxy.data)[m_Batch, index + m_Offset + writeOffset] = data[index]; } } } /// /// Write the Vector3 components. /// /// The Vector3 to be written. /// Optional write offset. public void Add(Vector3 vec, int writeOffset = 0) { if (m_Data != null) { m_Data[m_Offset + writeOffset + 0] = vec.x; m_Data[m_Offset + writeOffset + 1] = vec.y; m_Data[m_Offset + writeOffset + 2] = vec.z; } else { var start = m_Offset + writeOffset; var remaining = m_Capacity - start; if (remaining <= 0) return; m_Proxy.data.CompleteAllPendingOperations(); ((Tensor)m_Proxy.data)[m_Batch, start + 0] = vec.x; if (remaining <= 1) return; ((Tensor)m_Proxy.data)[m_Batch, start + 1] = vec.y; if (remaining <= 2) return; ((Tensor)m_Proxy.data)[m_Batch, start + 2] = vec.z; } } /// /// Write the Vector4 components. /// /// The Vector4 to be written. /// Optional write offset. public void Add(Vector4 vec, int writeOffset = 0) { if (m_Data != null) { m_Data[m_Offset + writeOffset + 0] = vec.x; m_Data[m_Offset + writeOffset + 1] = vec.y; m_Data[m_Offset + writeOffset + 2] = vec.z; m_Data[m_Offset + writeOffset + 3] = vec.w; } else { var start = m_Offset + writeOffset; var remaining = m_Capacity - start; if (remaining <= 0) return; m_Proxy.data.CompleteAllPendingOperations(); ((Tensor)m_Proxy.data)[m_Batch, start + 0] = vec.x; if (remaining <= 1) return; ((Tensor)m_Proxy.data)[m_Batch, start + 1] = vec.y; if (remaining <= 2) return; ((Tensor)m_Proxy.data)[m_Batch, start + 2] = vec.z; if (remaining <= 3) return; ((Tensor)m_Proxy.data)[m_Batch, start + 3] = vec.w; } } /// /// Write the Quaternion components. /// /// The Quaternion to be written. /// Optional write offset. public void Add(Quaternion quat, int writeOffset = 0) { if (m_Data != null) { m_Data[m_Offset + writeOffset + 0] = quat.x; m_Data[m_Offset + writeOffset + 1] = quat.y; m_Data[m_Offset + writeOffset + 2] = quat.z; m_Data[m_Offset + writeOffset + 3] = quat.w; } else { var start = m_Offset + writeOffset; var remaining = m_Capacity - start; if (remaining <= 0) return; m_Proxy.data.CompleteAllPendingOperations(); ((Tensor)m_Proxy.data)[m_Batch, start + 0] = quat.x; if (remaining <= 1) return; ((Tensor)m_Proxy.data)[m_Batch, start + 1] = quat.y; if (remaining <= 2) return; ((Tensor)m_Proxy.data)[m_Batch, start + 2] = quat.z; if (remaining <= 3) return; ((Tensor)m_Proxy.data)[m_Batch, start + 3] = quat.w; } } } /// /// Provides extension methods for the ObservationWriter. /// public static class ObservationWriterExtension { /// /// Writes a Texture2D into a ObservationWriter. /// /// /// Writer to fill with Texture data. /// /// /// The texture to be put into the tensor. /// /// /// If set to true the textures will be converted to grayscale before /// being stored in the tensor. /// /// The number of floats written public static int WriteTexture( this ObservationWriter obsWriter, Texture2D texture, bool grayScale) { if (texture.format == TextureFormat.RGB24) { return obsWriter.WriteTextureRGB24(texture, grayScale); } var width = texture.width; var height = texture.height; var texturePixels = texture.GetPixels32(); // During training, we convert from Texture to PNG before sending to the trainer, which has the // effect of flipping the image. We need another flip here at inference time to match this. for (var h = height - 1; h >= 0; h--) { for (var w = 0; w < width; w++) { var currentPixel = texturePixels[(height - h - 1) * width + w]; if (grayScale) { obsWriter[0, h, w] = (currentPixel.r + currentPixel.g + currentPixel.b) / 3f / 255.0f; } else { // For Color32, the r, g and b values are between 0 and 255. obsWriter[0, h, w] = currentPixel.r / 255.0f; obsWriter[1, h, w] = currentPixel.g / 255.0f; obsWriter[2, h, w] = currentPixel.b / 255.0f; } } } return height * width * (grayScale ? 1 : 3); } internal static int WriteTextureRGB24( this ObservationWriter obsWriter, Texture2D texture, bool grayScale ) { var width = texture.width; var height = texture.height; var rawBytes = texture.GetRawTextureData(); // During training, we convert from Texture to PNG before sending to the trainer, which has the // effect of flipping the image. We need another flip here at inference time to match this. for (var h = height - 1; h >= 0; h--) { for (var w = 0; w < width; w++) { var offset = (height - h - 1) * width + w; var r = rawBytes[3 * offset]; var g = rawBytes[3 * offset + 1]; var b = rawBytes[3 * offset + 2]; if (grayScale) { obsWriter[0, h, w] = (r + g + b) / 3f / 255.0f; } else { // For Color32, the r, g and b values are between 0 and 255. obsWriter[0, h, w] = r / 255.0f; obsWriter[1, h, w] = g / 255.0f; obsWriter[2, h, w] = b / 255.0f; } } } return height * width * (grayScale ? 1 : 3); } } }