// note:: This script is using code snippets in InputSystem. // https://github.com/Unity-Technologies/InputSystem/blob/develop/Packages/com.unity.inputsystem/InputSystem/Devices/Remote/InputRemoting.cs using System; using System.Linq; using System.Runtime.InteropServices; using System.Text; using Unity.Collections.LowLevel.Unsafe; using UnityEngine; using UnityEngine.InputSystem.Layouts; using UnityEngine.InputSystem.LowLevel; using UnityEngine.InputSystem.Utilities; using UnityEngine.InputSystem; ////TODO: show remote device IDs in the debugger ////TODO: remote timestamps need to be translated to local timestamps; doesn't make sense for remote events getting //// processed on the local timeline as is when the originating timeline may be quite different ////TODO: support actions ////TODO: support input users ////TODO: text input events ////REVIEW: it seems that the various XXXMsg struct should be public; ATM doesn't seem like working with the message interface is practical ////REVIEW: the namespacing mechanism for layouts which changes base layouts means that layouts can't be played //// around with on the editor side but will only be changed once they're updated in the player namespace Unity.RenderStreaming.InputSystem { /// /// Makes the activity and data of an InputManager observable in message form. /// /// /// Can act as both the sender and Receiver of these message so the flow is fully bidirectional, /// i.e. the InputManager on either end can mirror its layouts, devices, and events over /// to the other end. This permits streaming input not just from the player to the editor but /// also feeding input from the editor back into the player. /// /// Remoting sits entirely on top of the input system as an optional piece of functionality. /// In development players and the editor, we enable it automatically but in non-development /// players it has to be explicitly requested by the user. /// /// To see devices and input from players in the editor, open the Input Debugger through /// "Windows >> Input Debugger". /// /// /// \todo Reuse memory allocated for messages instead of allocating separately for each message. /// \todo Inteface to determine what to mirror from the local manager to the remote system. public sealed class InputRemoting : IObservable, IObserver { /// /// Enumeration of possible types of messages exchanged between two InputRemoting instances. /// public enum MessageType { Connect, Disconnect, NewLayout, NewDevice, NewEvents, RemoveDevice, RemoveLayout, ChangeUsages, StartSending, StopSending, NewOtherMessage, } /// /// A message exchanged between two InputRemoting instances. /// [StructLayout(LayoutKind.Sequential)] public struct Message { /// /// For messages coming in, numeric ID of the sender of the message. For messages /// going out, numeric ID of the targeted Receiver of the message. /// public int participantId; public MessageType type; public byte[] data; } public bool sending { get => (m_Flags & Flags.Sending) == Flags.Sending; private set { if (value) m_Flags |= Flags.Sending; else m_Flags &= ~Flags.Sending; } } static InputRemoting() { #if UNITY_EDITOR // // note: This lines are for avoiding issues when running the editor // on background. When moved the focus from the editor, input events // from another process are ignored. // Please attention behaviours are difference several platforms when // moving focus from Unity Editor. // // Additionally, The behaviour is changed Unity2021.2 later (using // InputSystem 1.1). Please see "Background behaviour". // https://docs.unity3d.com/Packages/com.unity.inputsystem@1.1/manual/Settings.html#background-behavior #if INPUTSYSTEM_1_1_OR_NEWER // todo(kazuki): #else // Make sure we're not affected by the user giving focus away from the // game view. // InputEditorUserSettings.lockInputToGameView = true; #endif #endif } internal InputRemoting(IInputManager manager, bool startSendingOnConnect = false) { if (manager == null) throw new ArgumentNullException("manager"); m_LocalManager = manager; if (startSendingOnConnect) m_Flags |= Flags.StartSendingOnConnect; //when listening for newly added layouts, must filter out ones we've added from remote } /// /// Start sending messages for data and activity in the local input system /// to observers. /// /// /// public void StartSending() { if (sending) return; ////TODO: send events in bulk rather than one-by-one m_LocalManager.onMessage += Send; m_LocalManager.onEvent += SendEvent; m_LocalManager.onDeviceChange += SendDeviceChange; m_LocalManager.onLayoutChange += SendLayoutChange; sending = true; SendInitialMessages(); } public void StopSending() { if (!sending) return; m_LocalManager.onMessage -= Send; m_LocalManager.onEvent -= SendEvent; m_LocalManager.onDeviceChange -= SendDeviceChange; m_LocalManager.onLayoutChange -= SendLayoutChange; sending = false; } //RTC 接收到的Message(接收端) void IObserver.OnNext(Message msg) { switch (msg.type) { case MessageType.Connect: ConnectMsg.Process(this); break; case MessageType.Disconnect: DisconnectMsg.Process(this, msg); break; case MessageType.NewLayout: NewLayoutMsg.Process(this, msg); break; case MessageType.RemoveLayout: RemoveLayoutMsg.Process(this, msg); break; case MessageType.NewDevice: NewDeviceMsg.Process(this, msg); break; case MessageType.NewEvents: NewEventsMsg.Process(this, msg); break; case MessageType.ChangeUsages: ChangeUsageMsg.Process(this, msg); break; case MessageType.RemoveDevice: RemoveDeviceMsg.Process(this, msg); break; case MessageType.StartSending: StartSendingMsg.Process(this); break; case MessageType.StopSending: StopSendingMsg.Process(this); break; case MessageType.NewOtherMessage: Debug.Log("NewOtherMessage===>"+ Encoding.UTF8.GetString(msg.data)); break; } } void IObserver.OnError(Exception error) { } void IObserver.OnCompleted() { } public IDisposable Subscribe(IObserver observer) { if (observer == null) throw new ArgumentNullException("observer"); var subscriber = new Subscriber { owner = this, observer = observer }; ArrayHelpers.Append(ref m_Subscribers, subscriber); return subscriber; } private void SendInitialMessages() { SendAllGeneratedLayouts(); SendAllDevices(); } private void SendAllGeneratedLayouts() { // todo(kazuki):: // layputBuilders property is not published from InputSystem // //foreach (var entry in m_LocalManager.m_Layouts.layoutBuilders) // SendLayout(entry.Key); foreach (var layout in m_LocalManager.layouts) SendLayout(layout); } private void SendLayout(string layoutName) { if (m_Subscribers == null) return; var message = NewLayoutMsg.Create(this, layoutName); if (message != null) Send(message.Value); } private void SendAllDevices() { var devices = m_LocalManager.devices; foreach (var device in devices) SendDevice(device); } private void SendDevice(InputDevice device) { if (m_Subscribers == null) return; // Don't mirror remote devices to other remotes. if (device.remote) return; var newDeviceMessage = NewDeviceMsg.Create(device); Send(newDeviceMessage); // Send current state. We do this here in this case as the device // may have been added some time ago and thus have already received events. var stateEventMessage = NewEventsMsg.CreateStateEvent(device); Send(stateEventMessage); } private unsafe void SendEvent(InputEventPtr eventPtr, InputDevice device) { if (m_Subscribers == null) return; ////REVIEW: we probably want to have better control over this and allow producing local events //// against remote devices which *are* indeed sent across the wire // Don't send events that came in from remote devices. if (device != null && device.remote) return; var message = NewEventsMsg.Create(eventPtr.data, 1); Debug.Log("SendEvent==>" + Encoding.UTF8.GetString(message.data)); Send(message); } private void SendDeviceChange(InputDevice device, InputDeviceChange change) { if (m_Subscribers == null) return; // Don't mirror remoted devices to other remotes. if (device.remote) return; Message msg; switch (change) { case InputDeviceChange.Added: msg = NewDeviceMsg.Create(device); break; case InputDeviceChange.Removed: msg = RemoveDeviceMsg.Create(device); break; case InputDeviceChange.UsageChanged: msg = ChangeUsageMsg.Create(device); break; default: return; } Send(msg); } private void SendLayoutChange(string layout, InputControlLayoutChange change) { if (m_Subscribers == null) return; Message msg; switch (change) { case InputControlLayoutChange.Added: case InputControlLayoutChange.Replaced: var message = NewLayoutMsg.Create(this, layout); if (message == null) return; msg = message.Value; break; case InputControlLayoutChange.Removed: msg = RemoveLayoutMsg.Create(layout); break; default: return; } Send(msg); } public void Send(Message msg) { foreach (var subscriber in m_Subscribers) subscriber.observer.OnNext(msg); } ////TODO: with C#7 this should be a ref return private int FindOrCreateSenderRecord(int senderId) { // Try to find existing. if (m_Senders != null) { var senderCount = m_Senders.Length; for (var i = 0; i < senderCount; ++i) if (m_Senders[i].senderId == senderId) return i; } // Create new. var sender = new RemoteSender { senderId = senderId, }; return ArrayHelpers.Append(ref m_Senders, sender); } private int FindLocalDeviceId(int remoteDeviceId, int senderIndex) { var localDevices = m_Senders[senderIndex].devices; if (localDevices != null) { var numLocalDevices = localDevices.Length; for (var i = 0; i < numLocalDevices; ++i) { if (localDevices[i].remoteId == remoteDeviceId) return localDevices[i].localId; } } return InputDevice.InvalidDeviceId; } private InputDevice TryGetDeviceByRemoteId(int remoteDeviceId, int senderIndex) { var localId = FindLocalDeviceId(remoteDeviceId, senderIndex); return m_LocalManager.GetDeviceById(localId); } private Flags m_Flags; private IInputManager m_LocalManager; // Input system we mirror input from and to. private Subscriber[] m_Subscribers; // Receivers we send input to. private RemoteSender[] m_Senders; // Senders we receive input from. [Flags] private enum Flags { Sending = 1 << 0, StartSendingOnConnect = 1 << 1 } // Data we keep about a unique sender that we receive input data // from. We keep track of the layouts and devices we added to // the local system. [Serializable] internal struct RemoteSender { public int senderId; public string[] layouts; public RemoteInputDevice[] devices; } [Serializable] internal struct RemoteInputDevice { public int remoteId; // Device ID used by sender. public int localId; // Device ID used by us in local system. public InputDeviceDescription description; } internal class Subscriber : IDisposable { public InputRemoting owner; public IObserver observer; public void Dispose() { ArrayHelpers.Erase(ref owner.m_Subscribers, this); } } private static class ConnectMsg { public static void Process(InputRemoting Receiver) { if (Receiver.sending) { Receiver.SendAllDevices(); } else if ((Receiver.m_Flags & Flags.StartSendingOnConnect) == Flags.StartSendingOnConnect) Receiver.StartSending(); } } private static class StartSendingMsg { public static void Process(InputRemoting Receiver) { Receiver.StartSending(); } } private static class StopSendingMsg { public static void Process(InputRemoting Receiver) { Receiver.StopSending(); } } public void RemoveRemoteDevices(int participantId) { var senderIndex = FindOrCreateSenderRecord(participantId); // Remove devices added by remote. var devices = m_Senders[senderIndex].devices; if (devices != null) { foreach (var remoteDevice in devices) { var device = m_LocalManager.GetDeviceById(remoteDevice.localId); if (device != null) m_LocalManager.RemoveDevice(device); } } ////TODO: remove layouts added by remote ArrayHelpers.EraseAt(ref m_Senders, senderIndex); } private static class DisconnectMsg { public static void Process(InputRemoting Receiver, Message msg) { Receiver.RemoveRemoteDevices(msg.participantId); Receiver.StopSending(); } } // Tell remote input system that there's a new layout. private static class NewLayoutMsg { [Serializable] public struct Data { public string name; public string layoutJson; public bool isOverride; } public static Message? Create(InputRemoting sender, string layoutName) { // Try to load the layout. Ignore the layout if it couldn't // be loaded. InputControlLayout layout; try { layout = sender.m_LocalManager.LoadLayout(new InternedString(layoutName)); if (layout == null) { Debug.Log(string.Format( "Could not find layout '{0}' meant to be sent through remote connection; this should not happen", layoutName)); return null; } } catch (Exception exception) { Debug.Log(string.Format( "Could not load layout '{0}'; not sending to remote listeners (exception: {1})", layoutName, exception)); return null; } var data = new Data { name = layoutName, layoutJson = layout.ToJson(), isOverride = layout.isOverride }; return new Message { type = MessageType.NewLayout, data = SerializeData(data) }; } public static void Process(InputRemoting Receiver, Message msg) { var data = DeserializeData(msg.data); var senderIndex = Receiver.FindOrCreateSenderRecord(msg.participantId); var internedLayoutName = new InternedString(data.name); Receiver.m_LocalManager.RegisterControlLayout(data.layoutJson, data.name, data.isOverride); ArrayHelpers.Append(ref Receiver.m_Senders[senderIndex].layouts, internedLayoutName); } } private static class RemoveLayoutMsg { public static Message Create(string layoutName) { var bytes = Encoding.UTF8.GetBytes(layoutName); return new Message { type = MessageType.RemoveLayout, data = bytes }; } public static void Process(InputRemoting Receiver, Message msg) { ////REVIEW: we probably don't want to do this blindly var layoutName = Encoding.UTF8.GetString(msg.data); Receiver.m_LocalManager.RemoveLayout(layoutName); } } // Tell remote input system that there's a new device. private static class NewDeviceMsg { [Serializable] public struct Data { public string name; public string layout; public int deviceId; public string[] usages; public InputDeviceDescription description; } public static Message Create(InputDevice device) { Debug.Assert(!device.remote, "Device being sent to remotes should be a local device, not a remote one"); var data = new Data { name = device.name, layout = device.layout, deviceId = device.deviceId, description = device.description, usages = device.usages.Select(x => x.ToString()).ToArray() }; var json = JsonUtility.ToJson(data); var bytes = Encoding.UTF8.GetBytes(json); return new Message { type = MessageType.NewDevice, data = bytes }; } public static void Process(InputRemoting Receiver, Message msg) { var senderIndex = Receiver.FindOrCreateSenderRecord(msg.participantId); var data = DeserializeData(msg.data); // Make sure we haven't already seen the device. var devices = Receiver.m_Senders[senderIndex].devices; if (devices != null) { foreach (var entry in devices) if (entry.remoteId == data.deviceId) { Debug.LogError(string.Format( "Already received device with id {0} (layout '{1}', description '{3}) from remote {2}", data.deviceId, data.layout, msg.participantId, data.description)); return; } } // Create device. InputDevice device; try { ////REVIEW: this gives remote devices names the same way that local devices receive them; should we make remote status visible in the name? device = Receiver.m_LocalManager.AddDevice(data.layout, data.name); } catch (Exception exception) { Debug.LogError( $"Could not create remote device '{data.description}' with layout '{data.layout}' locally (exception: {exception})"); return; } // todo(kazuki)::Avoid to use reflection // device.m_ParticipantId = msg.participantId; device.SetParticipantId(msg.participantId); // todo(kazuki)::Avoid to use reflection // device.m_Description = data.description; // device.m_DeviceFlags |= InputDevice.DeviceFlags.Remote; device.SetDescription(data.description); var deviceFlagsRemote = 1 << 3; device.SetDeviceFlags(device.GetDeviceFlags() | deviceFlagsRemote); if(data.usages != null) foreach (var usage in data.usages) Receiver.m_LocalManager.AddDeviceUsage(device, usage); // Remember it. var record = new RemoteInputDevice { remoteId = data.deviceId, localId = device.deviceId, description = data.description }; ArrayHelpers.Append(ref Receiver.m_Senders[senderIndex].devices, record); } } // Tell remote system there's new input events. private static class NewEventsMsg { // todo(kazuki):: not found DeviceResetEvent //public static unsafe Message CreateResetEvent(InputDevice device, bool isHardReset) //{ // var resetEvent = DeviceResetEvent.Create(device.deviceId, isHardReset); // return Create((InputEvent*)UnsafeUtility.AddressOf(ref resetEvent), 1); //} public static unsafe Message CreateStateEvent(InputDevice device) { using (StateEvent.From(device, out var eventPtr)) return Create(eventPtr.data, 1); } public static unsafe Message Create(InputEvent* events, int eventCount) { // Find total size of event buffer we need. var totalSize = 0u; var eventPtr = new InputEventPtr(events); for (var i = 0; i < eventCount; ++i, eventPtr = eventPtr.Next()) { totalSize += eventPtr.sizeInBytes; } // Copy event data to buffer. Would be nice if we didn't have to do that // but unfortunately we need a byte[] and can't just pass the 'events' IntPtr // directly. var data = new byte[totalSize]; fixed (byte* dataPtr = data) { UnsafeUtility.MemCpy(dataPtr, events, totalSize); } // Done. return new Message { type = MessageType.NewEvents, data = data }; } public static unsafe void Process(InputRemoting Receiver, Message msg) { var manager = Receiver.m_LocalManager; fixed (byte* dataPtr = msg.data) { var dataEndPtr = new IntPtr(dataPtr + msg.data.Length); var eventCount = 0; var eventPtr = new InputEventPtr((InputEvent*)dataPtr); var senderIndex = Receiver.FindOrCreateSenderRecord(msg.participantId); while ((Int64)eventPtr.data < dataEndPtr.ToInt64()) { // Patch up device ID to refer to local device and send event. var remoteDeviceId = eventPtr.deviceId; var localDeviceId = Receiver.FindLocalDeviceId(remoteDeviceId, senderIndex); eventPtr.deviceId = localDeviceId; if (localDeviceId != InputDevice.InvalidDeviceId) { ////TODO: add API to send events in bulk rather than one by one manager.QueueEvent(eventPtr); } ++eventCount; eventPtr = eventPtr.Next(); } } } } private static class ChangeUsageMsg { [Serializable] public struct Data { public int deviceId; public string[] usages; } public static Message Create(InputDevice device) { var data = new Data { deviceId = device.deviceId, usages = device.usages.Select(x => x.ToString()).ToArray() }; return new Message { type = MessageType.ChangeUsages, data = SerializeData(data) }; } public static void Process(InputRemoting Receiver, Message msg) { var senderIndex = Receiver.FindOrCreateSenderRecord(msg.participantId); var data = DeserializeData(msg.data); var device = Receiver.TryGetDeviceByRemoteId(data.deviceId, senderIndex); if (device != null) { foreach (var deviceUsage in device.usages) { if (!data.usages.Contains(deviceUsage)) Receiver.m_LocalManager.RemoveDeviceUsage(device, new InternedString(deviceUsage)); } if (data.usages.Length == 1) Receiver.m_LocalManager.AddDeviceUsage(device, new InternedString(data.usages[0])); foreach (var dataUsage in data.usages) { var internedDataUsage = new InternedString(dataUsage); if (!device.usages.Contains(internedDataUsage)) Receiver.m_LocalManager.AddDeviceUsage(device, new InternedString(dataUsage)); } } } } private static class RemoveDeviceMsg { public static Message Create(InputDevice device) { return new Message { type = MessageType.RemoveDevice, data = BitConverter.GetBytes(device.deviceId) }; } public static void Process(InputRemoting Receiver, Message msg) { var senderIndex = Receiver.FindOrCreateSenderRecord(msg.participantId); var remoteDeviceId = BitConverter.ToInt32(msg.data, 0); var device = Receiver.TryGetDeviceByRemoteId(remoteDeviceId, senderIndex); if (device != null) Receiver.m_LocalManager.RemoveDevice(device); } } private static byte[] SerializeData(TData data) { var json = JsonUtility.ToJson(data); return Encoding.UTF8.GetBytes(json); } private static TData DeserializeData(byte[] data) { var json = Encoding.UTF8.GetString(data); return JsonUtility.FromJson(json); } #if UNITY_EDITOR || DEVELOPMENT_BUILD // State we want to take across domain reloads. We can only take some of the // state across. Subscriptions will be lost and have to be manually restored. [Serializable] internal struct SerializedState { public int senderId; public RemoteSender[] senders; // We can't take these across domain reloads but we want to take them across // InputSystem.Save/Restore. [NonSerialized] public Subscriber[] subscribers; } internal SerializedState SaveState() { return new SerializedState { senders = m_Senders, subscribers = m_Subscribers }; } internal void RestoreState(SerializedState state, IInputManager manager) { m_LocalManager = manager; m_Senders = state.senders; } #endif } } // #endif