using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.InputSystem;
using System.Linq;
using UnityEngine.InputSystem.Layouts;
using System.IO;
using Ximmerse.XR.Internal.Mathmatics;
using System.Xml;
using SXR;
namespace Ximmerse.XR.InputSystems
{
///
/// Gesture type: open hand / fist
///
public enum GestureType_Fist_OpenHand
{
None = 0,
///
/// Five fingers opened.
///
Opened = 1,
///
/// Gesture of making hand as a fist
///
Fist = 2,
}
///
/// Gesture type : grisp opened / closed
///
public enum GestureType_Grisp
{
None = 0,
///
/// Grisp open
///
GrispOpen,
///
/// Grisp close
///
GraspClosed,
}
///
/// Hand tracking API expose interface for hand tracking functionality with RhinoX in XR platform.
///
public static class HandTracking
{
static I_HandleTrackingModule currentHandTrackModule = null;
static Vector3 rgbCameraAnchor = new Vector3(0.02631f, 0.05096f, 0.10121f);
static Quaternion rgbCameraQ = Quaternion.Euler(0, 0, 0);
///
/// The rgb camera anchor
///
public static Transform rgbCamAnchor;
///
/// The virtual XimXR gesture input device.
///
public static XimmerseXRGesture sGestureInputDevice
{
get; private set;
}
static bool sIsGestureDeviceLayoutRegistered = false;
static bool isDeviceCalibrated;
static string pathDeviceCalibration = "/backup/slam/device_calibration.xml";
static string pathRGB2VIO = "/backup/rgb_vio_camera_params.xml";
///
/// Read RGB calibration data.
///
static void ReadRGBCalibration()
{
try
{
//RGB to VIO R:
XmlDocument xmlDoc_rgb = new XmlDocument(); // Create an XML document object
xmlDoc_rgb.Load(pathRGB2VIO); // Load the XML document from the specified file
XmlNode RotMat = xmlDoc_rgb.GetElementsByTagName("RotMat").Item(0);
XmlNode TransVec = xmlDoc_rgb.GetElementsByTagName("TransVec").Item(0);
ParseCalibrationMatrixFromText(RotMat["data"].InnerText, TransVec["data"].InnerText, out Matrix4x4 rgb2vioR);
//VIO L to VIO R:
XmlDocument xmlDoc_deviceCali = new XmlDocument(); // Create an XML document object
xmlDoc_deviceCali.Load(pathDeviceCalibration); // Load the XML document from the specified file
XmlNode camera_TrackingB = xmlDoc_deviceCali.GetElementsByTagName("Camera").Item(1);
XmlNode rig = camera_TrackingB["Rig"];
var rigAttri = rig.Attributes;
ParseCalibrationMatrixFromText(rigAttri["rowMajorRotationMat"].InnerText, rigAttri["translation"].InnerText, out Matrix4x4 vioL2R);
//VIO L to IMU:
XmlNode SFConfig = xmlDoc_deviceCali.GetElementsByTagName("SFConfig").Item(0);
XmlNode Stateinit = SFConfig["Stateinit"];
var StateinitAttr = Stateinit.Attributes;
ParseRotationVectorMatrixFromText(StateinitAttr["ombc"].InnerText, StateinitAttr["tbc"].InnerText, out Matrix4x4 vioL2Imu);
Debug.LogFormat("Rgb to VIO L: {0}", rgb2vioR);
Debug.LogFormat("VIO L to R: {0}", vioL2R);
Debug.LogFormat("VIO L to IMU: {0}", vioL2Imu);
Matrix4x4 rgb_to_IMU_space = vioL2Imu * vioL2R.inverse * rgb2vioR;
Matrix4x4 righthand_rgb_2_imu = IMUSpaceToRightHandSpace(rgb_to_IMU_space);
var eye2imu = ParamLoaderFloat16ToMatrix((int)ParamType.Design_Eye2IMU_TransMat_OpenGL_ARRAY16);
Matrix4x4 righthand_rgb_2_eyecenter = eye2imu.inverse * righthand_rgb_2_imu; //imu2eye * rgb2imu
Matrix4x4 unity_rgb_2_eyecenter = UnityRightHandSpaceToLeftHandSpace(righthand_rgb_2_eyecenter);
Debug.LogFormat("(Unity space) RGB 2 EyeCenter : {0}, {1}", ((Vector3)unity_rgb_2_eyecenter.GetColumn(3)).ToString("F5"), unity_rgb_2_eyecenter.rotation.eulerAngles.ToString("F5"));
rgbCameraAnchor = (Vector3)unity_rgb_2_eyecenter.GetColumn(3);
rgbCameraQ = unity_rgb_2_eyecenter.rotation;
isDeviceCalibrated = true;
}
catch (System.Exception e)
{
Debug.LogException(e);
}
}
static Matrix4x4 ParamLoaderFloat16ToMatrix(int type)
{
float[] float16 = new float[16];
ParamLoader.ParamLoaderGetFloatArray(type, float16, System.Runtime.InteropServices.Marshal.SizeOf() * 16);
return new Matrix4x4()
{
m00 = float16[0],
m01 = float16[1],
m02 = float16[2],
m03 = float16[3],
m10 = float16[4],
m11 = float16[5],
m12 = float16[6],
m13 = float16[7],
m20 = float16[8],
m21 = float16[9],
m22 = float16[10],
m23 = float16[11],
m30 = float16[12],
m31 = float16[13],
m32 = float16[14],
m33 = float16[15],
};
}
static Matrix4x4 UnityRightHandSpaceToLeftHandSpace(Matrix4x4 rightHandSpace)
{
var rightHandT = (Vector3)rightHandSpace.GetColumn(3);
var rightHandQ = rightHandSpace.rotation.eulerAngles;
return Matrix4x4.TRS(new Vector3(-rightHandT.x, -rightHandT.y, rightHandT.z), Quaternion.Euler(rightHandQ.x, -rightHandQ.y, -rightHandQ.z), Vector3.one);
}
static Matrix4x4 IMUSpaceToRightHandSpace(Matrix4x4 imuSpace)
{
// P: (-0.00092, -0.06631, -0.01755), (-0.37781, -169.53320, -89.64063)
var imu_t = (Vector3)imuSpace.GetColumn(3);
Vector3 rightHand_t = new Vector3(imu_t.y, imu_t.x, -imu_t.z);
Quaternion righthand_q = Quaternion.Euler(0, 180, 90) * imuSpace.rotation * Quaternion.Euler(0, 0, 180);
return Matrix4x4.TRS(rightHand_t, righthand_q, Vector3.one);
}
private static void ParseRotationVectorMatrixFromText(string RotationVector, string TextPosition, out Matrix4x4 translate)
{
translate = Matrix4x4.identity;
int row = 0, col = 0;
try
{
var textInArray = RotationVector.Trim().Split(' ');
Vector3 rotationVector = Vector3.zero;
for (int i = 0; i < textInArray.Length; i++)
{
string t = textInArray[i];
if (float.TryParse(t, out float value))
{
rotationVector[col] = value;
col++;
}
}
Matrix3x3 qMatrix = RotationVectorToMatrix(rotationVector);
textInArray = TextPosition.Trim().Split(' ');
Vector4 column = new Vector4(0, 0, 0, 1);
col = 0;
for (int i = 0; i < textInArray.Length; i++)
{
string e = textInArray[i];
if (string.IsNullOrEmpty(e.Trim()))
{
continue;
}
column[col++] = float.Parse(e.Trim());
}
translate = Matrix3x3.ToTRS(qMatrix, (Vector3)column);
}
catch (System.Exception exc)
{
Debug.LogException(exc);
}
}
private static void ParseCalibrationMatrixFromText(string textQuaternion, string textPosition, out Matrix4x4 translate)
{
translate = Matrix4x4.identity;
int row = 0, col = 0;
try
{
var textInArray = textQuaternion.Trim().Split(' ');
for (int i = 0; i < textInArray.Length; i++)
{
string t = textInArray[i];
if (float.TryParse(t, out float value))
{
translate[row, col] = value;
col++;
if (col >= 3)
{
col = 0;
row++;
}
}
}
textInArray = textPosition.Trim().Split(' ');
Vector4 column = new Vector4(0, 0, 0, 1);
col = 0;
for (int i = 0; i < textInArray.Length; i++)
{
string e = textInArray[i];
if (string.IsNullOrEmpty(e.Trim()))
{
continue;
}
column[col++] = float.Parse(e.Trim());
}
translate.SetColumn(3, column);
}
catch (System.Exception exc)
{
Debug.LogException(exc);
}
}
///
/// Convert rotation vector in radians to a rotation matrix 3x3, using Rodrigues fomular.
///
///
/// A 3x3 matrix represents a rotation quaternion.
static Matrix3x3 RotationVectorToMatrix(Vector3 RotationVectorInRadians)
{
Vector3 r = RotationVectorInRadians.normalized;
float theta = RotationVectorInRadians.magnitude;
return Mathf.Cos(theta) * Matrix3x3.identity + (1 - Mathf.Cos(theta)) * new Matrix3x3(r, r) + Mathf.Sin(theta) * new Matrix3x3(0, -r.z, r.y, r.z, 0, -r.x, -r.y, r.x, 0);
}
///
/// Enable handle tracking
///
public static void EnableHandTracking()
{
if(!isDeviceCalibrated)
{
ReadRGBCalibration();
}
if (!rgbCamAnchor)
{
rgbCamAnchor = new GameObject("RGB Camera Anchor").transform;
rgbCamAnchor.localPosition = rgbCameraAnchor;
rgbCamAnchor.localRotation = rgbCameraQ;
rgbCamAnchor.SetParent(Camera.main.transform, false);
}
if (!sIsGestureDeviceLayoutRegistered)
{
RegisterXRGestureLayout();
}
if (currentHandTrackModule == null)
{
currentHandTrackModule = new HandTrackingT3D();
if (currentHandTrackModule.EnableModule(new InitializeHandTrackingModuleParameter()
{
TrackingAnchor = rgbCamAnchor,
}))
{
if (sGestureInputDevice == null)
{
//Adds a virtural input device for gesture input:
XimmerseXRGesture gestureInputDevice = (XimmerseXRGesture)InputSystem.AddDevice(new InputDeviceDescription
{
interfaceName = "HandState",
product = "GestureInputDevice",
});
sGestureInputDevice = gestureInputDevice;
}
}
XRManager.OnPostTrackUpdate += currentHandTrackModule.Tick;
}
}
private static void RegisterXRGestureLayout()
{
InputSystem.RegisterLayout(matches: new InputDeviceMatcher()
.WithInterface("HandState"));
sIsGestureDeviceLayoutRegistered = true;
Debug.LogFormat("Gesture input device layout has been registered.");
}
///
///
///
public static I_HandleTrackingModule handTrackModule
{
get => currentHandTrackModule;
}
///
/// Disable hand tracking.
///
public static void DisableHandTracking()
{
if (currentHandTrackModule != null)
{
currentHandTrackModule.DisableModule();
XRManager.OnPostTrackUpdate -= currentHandTrackModule.Tick;
currentHandTrackModule = null;
if (sGestureInputDevice != null)
{
InputSystem.RemoveDevice(sGestureInputDevice);
}
}
}
///
/// Is hand tracking module currently enabled and running ?
///
public static bool IsHandTrackingEnable
{
get => currentHandTrackModule != null && currentHandTrackModule.IsModuleEnabled;
}
///
/// If IsHandTrackingEnable is true, this is the hand track info.
///
public static HandTrackingInfo HandTrackingInfo
{
get
{
return currentHandTrackModule != null ? currentHandTrackModule.HandleTrackInfo : default(HandTrackingInfo);
}
}
#if UNITY_EDITOR
//[UnityEditor.MenuItem("Ximmerse/Create Gesture Device")]
private static void TestAddGestureDevice()
{
if (!sIsGestureDeviceLayoutRegistered)
{
InputSystem.RegisterLayout(matches: new InputDeviceMatcher()
.WithInterface("HandState"));
sIsGestureDeviceLayoutRegistered = true;
}
XimmerseXRGesture gestureID = (XimmerseXRGesture)InputSystem.AddDevice(new InputDeviceDescription
{
interfaceName = "HandState",
product = "GestureInputDevice",
});
Debug.Log("Gesture device is added.");
}
//[UnityEditor.MenuItem("Ximmerse/Remove Gesture Device")]
private static void RemoveDevice()
{
var gestureDevice = InputSystem.devices.FirstOrDefault(x => x is XimmerseXRGesture);
if (gestureDevice != null)
{
InputSystem.RemoveDevice(gestureDevice);
Debug.Log("Gesture device is removed.");
}
}
#endif
}
}