using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Text;
using Unity.Collections.LowLevel.Unsafe;
using UnityEngine;
using UnityEngine.Events;
using Unity.Collections;
using OpenCVForUnity.CoreModule;
using OpenCVForUnity.ImgcodecsModule;
using OpenCVForUnity.ImgprocModule;
using OpenCVForUnity.UnityUtils;
using OpenCVForUnity.Calib3dModule;
//using NRKernal;
namespace Seengene.XDKUnityPluginCloud {
[RequireComponent(typeof(XDKCloudInterface))]
public class XDKCloudSession : MonoBehaviour {
#region Properties
[SerializeField]
private SvrManager m_SvrManager = null;
///
/// 重定位间隔时间
///
private float m_RelocationInterval = 0.1f;
private float m_RelocationIntervalStart = 0.5f;
private float m_RelocationIntervalSuccess = 1f;
///
/// 网络定位SessionID是否初始化?
///
private bool m_NetSessionIDInitialized = false;
///
/// 相机内参是否初始化?
/// 相机内参需要从JIMO机器本地的配置文件中读取
/// 每台JIMO的相机内参是不相同的
///
private bool m_CameraCalibrationInitialized = false;
///
/// 云定位接口
///
private XDKCloudInterface m_InterfaceInstance;
/// 上一次空间映射位置、旋转、缩放
private Vector3 m_PreviousPosition;
private Quaternion m_PreviousRotation;
private Vector3 m_PreviousScale;
/// 空间映射位置、旋转、缩放
private Vector3 m_SpatialPosition;
private Quaternion m_SpatialRotation;
private Vector3 m_SpatialScale;
///
/// 重定位成功次数
///
private int m_RelocSuccessCounter = 0;
///
/// 给算法更新图片次数
///
private int m_UpdataFrameCounter = 0;
private Transform m_ARCameraTransform;
private float m_ElapsedTime;
private string m_ImageSavePath;
private Int32 m_LastSeq;
private float m_Scale;
private string m_SessionID;
private RelocationResponse m_LastRelocResponse = null;
private CameraCalibration m_CameraCalibration = null;
[SerializeField]
private CameraCalibrationLoader.EyeType m_EyeType = CameraCalibrationLoader.EyeType.LeftEye;
#endregion
#region UnityAction
///
/// 更新预览图事件
///
public UnityAction UpdatePreviewImageEvent;
///
/// 空间映射事件
///
public UnityAction SpatialMappedEvent;
///
/// 更新空间特征点事件
///
public UnityAction> UpdateKeyPointsEvent;
///
/// 首次定位成功事件
///
public UnityAction OnFirstRelocationSuccessEvent;
///
/// Debug回调事件
///
public UnityAction OnUpdateDebugInfoEvent;
#endregion
#region Unity Methods
private void Awake() {
m_InterfaceInstance = GetComponent();
}
private void Start() {
m_SpatialPosition = Vector3.zero;
m_SpatialRotation = Quaternion.identity;
m_SpatialScale = Vector3.one;
m_PreviousPosition = Vector3.zero;
m_PreviousRotation = Quaternion.identity;
m_PreviousScale = Vector3.one;
m_UpdataFrameCounter = 0;
m_RelocSuccessCounter = 0;
m_ElapsedTime = 0;
// 清空缓存文文件
m_ImageSavePath = Path.Combine(Application.persistentDataPath, "seengene");
if (Directory.Exists(m_ImageSavePath))
Directory.Delete(m_ImageSavePath, true);
InintializeCameraCalibration();
InintializeNetSessionID();
}
private void OnEnable() {
m_InterfaceInstance.OnAuthorized += HandleAuthorizationResponse;
m_InterfaceInstance.OnRelocated += HandleRelocationResponse;
}
private void OnDisable() {
m_InterfaceInstance.OnAuthorized -= HandleAuthorizationResponse;
m_InterfaceInstance.OnRelocated -= HandleRelocationResponse;
}
private void Update() {
if (Input.GetKeyUp(KeyCode.Escape)) {
Resources.UnloadUnusedAssets();
GC.Collect();
Application.Quit();
}
if (m_NetSessionIDInitialized && m_CameraCalibrationInitialized) {
m_ElapsedTime += Time.deltaTime;
if (m_ElapsedTime < m_RelocationInterval) return;
m_ElapsedTime = 0f;
GetLatestFrameCameraImageAndPose();
}
}
#endregion
#region Private Methods
private void InintializeCameraCalibration() {
m_CameraCalibration = CameraCalibrationLoader.GetCameraCalibration(m_EyeType);
if (m_CameraCalibration != null) {
Debug.LogFormat("初始化相机内参成功:{0}, {1}", m_EyeType, m_CameraCalibration.ToString());
m_CameraCalibrationInitialized = true;
} else {
Debug.LogError("初始化相机内参失败!");
}
}
private void InintializeNetSessionID() {
m_RelocationInterval = m_RelocationIntervalStart;
m_InterfaceInstance.Authorize();
}
private void HandleAuthorizationResponse(AuthorizationResponse response) {
if (response.status == (int)MapQueryStatus.MAP_SUCCESS) {
if (XDKConfigs.IfLogOn) Debug.Log("Authorize Succes!");
m_SessionID = response.sessionID;
m_Scale = (float)response.scale;
m_NetSessionIDInitialized = true;
} else if (response.status == (int)MapQueryStatus.MAP_FAIL) {
Debug.LogError("Authorize Failed, Code: " + response.status);
} else {
Debug.LogError("Authorize Failed, Undefined Code: " + response.status);
}
}
private void HandleRelocationResponse(RelocationResponse response) {
switch (response.status) {
case (int)RelocalizeQueryStatus.SUCCESS:
if (XDKConfigs.IfLogOn) Debug.LogFormat("Relocation Succes! seq: {0}", response.seq);
if (response.seq < m_LastSeq) return;
m_LastSeq = response.seq;
UpdateSpatialRoot(response);
break;
case (int)RelocalizeQueryStatus.FRAME_SUCCESS_BA_FAIL:
if (XDKConfigs.IfLogOn)
Debug.LogWarningFormat("Relocation Failed, QueryStatus: 1-{0}, seq: {1}", RelocalizeQueryStatus.FRAME_SUCCESS_BA_FAIL, response.seq);
break;
case (int)RelocalizeQueryStatus.MAP_FAIL:
if (XDKConfigs.IfLogOn)
Debug.LogWarningFormat("Relocation Failed, QueryStatus: 2-{0}, seq: {1}", RelocalizeQueryStatus.MAP_FAIL, response.seq);
break;
case (int)RelocalizeQueryStatus.RELOCALIZE_FAIL:
if (XDKConfigs.IfLogOn)
Debug.LogWarningFormat("Relocation Failed, QueryStatus: 3-{0}, seq: {1}", RelocalizeQueryStatus.RELOCALIZE_FAIL, response.seq);
break;
case (int)RelocalizeQueryStatus.BA_SESSION_EXPIRED:
if (XDKConfigs.IfLogOn)
Debug.LogWarningFormat("Relocation Failed, QueryStatus: 4-{0} ,seq: {1}", RelocalizeQueryStatus.BA_SESSION_EXPIRED, response.seq);
Debug.LogError("服务器 Session 超时,尝试重新获取Session");
m_NetSessionIDInitialized = false;
m_SessionID = "";
InintializeNetSessionID();
break;
default:
break;
}
}
///
/// 更新空间映射根坐标
///
private void UpdateSpatialRoot(RelocationResponse response) {
m_RelocSuccessCounter++;
if (XDKConfigs.IfDebugOn) {
if (MusicPlayer.Instance != null) {
MusicPlayer.Instance.PlayMusic();
}
}
if (m_LastRelocResponse != null) {
float offset = 0;
offset = CheckWithProjectPointsAndroid(response);
if (XDKConfigs.IfLogOn)
Debug.LogFormat("offset(反投影偏移量): {0}", offset);
if (offset <= 10.0f) {
m_LastRelocResponse = response;
if (OnUpdateDebugInfoEvent != null) {
string strDebug = string.Format("重定位成功 {0}! offset = {1},反投影偏移量结果小于10,故放弃本次位姿校正", m_RelocSuccessCounter, offset);
OnUpdateDebugInfoEvent.Invoke(strDebug, DebugInfoType.GetSpatialMappingNative);
}
return;
}
}
m_LastRelocResponse = response;
List pose = response.transform_ltg;
if (pose.Count >= 16) {
Matrix4x4 matrix = new Matrix4x4();
matrix.m00 = (float)pose[0];
matrix.m10 = (float)pose[1];
matrix.m20 = (float)pose[2];
matrix.m30 = (float)pose[3];
matrix.m10 = (float)pose[4];
matrix.m11 = (float)pose[5];
matrix.m21 = (float)pose[6];
matrix.m31 = (float)pose[7];
matrix.m02 = (float)pose[8];
matrix.m12 = (float)pose[9];
matrix.m22 = (float)pose[10];
matrix.m32 = (float)pose[11];
matrix.m03 = (float)pose[12];
matrix.m13 = (float)pose[13];
matrix.m23 = (float)pose[14];
matrix.m33 = (float)pose[15];
matrix = matrix.transpose;
matrix.m01 = -matrix.m01;
matrix.m02 = -matrix.m02;
matrix.m10 = -matrix.m10;
matrix.m13 = -matrix.m13;
matrix.m20 = -matrix.m20;
matrix.m23 = -matrix.m23;
m_SpatialPosition = matrix.GetPosition();
m_SpatialPosition.z = -m_SpatialPosition.z;
m_SpatialRotation = matrix.GetRotation();
m_SpatialRotation.z = -m_SpatialRotation.z;
m_SpatialRotation.w = -m_SpatialRotation.w;
m_SpatialScale = new Vector3(m_Scale, m_Scale, m_Scale);
}
if (m_PreviousPosition != m_SpatialPosition || m_PreviousRotation != m_SpatialRotation || m_PreviousScale != m_SpatialScale) {
m_PreviousPosition = m_SpatialPosition;
m_PreviousRotation = m_SpatialRotation;
m_PreviousScale = m_SpatialScale;
SpatialMappedEvent?.Invoke(m_SpatialPosition, m_SpatialRotation, m_SpatialScale, m_RelocSuccessCounter);
if (m_RelocSuccessCounter == 1) {
m_RelocationInterval = m_RelocationIntervalSuccess;
if (OnFirstRelocationSuccessEvent != null) {
OnFirstRelocationSuccessEvent.Invoke();
}
}
if (XDKConfigs.IfDebugOn) {
if (response.point3d_vec != null && UpdateKeyPointsEvent != null) {
UpdateKeyPointsEvent.Invoke(response.point3d_vec);
}
if (MusicPlayer.Instance != null) {
MusicPlayer.Instance.PlayMusic();
}
string strDebug = string.Format("重定位成功 {0} : {1},{2},{3},{4}", m_RelocSuccessCounter, m_SpatialPosition.ToString("f2"),
m_SpatialRotation.ToString("f2"), m_SpatialRotation.ToString("f2"), m_SpatialScale.ToString("f2"));
if (XDKConfigs.IfLogOn) {
Debug.Log(strDebug);
}
if (OnUpdateDebugInfoEvent != null)
OnUpdateDebugInfoEvent.Invoke(strDebug, DebugInfoType.GetSpatialMappingNative);
}
}
}
private float CheckWithProjectPointsAndroid(RelocationResponse response) {
// MatOfPoint3f objectPoints
MatOfPoint3f objectPoints = new MatOfPoint3f();
List listPoint3 = new List();
for (int i = 0; i < response.point3d_vec.Count; i++) {
Vector3 v3 = response.point3d_vec[i];
Point3 p = new Point3(v3.x, v3.y, v3.z);
listPoint3.Add(p);
}
objectPoints.fromList(listPoint3);
//Debug.LogFormat("objectPoints; {0}", objectPoints.dump());
// Mat rvec(Rotation vector)
// Mat tvec(Translation vector)
Mat p1T = new Mat(4, 4, CvType.CV_32FC1);
Matrix4x4 matrixCam = response.cameraPos;
float[] arrCamTrans =
new float[] { matrixCam.m00, matrixCam.m10, matrixCam.m20, matrixCam.m30,
matrixCam.m01, matrixCam.m11, matrixCam.m21, matrixCam.m31,
matrixCam.m02, matrixCam.m12, matrixCam.m22, matrixCam.m32,
matrixCam.m03, matrixCam.m13, matrixCam.m23, matrixCam.m33};
//float[] arrCamTrans =
// new float[] { matrixCam.m00, matrixCam.m01, matrixCam.m02, matrixCam.m03,
// matrixCam.m10, matrixCam.m11, matrixCam.m12, matrixCam.m13,
// matrixCam.m20, matrixCam.m21, matrixCam.m22, matrixCam.m23,
// matrixCam.m30, matrixCam.m31, matrixCam.m32, matrixCam.m33};
p1T.put(0, 0, arrCamTrans);
//Debug.LogFormat("p1T(Before): {0}", p1T.dump());
p1T.put(0, 1, -p1T.get(0, 1)[0]);
p1T.put(0, 2, -p1T.get(0, 2)[0]);
p1T.put(1, 0, -p1T.get(1, 0)[0]);
p1T.put(1, 3, -p1T.get(1, 3)[0]);
p1T.put(2, 0, -p1T.get(2, 0)[0]);
p1T.put(2, 3, -p1T.get(2, 3)[0]);
//Debug.LogFormat("p1T(After): {0}", p1T.dump());
//Debug.LogFormat("p1T.row(3): {0}", p1T.row(3).dump());
List pose = m_LastRelocResponse.transform_ltg;
Matrix4x4 matrixMap = new Matrix4x4();
matrixMap.m00 = (float)pose[0];
matrixMap.m10 = (float)pose[1];
matrixMap.m20 = (float)pose[2];
matrixMap.m30 = (float)pose[3];
matrixMap.m01 = (float)pose[4];
matrixMap.m11 = (float)pose[5];
matrixMap.m21 = (float)pose[6];
matrixMap.m31 = (float)pose[7];
matrixMap.m02 = (float)pose[8];
matrixMap.m12 = (float)pose[9];
matrixMap.m22 = (float)pose[10];
matrixMap.m32 = (float)pose[11];
matrixMap.m03 = (float)pose[12];
matrixMap.m13 = (float)pose[13];
matrixMap.m23 = (float)pose[14];
matrixMap.m33 = (float)pose[15];
float[] arrMapTrans =
new float[] { matrixMap.m00, matrixMap.m10, matrixMap.m20, matrixMap.m30,
matrixMap.m01, matrixMap.m11, matrixMap.m21, matrixMap.m31,
matrixMap.m02, matrixMap.m12, matrixMap.m22, matrixMap.m32,
matrixMap.m03, matrixMap.m13, matrixMap.m23, matrixMap.m33};
//float[] arrMapTrans =
// new float[] { matrixMap.m00, matrixMap.m01, matrixMap.m02, matrixMap.m03,
// matrixMap.m10, matrixMap.m11, matrixMap.m12, matrixMap.m13,
// matrixMap.m20, matrixMap.m21, matrixMap.m22, matrixMap.m23,
// matrixMap.m30, matrixMap.m31, matrixMap.m32, matrixMap.m33};
Mat p2T = new Mat(4, 4, CvType.CV_32FC1);
p2T.put(0, 0, arrMapTrans);
//Debug.LogFormat("p2T: {0}", p2T.dump());
//Debug.LogFormat("p2T.row(3): {0}", p2T.row(3).dump());
Mat t12T = p1T.inv() * p2T;
//Debug.LogFormat("t12T; {0}", t12T.dump());
Mat cvMat = Mat.zeros(3, 3, t12T.type());
Mat t = Mat.zeros(3, 1, t12T.type());
t12T.rowRange(0, 3).colRange(0, 3).copyTo(cvMat);
t12T.rowRange(0, 3).col(3).copyTo(t);
Mat r = new Mat();
Calib3d.Rodrigues(cvMat, r);
//Debug.LogFormat("r: {0}", r.dump());
//Debug.LogFormat("t: {0}", t.dump());
// Mat cameraMatrix (Intrisic matrix)
Mat intrisicMat = new Mat(3, 3, CvType.CV_64FC1);
float focalLengthX = response.cameraCalibration.focal_length.x;
float focalLengthY = response.cameraCalibration.focal_length.y;
float principalPointX = response.cameraCalibration.principal_point.x;
float principalPointY = response.cameraCalibration.principal_point.y;
intrisicMat.put(0, 0,
focalLengthX, 0, principalPointX,
0, focalLengthY, principalPointY,
0, 0, 1);
//Debug.LogFormat("intrisicMat: {0}", intrisicMat.dump());
// MatOfDouble distCoeffs
MatOfDouble distCoeffs = new MatOfDouble(0, 0, 0, 0);
//Debug.LogFormat("distCoeffs: {0}", distCoeffs.dump());
// MatOfPoint2f imagePoints
MatOfPoint2f tempVecPoints2D = new MatOfPoint2f();
//Debug.LogFormat("tempVecPoints2D(before): {0}", tempVecPoints2D.dump());
Calib3d.projectPoints(objectPoints, r, t, intrisicMat, distCoeffs, tempVecPoints2D);
//Debug.LogFormat("tempVecPoints2D(after): {0}", tempVecPoints2D.dump());
List listDistance = new List();
for (int i = 0; i < tempVecPoints2D.toArray().Length; i++) {
Point p = tempVecPoints2D.toArray()[i];
Vector2 v2 = response.point2d_vec[i];
float dis = Mathf.Pow(((float)p.x - v2.x) * ((float)p.x - v2.x) + ((float)p.y - v2.y) * ((float)p.y - v2.y), 0.5f);
listDistance.Add(dis);
}
listDistance.Sort();
//Debug.LogFormat("listDistance: {0}", Tools.ListFloatToString(listDistance));
float total = 0;
int num = 0;
float last = 0;
for (int i = 0; i < listDistance.Count; i++) {
float f = listDistance[i];
if (num > listDistance.Count / 2) {
if (f > last * 3 && f > 10) {
break;
}
}
total = total + f;
last = f;
num++;
}
return total / num;
}
private void GetLatestFrameCameraImageAndPose() {
if (!SvrManager.Instance.Initialized) return;
if (!XDKConfigs.IfRelocationOn) return;
int seq = m_UpdataFrameCounter++;
int imageWidth = 640;
int imageHeight = 400;
bool outBUdate = true;
uint outCurrFrameIndex = 0;
ulong outFrameExposureNano = 0;
byte[] outFrameData = new byte[imageWidth * imageHeight];
float[] outTRDataArray = new float[7];
TextureFormat textureFormat = TextureFormat.R8;
// SvrPluginAndroid.SvrGetLatestQVRCameraFrameData(ref outBUdate, ref outCurrFrameIndex, ref outFrameExposureNano, outFrameData, outTRDataArray);
SvrPluginAndroid.SvrGetLatestQVRCameraFrameDataNoTransform(ref outBUdate, ref outCurrFrameIndex, ref outFrameExposureNano, outFrameData, outTRDataArray);
if (outBUdate) {
/// 获取相机位姿 从接口
Vector3 cameraPosition = new Vector3(outTRDataArray[4], outTRDataArray[5], outTRDataArray[6]);
Quaternion cameraRotation = new Quaternion(outTRDataArray[0], outTRDataArray[1], outTRDataArray[2], outTRDataArray[3]);
Debug.LogFormat("接口数据:position:{0},rotation:{1}", cameraPosition, cameraRotation);
Vector3 posIn = cameraPosition;
posIn.z = -posIn.z;
Quaternion rotIn = cameraRotation;
rotIn.z = -rotIn.z;
rotIn.w = -rotIn.w;
/// 获取head位姿,直接从场景中的head物体
Transform head = m_SvrManager.head;
Debug.LogFormat("Head:position:{0},rotation:{1}", head.position, head.rotation);
//Vector3 posIn_Head = head.position;
//posIn_Head.z = -posIn_Head.z;
//Quaternion rotIn_Head = head.rotation;
//rotIn_Head.z = -rotIn_Head.z;
//rotIn_Head.w = -rotIn_Head.w;
Matrix4x4 matFinal = Matrix4x4.TRS(posIn, rotIn, Vector3.one);
matFinal = matFinal.transpose;
// compress grayscale image data into JPG format.
byte[] jpegBytes;
jpegBytes = XDKTools.GetGrayTextureBytes(outFrameData, textureFormat, imageHeight, imageWidth);
if (jpegBytes == null) {
Debug.LogErrorFormat("第{0}张图片转单通道灰度图JPG格式时出错!", seq);
return;
}
if (XDKConfigs.IfDebugOn) {
string strDebug = string.Format("更新图片 {0}:", seq);
if (OnUpdateDebugInfoEvent != null)
OnUpdateDebugInfoEvent.Invoke(strDebug, DebugInfoType.UpdateFrameNative);
if (XDKConfigs.IfSaveImages) {
Texture2D texture = new Texture2D(imageWidth, imageHeight, textureFormat, false);
texture.LoadImage(jpegBytes);
texture.Apply();
UpdatePreviewImageEvent?.Invoke(texture);
SaveCameraImageToLocal(seq, jpegBytes);
SaveCameraPoseDataToLocal(seq, matFinal, m_CameraCalibration);
SaveCameraPoseOriginDataToLocal(seq, cameraPosition, cameraRotation, m_CameraCalibration);
Debug.LogFormat("保存第{0}张图片成功!", seq);
}
}
m_InterfaceInstance.Relocate(m_SessionID, seq, matFinal, m_CameraCalibration, jpegBytes);
}
}
private void SaveCameraImageToLocal(int seq, byte[] imageBuffer) {
if (!Directory.Exists(m_ImageSavePath)) Directory.CreateDirectory(m_ImageSavePath);
File.WriteAllBytes(Path.Combine(m_ImageSavePath, seq + ".jpg"), imageBuffer);
}
private void SaveCameraPoseDataToLocal(int seq, Matrix4x4 cameraPose, CameraCalibration cameraCalibration) {
if (!Directory.Exists(m_ImageSavePath)) Directory.CreateDirectory(m_ImageSavePath);
// 计算相机内参
float intrinsics_00 = cameraCalibration.focal_length.x;
float intrinsics_01 = 0;
float intrinsics_02 = cameraCalibration.principal_point.x;
float intrinsics_10 = 0;
float intrinsics_11 = cameraCalibration.focal_length.y;
float intrinsics_12 = cameraCalibration.principal_point.y;
float intrinsics_20 = 0;
float intrinsics_21 = 0;
float intrinsics_22 = 1;
string poseData = string.Format(
"{0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10} {11} {12} {13} {14} {15} {16} {17} {18} {19} {20} {21} {22} {23} {24}",
cameraPose.m00, cameraPose.m01, cameraPose.m02, cameraPose.m03,
cameraPose.m10, cameraPose.m11, cameraPose.m12, cameraPose.m13,
cameraPose.m20, cameraPose.m21, cameraPose.m22, cameraPose.m32,
cameraPose.m30, cameraPose.m31, cameraPose.m32, cameraPose.m33,
intrinsics_00, intrinsics_01, intrinsics_02,
intrinsics_10, intrinsics_11, intrinsics_12,
intrinsics_20, intrinsics_21, intrinsics_22);
File.WriteAllText(Path.Combine(m_ImageSavePath, seq + ".txt"), poseData);
}
private void SaveCameraPoseOriginDataToLocal(int seq, Vector3 cameraPosition, Quaternion cameraRotation, CameraCalibration cameraCalibration) {
if (!Directory.Exists(m_ImageSavePath)) Directory.CreateDirectory(m_ImageSavePath);
string poseDataOrigin = string.Format("{0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10} {11} {12} {13} {14} {15}",
cameraPosition.x, cameraPosition.y, cameraPosition.z, cameraRotation.x, cameraRotation.y, cameraRotation.z, cameraRotation.w,
cameraCalibration.focal_length.x, 0, cameraCalibration.principal_point.x,
0, cameraCalibration.focal_length.y, cameraCalibration.principal_point.y,
0, 0, 1);
File.WriteAllText(Path.Combine(m_ImageSavePath, seq + "-origin" + ".txt"), poseDataOrigin);
}
#endregion
#region Public Methods
///
/// 手动模拟首次定位成功
///
public void SimulateFirstRelocatSuccess() {
if (m_RelocSuccessCounter >= 1) return;
m_RelocSuccessCounter = 1;
if (OnFirstRelocationSuccessEvent != null) {
OnFirstRelocationSuccessEvent.Invoke();
}
}
#endregion
}
}