|
@@ -1,26 +1,22 @@
|
|
|
-
|
|
|
-Copyright (C) 2020 Immersal Ltd. All Rights Reserved.
|
|
|
+using Immersal.REST;
|
|
|
+using NRKernal;
|
|
|
+using System;
|
|
|
+using System.Threading.Tasks;
|
|
|
+using Unity.Collections.LowLevel.Unsafe;
|
|
|
+using UnityEngine;
|
|
|
|
|
|
-This file is part of the Immersal SDK.
|
|
|
|
|
|
+
|
|
|
+Copyright (C) 2022 Immersal - Part of Hexagon. All Rights Reserved.
|
|
|
+This file is part of the Immersal SDK.
|
|
|
The Immersal SDK cannot be copied, distributed, or made available to
|
|
|
third-parties for commercial purposes without written permission of Immersal Ltd.
|
|
|
-
|
|
|
Contact sdk@immersal.com for licensing requests.
|
|
|
===============================================================================*/
|
|
|
|
|
|
-using UnityEngine;
|
|
|
-using System;
|
|
|
-using System.Runtime.InteropServices;
|
|
|
-using System.Threading.Tasks;
|
|
|
-using NRKernal;
|
|
|
-using Immersal.REST;
|
|
|
-using Unity.Collections.LowLevel.Unsafe;
|
|
|
-using UnityEngine.UI;
|
|
|
-
|
|
|
namespace Immersal.AR.Nreal
|
|
|
{
|
|
|
- public class NRLocalizer : LocalizerBase
|
|
|
+ public class NRLocalizer : LocalizerBase
|
|
|
{
|
|
|
[Tooltip("Disable if you want to use RGB video capture while localizing. Also disable Multithreaded Rendering.")]
|
|
|
[SerializeField]
|
|
@@ -28,7 +24,7 @@ namespace Immersal.AR.Nreal
|
|
|
|
|
|
private static NRLocalizer instance = null;
|
|
|
|
|
|
- private CameraModelView CamTexture { get; set; }
|
|
|
+ private CameraModelView CamTexture { get; set; }
|
|
|
|
|
|
public static NRLocalizer Instance
|
|
|
{
|
|
@@ -62,16 +58,15 @@ namespace Immersal.AR.Nreal
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- public override void Start()
|
|
|
- {
|
|
|
+ public override void Start()
|
|
|
+ {
|
|
|
base.Start();
|
|
|
-
|
|
|
- m_Sdk.RegisterLocalizer(instance);
|
|
|
+ m_Sdk.RegisterLocalizer(instance);
|
|
|
|
|
|
if (m_UseYUV)
|
|
|
{
|
|
|
CamTexture = new NRRGBCamTextureYUV();
|
|
|
- (CamTexture as NRRGBCamTextureYUV).OnUpdate += OnYUVCaptureUpdate;
|
|
|
+ (CamTexture as NRRGBCamTextureYUV).OnUpdate += OnYUVCaptureUpdate;
|
|
|
}
|
|
|
else
|
|
|
{
|
|
@@ -81,15 +76,15 @@ namespace Immersal.AR.Nreal
|
|
|
|
|
|
if (autoStart)
|
|
|
{
|
|
|
- CamTexture.Play();
|
|
|
+ CamTexture.Play();
|
|
|
}
|
|
|
- }
|
|
|
+ }
|
|
|
|
|
|
- public override void StartLocalizing()
|
|
|
- {
|
|
|
- CamTexture.Play();
|
|
|
- base.StartLocalizing();
|
|
|
- }
|
|
|
+ public override void StartLocalizing()
|
|
|
+ {
|
|
|
+ CamTexture.Play();
|
|
|
+ base.StartLocalizing();
|
|
|
+ }
|
|
|
|
|
|
public override void StopLocalizing()
|
|
|
{
|
|
@@ -97,22 +92,22 @@ namespace Immersal.AR.Nreal
|
|
|
CamTexture.Stop();
|
|
|
}
|
|
|
|
|
|
- public override void Pause()
|
|
|
- {
|
|
|
- base.Pause();
|
|
|
+ public override void Pause()
|
|
|
+ {
|
|
|
+ base.Pause();
|
|
|
CamTexture.Pause();
|
|
|
- }
|
|
|
+ }
|
|
|
|
|
|
protected override void Update()
|
|
|
{
|
|
|
- isTracking = NRFrame.SessionStatus == SessionState.Running;
|
|
|
+ isTracking = NRFrame.SessionStatus == SessionState.Running;
|
|
|
|
|
|
- base.Update();
|
|
|
+ base.Update();
|
|
|
}
|
|
|
|
|
|
- public override void OnDestroy()
|
|
|
- {
|
|
|
- CamTexture.Stop();
|
|
|
+ public override void OnDestroy()
|
|
|
+ {
|
|
|
+ CamTexture.Stop();
|
|
|
|
|
|
if (m_UseYUV)
|
|
|
{
|
|
@@ -124,14 +119,14 @@ namespace Immersal.AR.Nreal
|
|
|
}
|
|
|
|
|
|
base.OnDestroy();
|
|
|
- }
|
|
|
+ }
|
|
|
|
|
|
- public override async void Localize()
|
|
|
+ public override async void Localize()
|
|
|
{
|
|
|
while (!CamTexture.DidUpdateThisFrame)
|
|
|
- {
|
|
|
- await Task.Yield();
|
|
|
- }
|
|
|
+ {
|
|
|
+ await Task.Yield();
|
|
|
+ }
|
|
|
|
|
|
if (m_PixelBuffer != IntPtr.Zero)
|
|
|
{
|
|
@@ -174,12 +169,16 @@ namespace Immersal.AR.Nreal
|
|
|
{
|
|
|
Reset();
|
|
|
}
|
|
|
-
|
|
|
+
|
|
|
lastLocalizedMapId = mapId;
|
|
|
OnMapChanged?.Invoke(mapId);
|
|
|
}
|
|
|
-
|
|
|
- rot *= Quaternion.Euler(0f, 0f, -90.0f);
|
|
|
+
|
|
|
+ rot *= Quaternion.Euler(0f, 0f, 180.0f);
|
|
|
+ pos = ARHelper.SwitchHandedness(pos);
|
|
|
+ rot = ARHelper.SwitchHandedness(rot);
|
|
|
+
|
|
|
+
|
|
|
MapOffset mo = ARSpace.mapIdToOffset[mapId];
|
|
|
|
|
|
Matrix4x4 offsetNoScale = Matrix4x4.TRS(mo.position, mo.rotation, Vector3.one);
|
|
@@ -213,12 +212,12 @@ namespace Immersal.AR.Nreal
|
|
|
base.Localize();
|
|
|
}
|
|
|
|
|
|
- public override async void LocalizeServer(SDKMapId[] mapIds)
|
|
|
- {
|
|
|
+ public override async void LocalizeServer(SDKMapId[] mapIds)
|
|
|
+ {
|
|
|
while (!CamTexture.DidUpdateThisFrame)
|
|
|
- {
|
|
|
- await Task.Yield();
|
|
|
- }
|
|
|
+ {
|
|
|
+ await Task.Yield();
|
|
|
+ }
|
|
|
|
|
|
if (m_PixelBuffer != IntPtr.Zero)
|
|
|
{
|
|
@@ -275,22 +274,25 @@ namespace Immersal.AR.Nreal
|
|
|
{
|
|
|
Reset();
|
|
|
}
|
|
|
-
|
|
|
+
|
|
|
lastLocalizedMapId = mapId;
|
|
|
OnMapChanged?.Invoke(mapId);
|
|
|
}
|
|
|
|
|
|
MapOffset mo = ARSpace.mapIdToOffset[mapId];
|
|
|
stats.localizationSuccessCount++;
|
|
|
-
|
|
|
+
|
|
|
Matrix4x4 responseMatrix = Matrix4x4.identity;
|
|
|
responseMatrix.m00 = result.r00; responseMatrix.m01 = result.r01; responseMatrix.m02 = result.r02; responseMatrix.m03 = result.px;
|
|
|
responseMatrix.m10 = result.r10; responseMatrix.m11 = result.r11; responseMatrix.m12 = result.r12; responseMatrix.m13 = result.py;
|
|
|
responseMatrix.m20 = result.r20; responseMatrix.m21 = result.r21; responseMatrix.m22 = result.r22; responseMatrix.m23 = result.pz;
|
|
|
|
|
|
- Vector3 pos = responseMatrix.GetColumn(3);
|
|
|
+ Vector3 pos = responseMatrix.GetColumn(3);
|
|
|
Quaternion rot = responseMatrix.rotation;
|
|
|
- rot *= Quaternion.Euler(0f, 0f, -90f);
|
|
|
+
|
|
|
+ rot *= Quaternion.Euler(0f, 0f, 180.0f);
|
|
|
+ pos = ARHelper.SwitchHandedness(pos);
|
|
|
+ rot = ARHelper.SwitchHandedness(rot);
|
|
|
|
|
|
Matrix4x4 offsetNoScale = Matrix4x4.TRS(mo.position, mo.rotation, Vector3.one);
|
|
|
Vector3 scaledPos = Vector3.Scale(pos, mo.scale);
|
|
@@ -324,14 +326,14 @@ namespace Immersal.AR.Nreal
|
|
|
}
|
|
|
|
|
|
base.LocalizeServer(mapIds);
|
|
|
- }
|
|
|
+ }
|
|
|
|
|
|
- public override async void LocalizeGeoPose(SDKMapId[] mapIds)
|
|
|
- {
|
|
|
+ public override async void LocalizeGeoPose(SDKMapId[] mapIds)
|
|
|
+ {
|
|
|
while (!CamTexture.DidUpdateThisFrame)
|
|
|
- {
|
|
|
- await Task.Yield();
|
|
|
- }
|
|
|
+ {
|
|
|
+ await Task.Yield();
|
|
|
+ }
|
|
|
|
|
|
if (m_PixelBuffer != IntPtr.Zero)
|
|
|
{
|
|
@@ -386,45 +388,37 @@ namespace Immersal.AR.Nreal
|
|
|
double[] ecef = new double[3];
|
|
|
double[] wgs84 = new double[3] { latitude, longitude, ellipsoidHeight };
|
|
|
Core.PosWgs84ToEcef(ecef, wgs84);
|
|
|
- Debug.Log("*************************** GeoPose Localization PosWgs84ToEcef ***************************");
|
|
|
-
|
|
|
- if (ARSpace.mapIdToMap.ContainsKey(mapId) && rot != Quaternion.identity&& rot.eulerAngles!=Vector3.zero)
|
|
|
+ if (ARSpace.mapIdToMap.ContainsKey(mapId))
|
|
|
{
|
|
|
ARMap map = ARSpace.mapIdToMap[mapId];
|
|
|
-
|
|
|
if (mapId != lastLocalizedMapId)
|
|
|
{
|
|
|
if (resetOnMapChange)
|
|
|
{
|
|
|
Reset();
|
|
|
}
|
|
|
-
|
|
|
lastLocalizedMapId = mapId;
|
|
|
OnMapChanged?.Invoke(mapId);
|
|
|
}
|
|
|
|
|
|
MapOffset mo = ARSpace.mapIdToOffset[mapId];
|
|
|
stats.localizationSuccessCount++;
|
|
|
-
|
|
|
double[] mapToEcef = map.MapToEcefGet();
|
|
|
Vector3 mapPos;
|
|
|
Quaternion mapRot;
|
|
|
- Debug.Log("*************************** GeoPose Localization PosEcefToMap ***************************");
|
|
|
Core.PosEcefToMap(out mapPos, ecef, mapToEcef);
|
|
|
- Debug.Log("*************************** GeoPose Localization RotEcefToMap ***************************");
|
|
|
Core.RotEcefToMap(out mapRot, rot, mapToEcef);
|
|
|
|
|
|
Matrix4x4 offsetNoScale = Matrix4x4.TRS(mo.position, mo.rotation, Vector3.one);
|
|
|
Vector3 scaledPos = Vector3.Scale(mapPos, mo.scale);
|
|
|
Matrix4x4 cloudSpace = offsetNoScale * Matrix4x4.TRS(scaledPos, mapRot, Vector3.one);
|
|
|
Matrix4x4 trackerSpace = Matrix4x4.TRS(camPos, camRot, Vector3.one);
|
|
|
- Matrix4x4 m = trackerSpace*(cloudSpace.inverse);
|
|
|
-
|
|
|
+ Matrix4x4 m = trackerSpace * (cloudSpace.inverse);
|
|
|
+
|
|
|
if (useFiltering)
|
|
|
mo.space.filter.RefinePose(m);
|
|
|
else
|
|
|
ARSpace.UpdateSpace(mo.space, m.GetColumn(3), m.rotation);
|
|
|
-
|
|
|
LocalizerBase.GetLocalizerPose(out lastLocalizedPose, mapId, cloudSpace.GetColumn(3), cloudSpace.rotation, m.inverse, mapToEcef);
|
|
|
map.NotifySuccessfulLocalization(mapId);
|
|
|
OnPoseFound?.Invoke(lastLocalizedPose);
|
|
@@ -445,11 +439,10 @@ namespace Immersal.AR.Nreal
|
|
|
}
|
|
|
|
|
|
base.LocalizeGeoPose(mapIds);
|
|
|
- }
|
|
|
- public RawImage rawimg;
|
|
|
- private void OnRGBCaptureUpdate(CameraTextureFrame frame)
|
|
|
- {
|
|
|
- rawimg.texture = (Texture2D)frame.texture;
|
|
|
+ }
|
|
|
+
|
|
|
+ private void OnRGBCaptureUpdate(CameraTextureFrame frame)
|
|
|
+ {
|
|
|
byte[] pixels = GetYBufFromTexture((Texture2D)frame.texture);
|
|
|
|
|
|
unsafe
|
|
@@ -459,10 +452,10 @@ namespace Immersal.AR.Nreal
|
|
|
m_PixelBuffer = (IntPtr)ptr;
|
|
|
UnsafeUtility.ReleaseGCObject(handle);
|
|
|
}
|
|
|
- }
|
|
|
+ }
|
|
|
|
|
|
- private void OnYUVCaptureUpdate(NRRGBCamTextureYUV.YUVTextureFrame frame)
|
|
|
- {
|
|
|
+ private void OnYUVCaptureUpdate(NRRGBCamTextureYUV.YUVTextureFrame frame)
|
|
|
+ {
|
|
|
if (m_PixelBuffer == IntPtr.Zero)
|
|
|
{
|
|
|
unsafe
|
|
@@ -473,7 +466,7 @@ namespace Immersal.AR.Nreal
|
|
|
UnsafeUtility.ReleaseGCObject(handle);
|
|
|
}
|
|
|
}
|
|
|
- }
|
|
|
+ }
|
|
|
|
|
|
private byte[] GetYBufFromTexture(Texture2D tex)
|
|
|
{
|
|
@@ -488,14 +481,14 @@ namespace Immersal.AR.Nreal
|
|
|
return pixels;
|
|
|
}
|
|
|
|
|
|
- private void GetIntrinsics(out Vector4 intrinsics)
|
|
|
- {
|
|
|
- intrinsics = Vector4.zero;
|
|
|
+ private void GetIntrinsics(out Vector4 intrinsics)
|
|
|
+ {
|
|
|
+ intrinsics = Vector4.zero;
|
|
|
NativeMat3f m = NRFrame.GetRGBCameraIntrinsicMatrix();
|
|
|
intrinsics.x = m.column0.X;
|
|
|
intrinsics.y = m.column1.Y;
|
|
|
intrinsics.z = m.column2.X;
|
|
|
intrinsics.w = m.column2.Y;
|
|
|
- }
|
|
|
+ }
|
|
|
}
|
|
|
-}
|
|
|
+}
|