123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651 |
- using System.Collections;
- using System.Collections.Generic;
- using Assets.NXR.Scripts.Slam;
- using UnityEngine;
- using Nxr.Internal;
- using NibiruTask;
- using System;
- using System.Runtime.InteropServices;
- using System.Threading;
- public class SlamApi : MonoBehaviour
- {
- enum TestMode
- {
- TM_Map,
- TM_Camera,
- TM_PlaneDetection
- }
- public struct MapStatus
- {
- public bool refresh;
- public int status;
- public int quality;
- public float percent;
- }
- public TextMesh CameraTextTip;
- public TextMesh MapTextTip;
- public TextMesh PlaneDetectionTip;
- public MeshRenderer ThermalCameraPreview;
- public MeshRenderer RgbCameraPreview;
- public MeshRenderer TofCameraPreview;
- public MeshRenderer FishEyeLeftCameraPreview;
- public MeshRenderer FishEyeRightCameraPreview;
- Texture2D RgbCamTexture2D_RGB;
- IntPtr[] RgbCameraDataPtrArray = new IntPtr[2] {IntPtr.Zero, IntPtr.Zero};
- public GameObject MapTestObj;
- public GameObject CameraTestObj;
- public GameObject PlaneDetectionTestObj;
- Texture2D ThermalCamTexture2D_Y;
- Texture2D ThermalCamTexture2D_U;
- Texture2D ThermalCamTexture2D_V;
- Texture2D TofCamTexture2D;
- Texture2D FishEyeLeftCamTexture2D;
- Texture2D FishEyeRightCamTexture2D;
- Color[] FishEyeLeftColors;
- Color[] FishEyeRightColors;
- TestMode CurTestMode = TestMode.TM_Map;
- Dictionary<NxrSlam.RawDataType, Queue<NxrSlam.NXRStreamData>> StreamDataDict =
- new Dictionary<NxrSlam.RawDataType, Queue<NxrSlam.NXRStreamData>>();
- Queue<NxrSlam.NXRPlaneData> PlaneDataQueue = new Queue<NxrSlam.NXRPlaneData>();
- Dictionary<int, NxrSlamPlane> PlaneDict = new Dictionary<int, NxrSlamPlane>();
- MapStatus mapStatus;
- public GameObject MapObjects;
- public void ShowMapTest()
- {
- CurTestMode = TestMode.TM_Map;
- MapTestObj.SetActive(true);
- CameraTestObj.SetActive(false);
- PlaneDetectionTestObj.SetActive(false);
- }
- public void ShowCameraTest()
- {
- CurTestMode = TestMode.TM_Camera;
- MapTestObj.SetActive(false);
- CameraTestObj.SetActive(true);
- PlaneDetectionTestObj.SetActive(false);
- }
- public void ShowPlaneDetectionTest()
- {
- CurTestMode = TestMode.TM_PlaneDetection;
- MapTestObj.SetActive(false);
- CameraTestObj.SetActive(false);
- PlaneDetectionTestObj.SetActive(true);
- }
- public void StartPlaneDetection()
- {
- PlaneDetectionTip.text = "等待检测结果...";
- Debug.Log("StartPlaneDetection");
- NxrSlam.Instance.StartPlaneDetection();
- }
- public void StopPlaneDetection()
- {
- PlaneDetectionTip.text = "请开始平面检测...";
- Debug.Log("StopPlaneDetection");
- NxrSlam.Instance.StopPlaneDetection();
- }
- string MapBinPath;
- public void BuildMap()
- {
- MapBinPath = Nxr.Internal.NxrViewer.Instance.GetStoragePath() + "/unity_map.bin";
- Debug.Log("BuildMap." + MapBinPath);
- NxrSlam.Instance.StartBuildMap();
- }
- public void FinishMap()
- {
- Debug.Log("FinishMap");
-
- Vector3 orginalVec = Camera.main.transform.position;
- PlayerPrefs.SetFloat("map_original_x", orginalVec.x);
- PlayerPrefs.SetFloat("map_original_y", orginalVec.y);
- PlayerPrefs.SetFloat("map_original_z", orginalVec.z);
- Debug.Log("Save Map Original Position : " + orginalVec.x + "," + orginalVec.y + "," + orginalVec.z);
- PlayerPrefs.Save();
- NxrSlam.Instance.StopBuildMap(MapBinPath);
- }
- public void ReadMap()
- {
- MapBinPath = Nxr.Internal.NxrViewer.Instance.GetStoragePath() + "/unity_map.bin";
- Debug.Log("ReadMap." + MapBinPath);
-
- NxrSlam.Instance.StartSlamWithMap(MapBinPath);
- }
- bool MapObjectsShowing = false;
- public void PutObjectInMap()
- {
- MapObjectsShowing = !MapObjectsShowing;
- Debug.Log("MapObjectsShowing." + MapObjectsShowing);
- MapObjects.SetActive(MapObjectsShowing);
- Vector3 mapOriginalVec;
- mapOriginalVec.x = PlayerPrefs.GetFloat("map_original_x");
- mapOriginalVec.y = PlayerPrefs.GetFloat("map_original_y");
- mapOriginalVec.z = PlayerPrefs.GetFloat("map_original_z");
- Debug.Log("Load Map Original Position : " + mapOriginalVec.x + "," + mapOriginalVec.y + "," + mapOriginalVec.z);
- MapObjects.transform.position = mapOriginalVec;
- }
- public void THERMALCamera()
- {
- Debug.Log("THERMALCamera");
- NxrSlam.Instance.StartStreaming(NxrSlam.RawDataType.THERMAL);
- }
- public void RGBCamera()
- {
- Debug.Log("RGBCamera");
- NxrSlam.Instance.StartStreaming(NxrSlam.RawDataType.RGB);
- }
- public void TOFCamera()
- {
- Debug.Log("TOFCamera");
- NxrSlam.Instance.SetTofStreamMode(NxrSlam.TOFStreamMode.ONLY_DEPTH);
- NxrSlam.Instance.StartStreaming(NxrSlam.RawDataType.TOF);
- }
- public void FISHEYECamera()
- {
- Debug.Log("FISHEYECamera");
- NxrSlam.Instance.StartStreaming(NxrSlam.RawDataType.FISHEYE);
- }
- public void StopCamera()
- {
- Debug.Log("StopCamera");
- NxrSlam.Instance.StopStreaming();
- }
-
- void Start()
- {
- NxrSlam.Instance.Init();
- NxrSlam.Instance.EnableNativeLog(false);
- NxrSlam.Instance.MapStatusAndQualityEvent = OnMapStatusAndQualityEvent;
- NxrSlam.Instance.MapPercentEvent = OnMapPercent;
- NxrSlam.Instance.StreamDataEvent = OnStreamDataEvent;
- NxrSlam.Instance.PlaneDetectionEvent = OnPlaneDetectionEvent;
- mapStatus.refresh = false;
- StreamDataDict[NxrSlam.RawDataType.EYETRACKING] = new Queue<NxrSlam.NXRStreamData>();
- StreamDataDict[NxrSlam.RawDataType.RGB] = new Queue<NxrSlam.NXRStreamData>();
- StreamDataDict[NxrSlam.RawDataType.TOF] = new Queue<NxrSlam.NXRStreamData>();
- StreamDataDict[NxrSlam.RawDataType.FISHEYE] = new Queue<NxrSlam.NXRStreamData>();
- StreamDataDict[NxrSlam.RawDataType.THERMAL] = new Queue<NxrSlam.NXRStreamData>();
- Thread thread = new Thread(ShowFishEyeImage);
- thread.Start();
- }
- void OnPlaneDetectionEvent(NxrSlam.NXRPlaneData data)
- {
-
-
-
-
-
-
- while (PlaneDataQueue.Count > 10)
- {
- PlaneDataQueue.Dequeue();
- }
- PlaneDataQueue.Enqueue(data);
- }
- void OnStreamDataEvent(NxrSlam.NXRStreamData data)
- {
-
-
-
-
-
- if (data.size > 0 || data.leftSize > 0 || data.rightSize > 0)
- {
- if (StreamDataDict[data.rawDataType].Count > 30)
- {
- StreamDataDict[data.rawDataType].Clear();
- }
- if (data.rawDataType == NxrSlam.RawDataType.RGB)
- {
- lock (rgbCameraDataLock)
- {
- StreamDataDict[data.rawDataType].Enqueue(data);
- }
- RgbFpsCheck();
- }
- else
- {
- StreamDataDict[data.rawDataType].Enqueue(data);
- }
- }
- }
- void OnMapStatusAndQualityEvent(int status, int quality)
- {
- mapStatus.refresh = true;
- mapStatus.status = status;
- mapStatus.quality = quality;
- }
- void OnMapPercent(float percentage)
- {
- mapStatus.refresh = true;
- mapStatus.percent = percentage;
- }
- public bool TestPoints = false;
- DateTime RgbFpsPrintTime = DateTime.Now;
- private int RgbFpsCount = 0;
- private void RgbFpsCheck()
- {
- TimeSpan diffSpan = new TimeSpan(DateTime.Now.Ticks - RgbFpsPrintTime.Ticks);
- if (diffSpan.TotalMilliseconds > 900)
- {
- RgbFpsPrintTime = DateTime.Now;
- Debug.LogError("Rgb Camera Frame Rate : " + RgbFpsCount);
- RgbFpsCount = 0;
- }
- RgbFpsCount++;
- }
- private readonly object rgbCameraDataLock = new object();
- void OnEnable()
- {
- StartCoroutine("EndOfFrame");
- }
- void OnDisable()
- {
- StopCoroutine("EndOfFrame");
- }
- IEnumerator EndOfFrame()
- {
- while (true)
- {
- yield return new WaitForEndOfFrame();
-
-
- }
- }
- private void ProcessRGBCameraData()
- {
- Queue<NxrSlam.NXRStreamData> rgbQueue = null;
- lock (rgbCameraDataLock)
- {
- rgbQueue = StreamDataDict[NxrSlam.RawDataType.RGB];
- }
- if (rgbQueue != null && rgbQueue.Count > 0)
- {
-
- NxrSlam.NXRStreamData data = rgbQueue.ToArray()[rgbQueue.Count - 1];
- if (data.size > 0)
- {
- if (RgbCamTexture2D_RGB == null)
- {
- Debug.Log("Create Texture2D : data=" + data.size + "," + data.width + "," + data.height + "," +
- data.data.Length);
- RgbCamTexture2D_RGB = new Texture2D(data.width, data.height, TextureFormat.RGB24, false);
- RgbCameraPreview.material.SetTexture("_MainTex", RgbCamTexture2D_RGB);
- }
-
- RgbCamTexture2D_RGB.LoadRawTextureData(data.data);
- RgbCamTexture2D_RGB.Apply();
- }
- }
- if (NxrSlam.Instance.IsRGBCameraSteaming)
- {
- int size = NxrSlam.getNxrStreamSize((int) NxrSlam.RawDataType.RGB);
- if (size > 0 && RgbCameraDataPtrArray[0] == IntPtr.Zero)
- {
- RgbCameraDataPtrArray[0] = Marshal.AllocHGlobal(size);
- RgbCameraDataPtrArray[1] = Marshal.AllocHGlobal(size);
- bool success = NxrSlam.initPtr((int) NxrSlam.RawDataType.RGB, RgbCameraDataPtrArray[0],
- RgbCameraDataPtrArray[1]);
- if (!success)
- {
- Debug.LogError("NxrSlam.initPtr Failed !!!");
- }
- }
- if (size > 0)
- {
- int width = 0;
- int height = 0;
-
- int validPtrIndex = NxrSlam.getNxrStreamData((int) NxrSlam.RawDataType.RGB, ref width, ref height);
-
-
- if (width > 0 && height > 0 && RgbCamTexture2D_RGB == null)
- {
- RgbCamTexture2D_RGB = new Texture2D(width, height, TextureFormat.RGB24, false);
- RgbCameraPreview.material.SetTexture("_MainTex", RgbCamTexture2D_RGB);
- }
- if (validPtrIndex >= 0 && RgbCamTexture2D_RGB != null)
- {
- RgbCamTexture2D_RGB.LoadRawTextureData(RgbCameraDataPtrArray[validPtrIndex], size);
- RgbCamTexture2D_RGB.Apply();
- NxrSlam.unlockStream(validPtrIndex);
- }
- }
- }
- }
-
- void Update()
- {
- if (TestPoints)
- {
- TestPoints = false;
- DebugPoints();
- }
- if (mapStatus.refresh)
- {
- mapStatus.refresh = false;
- MapTextTip.text = "地图质量:" + mapStatus.quality + ",地图匹配度(0~100):" + (int) (mapStatus.percent * 100);
- }
- if (NxrInput.GetKeyUp(CKeyEvent.KEYCODE_BACK) ||
- NxrInput.GetControllerKeyUp(CKeyEvent.KEYCODE_CONTROLLER_MENU) ||
- NxrInput.GetControllerKeyUp(CKeyEvent.KEYCODE_CONTROLLER_MENU,
- InteractionManager.NACTION_HAND_TYPE.HAND_LEFT) ||
- NxrInput.GetControllerKeyUp(CKeyEvent.KEYCODE_CONTROLLER_MENU,
- InteractionManager.NACTION_HAND_TYPE.HAND_RIGHT))
- {
-
- if (Application.isMobilePlatform) Application.Quit();
- }
-
- Queue<NxrSlam.NXRStreamData> thermalQueue = StreamDataDict[NxrSlam.RawDataType.THERMAL];
- if (thermalQueue.Count > 0)
- {
- NxrSlam.NXRStreamData data = thermalQueue.Dequeue();
- if (data.size > 0)
- {
- if (ThermalCamTexture2D_Y == null)
- {
-
- ThermalCamTexture2D_Y = new Texture2D(data.width, data.height, TextureFormat.Alpha8, false);
- ThermalCamTexture2D_U = new Texture2D(data.width >> 1, data.height, TextureFormat.Alpha8, false);
- ThermalCamTexture2D_V = new Texture2D(data.width >> 1, data.height, TextureFormat.Alpha8, false);
- ThermalCameraPreview.material.SetTexture("_MainTex", ThermalCamTexture2D_Y);
- ThermalCameraPreview.material.SetTexture("_UTex", ThermalCamTexture2D_U);
- ThermalCameraPreview.material.SetTexture("_VTex", ThermalCamTexture2D_V);
- }
- byte[] dataY = null;
- byte[] dataU = null;
- byte[] dataV = null;
- NxrSlam.Instance.LoadYUV(data.data, data.width, data.height, ref dataY, ref dataU, ref dataV,
- NxrSlam.Codec.YUYV);
- ThermalCamTexture2D_Y.LoadRawTextureData(dataY);
- ThermalCamTexture2D_U.LoadRawTextureData(dataU);
- ThermalCamTexture2D_V.LoadRawTextureData(dataV);
- ThermalCamTexture2D_Y.Apply();
- ThermalCamTexture2D_U.Apply();
- ThermalCamTexture2D_V.Apply();
- }
- }
-
- ProcessRGBCameraData();
-
- Queue<NxrSlam.NXRStreamData> tofQueue = StreamDataDict[NxrSlam.RawDataType.TOF];
- if (tofQueue.Count > 0)
- {
- NxrSlam.NXRStreamData data = tofQueue.Dequeue();
-
- if (data.size > 0)
- {
- if (TofCamTexture2D == null)
- {
- TofCamTexture2D = new Texture2D(data.width, data.height, TextureFormat.RGB565, false);
- TofCameraPreview.material.SetTexture("_MainTex", TofCamTexture2D);
- }
-
- float dmax = 7.5f;
-
- int dIndex = 0;
- for (int i = 0; i < data.size; i++)
- {
- if (i % 4 == 0)
- {
- float d = System.BitConverter.ToSingle(data.data, i);
- int u = (int) Mathf.Max(0, Mathf.Min(255.0f, d * 255.0f / dmax));
-
- Color pixelsColor = NxrSlam.Instance.Int_To_RGB(u);
-
- int x = dIndex % data.width;
- int y = dIndex / data.width;
-
- TofCamTexture2D.SetPixel(x, data.height - y, pixelsColor);
- dIndex++;
- }
- }
-
- TofCamTexture2D.Apply();
- }
- }
-
- Queue<NxrSlam.NXRStreamData> fishEyeQueue = StreamDataDict[NxrSlam.RawDataType.FISHEYE];
- if (fishEyeQueue.Count > 0)
- {
- if (!isStartConvert)
- {
- NxrSlam.NXRStreamData data = fishEyeQueue.Dequeue();
- _nxrStreamData = data;
- if (data.leftSize > 0 && data.rightSize > 0)
- {
- if (FishEyeLeftCamTexture2D == null)
- {
- FishEyeLeftCamTexture2D = new Texture2D(data.width, data.height, TextureFormat.RGB24, false);
- FishEyeRightCamTexture2D = new Texture2D(data.width, data.height, TextureFormat.RGB24, false);
- FishEyeLeftCameraPreview.material.SetTexture("_MainTex", FishEyeLeftCamTexture2D);
- FishEyeRightCameraPreview.material.SetTexture("_MainTex", FishEyeRightCamTexture2D);
- FishEyeLeftColors = new Color[data.leftSize];
- FishEyeRightColors = new Color[data.rightSize];
- }
- }
- isStartConvert = true;
- }
- }
-
- if (isConverted)
- {
- FishEyeLeftCamTexture2D.SetPixels(FishEyeLeftColors);
- FishEyeLeftCamTexture2D.Apply();
- FishEyeRightCamTexture2D.SetPixels(FishEyeRightColors);
- FishEyeRightCamTexture2D.Apply();
- isConverted = false;
- }
-
- if (PlaneDataQueue.Count > 0)
- {
- NxrSlam.NXRPlaneData data = PlaneDataQueue.Dequeue();
- if (PlaneDict.ContainsKey(data.id))
- {
- PlaneDict[data.id].MakePlane(data);
- }
- else
- {
- PlaneDict[data.id] = NxrSlam.Instance.GeneratePlaneObject(data);
- }
- string planeTip = "平面检测成功: \n";
- foreach (NxrSlamPlane plane in PlaneDict.Values)
- {
- planeTip += "[Id." + plane.PlaneId + "] 共 " + plane.PlanePointsCount + "个点\n";
- }
- PlaneDetectionTip.text = planeTip;
- }
- }
- private bool isRunning = true;
- private bool isStartConvert = false;
- private bool isConverted = false;
- private NxrSlam.NXRStreamData _nxrStreamData;
- void ShowFishEyeImage()
- {
- while (isRunning)
- {
- if (isStartConvert && !isConverted)
- {
- if (_nxrStreamData != null && _nxrStreamData.leftSize > 0 && _nxrStreamData.rightSize > 0)
- {
- var data = _nxrStreamData;
- for (int i = 0; i < data.leftSize; i++)
- {
- var dataLeft = data.left[i] / 255.0f;
- var dataRight = data.right[i] / 255.0f;
- FishEyeLeftColors[i].r = dataLeft;
- FishEyeLeftColors[i].g = dataLeft;
- FishEyeLeftColors[i].b = dataLeft;
- FishEyeRightColors[i].r = dataRight;
- FishEyeRightColors[i].g = dataRight;
- FishEyeRightColors[i].b = dataRight;
- }
- isStartConvert = false;
- isConverted = true;
- }
- }
- Thread.Sleep(10);
- }
- }
- private void OnDestroy()
- {
- if (RgbCameraDataPtrArray[0] != IntPtr.Zero)
- {
- Marshal.FreeHGlobal(RgbCameraDataPtrArray[0]);
- Marshal.FreeHGlobal(RgbCameraDataPtrArray[1]);
- }
- NxrSlam.Instance.UnInit();
- Debug.Log("OnDestroy");
- isRunning = false;
- isStartConvert = false;
- isConverted = false;
- }
- private void OnApplicationPause(bool pause)
- {
- Debug.Log("OnApplicationPause=" + pause);
- if (pause) NxrSlam.Instance.Pause();
- }
- private void OnApplicationQuit()
- {
- NxrSlam.Instance.UnInit();
- Debug.Log("OnApplicationQuit");
- }
- void DebugPoints()
- {
- NxrSlam.NXRPlaneData testdata = new NxrSlam.NXRPlaneData(null);
- testdata.id = 1;
- testdata.points = new NxrSlam.NXRPlaneData.Point3D[]
- {
-
- new NxrSlam.NXRPlaneData.Point3D(1.0432962179184, -0.302729845046997, 0.245670855045319),
- new NxrSlam.NXRPlaneData.Point3D(0.599149584770203, -0.302729845046997, 0.168545603752136),
- new NxrSlam.NXRPlaneData.Point3D(-0.248820841312408, -0.302729845046997, 0.573924124240875),
- new NxrSlam.NXRPlaneData.Point3D(-0.401118278503418, -0.302729845046997, 0.789052367210388),
- new NxrSlam.NXRPlaneData.Point3D(-0.450264513492584, -0.302729845046997, 0.886043250560761),
- new NxrSlam.NXRPlaneData.Point3D(-0.450264513492584, -0.302729845046997, 0.886043250560761),
- new NxrSlam.NXRPlaneData.Point3D(-0.0576245971024036, -0.302729845046997, 1.1546128988266),
- new NxrSlam.NXRPlaneData.Point3D(0.103346958756447, -0.302729845046997, 1.081174492836),
- new NxrSlam.NXRPlaneData.Point3D(0.836297452449799, -0.302729845046997, 0.65020763874054),
- new NxrSlam.NXRPlaneData.Point3D(1.04192793369293, -0.302729845046997, 0.37995707988739),
- new NxrSlam.NXRPlaneData.Point3D(1.0432962179184, -0.302729845046997, 0.245670855045319)
-
- };
- testdata.size = testdata.points.Length;
- testdata.Refresh();
- PlaneDataQueue.Enqueue(testdata);
- }
- }
|