using System.Collections.Generic; using UnityEngine; namespace Assets.NXR.Scripts.Slam { public class NxrSlam { [System.Runtime.InteropServices.DllImport("nxr_slam")] public static extern bool initPtr(int streamType, System.IntPtr dataPtr0, System.IntPtr dataPtr1); /// /// /// /// /// /// /// 0/1 has valid data, -1 no data [System.Runtime.InteropServices.DllImport("nxr_slam")] public static extern int getNxrStreamData(int streamType, ref int width, ref int height); [System.Runtime.InteropServices.DllImport("nxr_slam")] public static extern int getNxrStreamSize(int streamType); [System.Runtime.InteropServices.DllImport("nxr_slam")] public static extern void unlockStream(int idx); public enum TOFStreamMode { ONLY_DEPTH, // w*h*4 ONLY_CLOUD, DEPTH_CLOUD,// w*h*12 NONE, CLOUD_ON_LEFTHAND_SLAM } public enum TOFFPS { TOF_FPS_5, TOF_FPS_10, TOF_FPS_15, TOF_FPS_20, TOF_FPS_25, TOF_FPS_30 } public enum RgbResolution { RGB_1920x1080, RGB_1280x720, RGB_640x480, RGB_320x240, RGB_2560x1920, TOF } public enum Mode { MODE_3DOF, MODE_6DOF } public enum RawDataType { NONE, EYETRACKING, RGB, TOF, FISHEYE, THERMAL } public enum Codec { YUYV, YUV420p, JPEG, NV12, BITSTREAM } public enum RgbType { Preview, Capture } public enum RgbFormat { FORMAT_NONE, FORMAT_YUYV, FORMAT_YUV420, FORMAT_RGB, FORMAT_RGBA, FORMAT_H264 } public enum TofType { Depth_16, Depth_32, IR, Cloud, Raw, Eeprom } public class NXRStreamData { public RawDataType rawDataType; public byte[] data;//rgb or tof camera data public byte[] left; // fish eye left camera data public byte[] right;// fish eye right camera data public int type;//数据类型:TofType/RgbType // 224*172*3*4 bytes public TofType tofType; public RgbType rgbType; public Codec rgbCodecFormat;// only for rgb public int width; public int height; public int size; // rgb or tof size public int leftSize; // fish eye left size public int rightSize;// fish eye right size public long timestamp; public TOFStreamMode tofStreamMode; public NXRStreamData(RawDataType dataType, AndroidJavaObject javaObject) { rawDataType = dataType; data = AndroidJNIHelper.ConvertFromJNIArray(javaObject.Get("data").GetRawObject()); if (rawDataType == RawDataType.FISHEYE) { left = AndroidJNIHelper.ConvertFromJNIArray(javaObject.Get("left").GetRawObject()); right = AndroidJNIHelper.ConvertFromJNIArray(javaObject.Get("right").GetRawObject()); } type = javaObject.Get("type"); rgbCodecFormat = (Codec)javaObject.Get("format"); width = javaObject.Get("width"); height = javaObject.Get("height"); size = javaObject.Get("size"); leftSize = javaObject.Get("leftSize"); rightSize = javaObject.Get("rightSize"); timestamp = javaObject.Get("timestamp"); if (rawDataType == RawDataType.RGB) { rgbType = (RgbType)type; } else if (rawDataType == RawDataType.TOF) { tofType = (TofType)type; } } } public class NXRPlaneData { public long hostTimestamp = 0L; public long deviceTimestamp = 0L; public NXRPlaneData.Point3D[] points; public NXRPlaneData.Point3D normal = new Point3D(); public int size; public int id = -1; public void Refresh() { if ( Mathf.Abs((float)(points[0].x - points[size - 1].x)) < Mathf.Epsilon && Mathf.Abs((float)(points[0].y - points[size - 1].y)) < Mathf.Epsilon && Mathf.Abs((float)(points[0].z - points[size - 1].z)) < Mathf.Epsilon ) { // 首尾相同 size = size - 1; } } public NXRPlaneData(AndroidJavaObject javaObject) { if(javaObject != null) { hostTimestamp = javaObject.Get("hostTimestamp"); deviceTimestamp = javaObject.Get("deviceTimestamp"); size = javaObject.Get("size"); id = javaObject.Get("id"); AndroidJavaObject normalObj = javaObject.Get("normal"); if (normalObj != null) { normal.x = normalObj.Get("x"); normal.y = normalObj.Get("y"); normal.z = normalObj.Get("z"); normalObj.Dispose(); } AndroidJavaObject pointsObj = javaObject.Get("points"); if (pointsObj != null) { AndroidJavaObject[] pointsArray = AndroidJNIHelper.ConvertFromJNIArray(pointsObj.GetRawObject()); if (pointsArray != null) { if ( Mathf.Abs((float) (pointsArray[0].Get("x") - pointsArray[size-1].Get("x"))) < Mathf.Epsilon && Mathf.Abs((float)(pointsArray[0].Get("y") - pointsArray[size - 1].Get("y"))) < Mathf.Epsilon && Mathf.Abs((float)(pointsArray[0].Get("z") - pointsArray[size - 1].Get("z"))) < Mathf.Epsilon ) { // 首尾相同 size = size - 1; } points = new Point3D[size]; for (int i = 0; i < size; i++) { Point3D point3D = new Point3D(); point3D.x = pointsArray[i].Get("x"); point3D.y = pointsArray[i].Get("y"); point3D.z = pointsArray[i].Get("z"); // 右手坐标系转换,绕x轴旋转-180度 point3D.y = point3D.y * (-1); points[i] = point3D; pointsArray[i].Dispose(); } } pointsObj.Dispose(); } } } public class Point3D { public double x; public double y; public double z; public Point3D() { } public Point3D(double x, double y, double z) { this.x = x; this.y = y; this.z = z; } } } private static object syncRoot = new object(); private static NxrSlam _instance = null; public static NxrSlam Instance { get { if (_instance == null) //第一重判断,先判断实例是否存在,不存在再加锁处理 { lock (syncRoot) //加锁,在某一时刻只允许一个线程访问 { if (_instance == null) //第二重判断: 第一个线程进入Lock中执行创建代码,第二个线程处于排队等待状态,当第二个线程进入Lock后并不知道实例已创建,将会继续创建新的实例 { _instance = new NxrSlam(); } } } return _instance; } } public bool IsRGBCameraSteaming { get; set; } private NxrSlam() { #if !UNITY_EDITOR GetNxrSlamInstance(); #endif } AndroidJavaObject nxrSlamInstance; AndroidJavaClass nxrSlamClass; #if !UNITY_EDITOR public AndroidJavaObject GetNxrSlamInstance() { if (nxrSlamClass == null) { nxrSlamClass = new AndroidJavaClass("ruiyue.nxr.slam.NXRSlam"); } if (nxrSlamClass != null && nxrSlamInstance == null) { nxrSlamInstance = nxrSlamClass.CallStatic("getInstance"); Debug.Log("NXRSlam getInstance"); } return nxrSlamInstance; } #else private AndroidJavaObject GetJavaObject() { return null;} #endif public void EnableNativeLog(bool enable) { if (nxrSlamInstance != null) { Debug.Log("NXRSlam enableNativeLog"); nxrSlamInstance.Call("enableNativeLog", enable); } } bool Inited = false; public bool Init() { IsRGBCameraSteaming = false; if (nxrSlamInstance != null) { Debug.Log("NXRSlam init"); Inited = true; return nxrSlamInstance.Call("init"); } return false; } public void UnInit() { if (nxrSlamInstance != null && Inited) { StopPlaneDetection(); StopStreaming(); StopBuildMap(); Debug.Log("NXRSlam uninit"); Inited = false; nxrSlamInstance.Call("uninit"); } } /// /// /// /// 地图路径:sdcard/unity_map.bin /// public bool StartSlamWithMap(string path) { // 是否存在 //if (path == null || path.Length == 0 || !File.Exists(path)) //{ // Debug.Log("NXRSlam startSlamWithMap failed, file not exist !!! " + path); // return false; //} if (nxrSlamInstance != null) { IsBuildingMap = true; Debug.Log("NXRSlam startSlamWithMap." + path); return nxrSlamInstance.Call("startSlamWithMap", path, new MapBuildCallback()); } return false; } public void StopSlam() { if (nxrSlamInstance != null) { Debug.Log("NXRSlam stopSlam"); nxrSlamInstance.Call("stopSlam"); } } bool IsBuildingMap = false; public bool StartBuildMap() { if (nxrSlamInstance != null) { IsBuildingMap = true; Debug.Log("NXRSlam startRecordMap"); return nxrSlamInstance.Call("startRecordMap", new MapBuildCallback()); } return false; } /// /// /// /// sdcard/unity_map.bin /// public bool StopBuildMap(string path = null) { if (nxrSlamInstance != null && IsBuildingMap) { IsBuildingMap = false; //if(path == null) //{ // path = "system/etc/unity_map.bin"; //} Debug.Log("NXRSlam stopRecordMap." + path); return nxrSlamInstance.Call("stopRecordMap", path); } return false; } bool IsPlaneDetecting = false; public bool StartPlaneDetection() { if (nxrSlamInstance != null) { IsPlaneDetecting = true; Debug.Log("NXRSlam startPlaneDetection"); return nxrSlamInstance.Call("startPlaneDetection", new PlaneDetectionCallback()); } return false; } public bool StopPlaneDetection() { if (nxrSlamInstance != null && IsPlaneDetecting) { IsPlaneDetecting = false; Debug.Log("NXRSlam stopPlaneDetection"); return nxrSlamInstance.Call("stopPlaneDetection"); } return false; } public void Pause() { Debug.Log("NXRSlam pause"); //StopPlaneDetection(); //StopStreaming(); //StopBuildMap(); } List streamingDataTypeList = new List(); AndroidJavaClass rawDataTypeCls = null; AndroidJavaClass rgbFormatCls = null; AndroidJavaClass tofStreamModeCls = null; AndroidJavaClass tofFpsCls = null; AndroidJavaClass rgbResolutionCls = null; public bool StartStreaming(RawDataType rawDataType) { if (nxrSlamInstance != null) { if (rawDataTypeCls == null) { rawDataTypeCls = new AndroidJavaClass("ruiyue.nxr.slam.NXRSlam$RawDataType"); } if (rgbFormatCls == null) { rgbFormatCls = new AndroidJavaClass("ruiyue.nxr.slam.NXRSlam$RgbFormat"); } if (rawDataType == RawDataType.RGB) { AndroidJavaObject rgbFormatObj = rgbFormatCls.CallStatic("valueOf", RgbFormat.FORMAT_RGB.ToString()); nxrSlamInstance.Call("setRgbFormat", rgbFormatObj); IsRGBCameraSteaming = true; } AndroidJavaObject rawDataTypeObj = rawDataTypeCls.CallStatic("valueOf", rawDataType.ToString()); streamingDataTypeList.Add(rawDataType); return nxrSlamInstance.Call("startStreaming", rawDataTypeObj, new StreamDataCallback()); } return false; } public void SetLed(int select, int channel, int brightness) { if (nxrSlamInstance != null) { nxrSlamInstance.Call("setLed", select, channel, brightness); } } public void SetAec(int aecmode, int aecgain, int aectime) { if (nxrSlamInstance != null) { nxrSlamInstance.Call("setAec", aecmode, aecgain, aectime); } } TOFStreamMode tofStreamMode = TOFStreamMode.ONLY_DEPTH; public void SetTofStreamMode(TOFStreamMode mode) { if (nxrSlamInstance != null) { if (tofStreamModeCls == null) { tofStreamModeCls = new AndroidJavaClass("ruiyue.nxr.slam.NXRSlam$TOFStreamMode"); } AndroidJavaObject tofStreamModeObj = tofStreamModeCls.CallStatic("valueOf", mode.ToString()); nxrSlamInstance.Call("setTofStreamMode", tofStreamModeObj); tofStreamMode = mode; } } public void SetTofFps(TOFFPS fps) { if (nxrSlamInstance != null) { if (tofFpsCls == null) { tofFpsCls = new AndroidJavaClass("ruiyue.nxr.slam.NXRSlam$TOFFPS"); } AndroidJavaObject tofStreamModeObj = tofFpsCls.CallStatic("valueOf", fps.ToString()); nxrSlamInstance.Call("setTOFFps", tofStreamModeObj); } } public void SetRgbResolution(RgbResolution rgbResolution) { if (nxrSlamInstance != null) { if (rgbResolutionCls == null) { rgbResolutionCls = new AndroidJavaClass("ruiyue.nxr.slam.NXRSlam$RgbResolution"); } AndroidJavaObject tofStreamModeObj = rgbResolutionCls.CallStatic("valueOf", rgbResolution.ToString()); nxrSlamInstance.Call("setRgbResolution", tofStreamModeObj); } } public bool StopStreaming(RawDataType rawDataType = RawDataType.NONE) { if (nxrSlamInstance != null) { if (rawDataType == RawDataType.NONE) { foreach(RawDataType type in streamingDataTypeList) { AndroidJavaObject rawDataTypeObj = rawDataTypeCls.CallStatic("valueOf", type.ToString()); Debug.Log("NXRSlam stopStreaming." + type.ToString()); nxrSlamInstance.Call("stopStreaming", rawDataTypeObj); } IsRGBCameraSteaming = false; streamingDataTypeList.Clear(); return true; } else { if (rawDataType == RawDataType.RGB) IsRGBCameraSteaming = false; AndroidJavaObject rawDataTypeObj = rawDataTypeCls.CallStatic("valueOf", rawDataType.ToString()); return nxrSlamInstance.Call("stopStreaming", rawDataTypeObj); } } return false; } private Mode CurMode = Mode.MODE_3DOF; public bool SwitchMode(Mode mode) { if (nxrSlamInstance != null) { CurMode = mode; AndroidJavaClass modeCls = new AndroidJavaClass("ruiyue.nxr.slam.NXRSlam$Mode"); AndroidJavaObject modeObj = rawDataTypeCls.CallStatic("valueOf", mode.ToString()); return nxrSlamInstance.Call("switchMode", modeObj); } return false; } #region public delegate void MapStatusAndQualityCallback(int status, int quality); public delegate void MapPercentCallback(float percent); public delegate void StreamingDataCallback(NXRStreamData data); public delegate void PlaneDetectionCallaback(NXRPlaneData data); public MapStatusAndQualityCallback MapStatusAndQualityEvent; public MapPercentCallback MapPercentEvent; public StreamingDataCallback StreamDataEvent; public PlaneDetectionCallaback PlaneDetectionEvent; public class MapBuildCallback : AndroidJavaProxy { public MapBuildCallback() : base("ruiyue.nxr.slam.onMapListener") { } public void onMapStatusAndQuality(int status, int quality) { if (Instance.MapStatusAndQualityEvent != null) { Instance.MapStatusAndQualityEvent(status, quality); } } public void onMapPercent(float percent) { if (Instance.MapPercentEvent != null) { Instance.MapPercentEvent(percent); } } } public class StreamDataCallback : AndroidJavaProxy { public StreamDataCallback() : base("ruiyue.nxr.slam.onStreamDataListener") { } public void onStreamData(AndroidJavaObject rawDataType, AndroidJavaObject streamData) { RawDataType type = (RawDataType)rawDataType.Call("ordinal"); if (type == RawDataType.RGB) return; NXRStreamData nxrStreamData = new NXRStreamData(type, streamData); nxrStreamData.tofStreamMode = Instance.tofStreamMode; if (Instance.StreamDataEvent != null) { Instance.StreamDataEvent(nxrStreamData); } } } public class PlaneDetectionCallback : AndroidJavaProxy { public PlaneDetectionCallback() : base("ruiyue.nxr.slam.onPlaneDetectionListener") { } public void onPlaneDetected(AndroidJavaObject planeData) { if (planeData != null) { NXRPlaneData nxrPlaneData = new NXRPlaneData(planeData); if (Instance.PlaneDetectionEvent != null) { Instance.PlaneDetectionEvent(nxrPlaneData); } } else { Debug.LogError("AndroidJavaObject planeData is null !!!"); } } } #endregion public NxrSlamPlane GeneratePlaneObject(NXRPlaneData planeData) { // Debug.Log("GeneratePlaneObject:" + planeData.id); GameObject obj = new GameObject(); obj.name = "NxrSlamPlane " + planeData.id; obj.AddComponent(); MeshRenderer meshRenderer = obj.AddComponent(); meshRenderer.material = Resources.Load("PlaneMaterial", typeof(Material)) as Material; obj.AddComponent(); NxrSlamPlane plnScript = obj.AddComponent(); plnScript.MakePlane(planeData); return plnScript; } /// /// UYVY=>YUV422 /// [U0 Y0 V0 Y1] [U1 Y2 V1 Y3] [U2 Y4 V2 Y5] /// Y = w * h, u = w/2 *h, v = w/2 *h /// 2bytes=16bits = 8 + 4 + 4 /// /// YUV420 /// Y = w * h, u = w/2 *h/2, v = w/2 *h/2 /// 1.5bytes=12bits = 8 + 2 + 2 /// /// /// /// /// /// /// /// /// public void LoadYUV(byte[] buff, int videoW, int videoH, ref byte[] bufY, ref byte[] bufU, ref byte[] bufV, Codec codec = Codec.YUV420p) { float pixelBytes = codec == Codec.YUV420p ? 1.5f : 2.0f; int firstFrameEndIndex = (int)(videoH * videoW * pixelBytes); int yIndex = firstFrameEndIndex * 8 / 12; int uIndex = firstFrameEndIndex * 10 / 12; // int vIndex = firstFrameEndIndex; if(codec == Codec.YUV420p) { bufY = new byte[videoW * videoH]; bufU = new byte[videoW * videoH >> 2]; bufV = new byte[videoW * videoH >> 2]; for (int i = 0; i < firstFrameEndIndex; i++) { if (i < yIndex) { bufY[i] = buff[i]; } else if (i < uIndex) { bufU[i - yIndex] = buff[i]; } else { bufV[i - uIndex] = buff[i]; } } } else if(codec == Codec.YUYV) { bufY = new byte[videoW * videoH]; bufU = new byte[videoW/2 * videoH]; bufV = new byte[videoW/2 * videoH]; int uTmpIndex = 0; int vTmpIndex = 0; bool isUFind = false; for (int i = 0; i < firstFrameEndIndex; i++) { if (i % 2 == 1) { // Y bufY[i / 2] = buff[i]; } else { // U / V if (!isUFind) { bufU[uTmpIndex] = buff[i]; isUFind = true; uTmpIndex++; } else { bufV[vTmpIndex] = buff[i]; isUFind = false; vTmpIndex++; } } } } // byte[] bufUV = new byte[videoW * videoH >> 1]; //如果是把UV分量一起写入到一张RGBA4444的纹理中时,byte[] //里的字节顺序应该是 UVUVUVUV.... //这样在shader中纹理采样的结果 U 分量就存在r、g通道。 //V 分量就存在b、a通道。 //for(int i = 0; i < bufUV.Length; i+=2) //{ // bufUV[i] = bufU[i >> 1]; // bufUV[i + 1] = bufV[i >> 1]; //} } public Color Int_To_RGB(int color) { int b = color / (256 * 256); int g = (color - b * 256 * 256) / 256; int r = color - b * 256 * 256 - g * 256; Color outcolor = Color.white; outcolor.r = r; outcolor.g = g; outcolor.b = b; return outcolor; } public void YUV420P_TO_RGB24(int width, int height, byte[] dataY, byte[] dataU, byte[] dataV, byte[] dataRGB24) { int index_y=0, index_u = 0, index_v = 0; int index_r=0, index_g = 0, index_b = 0; for(int i=0; i< height; i++) { for(int j=0; j