using System; using System.Collections; using UnityEngine; using UnityEngine.Events; using UnityEngine.Serialization; using OpenCVForUnity.CoreModule; using OpenCVForUnity.UnityUtils; namespace OpenCVForUnity.UnityUtils.Helper { /// /// WebcamTexture to mat helper. /// v 1.1.0 /// public class WebCamTextureToMatHelper : MonoBehaviour { /// /// Set the name of the camera device to use. (or device index number) /// [SerializeField, FormerlySerializedAs ("requestedDeviceName"), TooltipAttribute ("Set the name of the device to use. (or device index number)")] protected string _requestedDeviceName = null; public virtual string requestedDeviceName { get { return _requestedDeviceName; } set { _requestedDeviceName = value; if (hasInitDone) { Initialize (); } } } /// /// Set the width of camera. /// [SerializeField, FormerlySerializedAs ("requestedWidth"), TooltipAttribute ("Set the width of camera.")] protected int _requestedWidth = 640; public virtual int requestedWidth { get { return _requestedWidth; } set { _requestedWidth = (int)Mathf.Clamp (value, 0f, float.MaxValue); if (hasInitDone) { Initialize (); } } } /// /// Set the height of camera. /// [SerializeField, FormerlySerializedAs ("requestedHeight"), TooltipAttribute ("Set the height of camera.")] protected int _requestedHeight = 480; public virtual int requestedHeight { get { return _requestedHeight; } set { _requestedHeight = (int)Mathf.Clamp (value, 0f, float.MaxValue); if (hasInitDone) { Initialize (); } } } /// /// Set whether to use the front facing camera. /// [SerializeField, FormerlySerializedAs ("requestedIsFrontFacing"), TooltipAttribute ("Set whether to use the front facing camera.")] protected bool _requestedIsFrontFacing = false; public virtual bool requestedIsFrontFacing { get { return _requestedIsFrontFacing; } set { _requestedIsFrontFacing = value; if (hasInitDone) { Initialize (_requestedIsFrontFacing, requestedFPS, rotate90Degree); } } } /// /// Set the frame rate of camera. /// [SerializeField, FormerlySerializedAs ("requestedFPS"), TooltipAttribute ("Set the frame rate of camera.")] protected float _requestedFPS = 30f; public virtual float requestedFPS { get { return _requestedFPS; } set { _requestedFPS = Mathf.Clamp (value, -1f, float.MaxValue); if (hasInitDone) { bool isPlaying = IsPlaying (); Stop (); webCamTexture.requestedFPS = _requestedFPS; if (isPlaying) Play (); } } } /// /// Sets whether to rotate camera frame 90 degrees. (clockwise) /// [SerializeField, FormerlySerializedAs ("rotate90Degree"), TooltipAttribute ("Sets whether to rotate camera frame 90 degrees. (clockwise)")] protected bool _rotate90Degree = false; public virtual bool rotate90Degree { get { return _rotate90Degree; } set { _rotate90Degree = value; if (hasInitDone) { Initialize (); } } } /// /// Determines if flips vertically. /// [SerializeField, FormerlySerializedAs ("flipVertical"), TooltipAttribute ("Determines if flips vertically.")] protected bool _flipVertical = false; public virtual bool flipVertical { get { return _flipVertical; } set { _flipVertical = value; } } /// /// Determines if flips horizontal. /// [SerializeField, FormerlySerializedAs ("flipHorizontal"), TooltipAttribute ("Determines if flips horizontal.")] protected bool _flipHorizontal = false; public virtual bool flipHorizontal { get { return _flipHorizontal; } set { _flipHorizontal = value; } } /// /// The number of frames before the initialization process times out. /// [SerializeField, FormerlySerializedAs ("timeoutFrameCount"), TooltipAttribute ("The number of frames before the initialization process times out.")] protected int _timeoutFrameCount = 300; public virtual int timeoutFrameCount { get { return _timeoutFrameCount; } set { _timeoutFrameCount = (int)Mathf.Clamp (value, 0f, float.MaxValue); } } /// /// UnityEvent that is triggered when this instance is initialized. /// public UnityEvent onInitialized; /// /// UnityEvent that is triggered when this instance is disposed. /// public UnityEvent onDisposed; /// /// UnityEvent that is triggered when this instance is error Occurred. /// public ErrorUnityEvent onErrorOccurred; /// /// The active WebcamTexture. /// protected WebCamTexture webCamTexture; /// /// The active WebcamDevice. /// protected WebCamDevice webCamDevice; /// /// The frame mat. /// protected Mat frameMat; /// /// The rotated frame mat /// protected Mat rotatedFrameMat; /// /// The buffer colors. /// protected Color32[] colors; /// /// Indicates whether this instance is waiting for initialization to complete. /// protected bool isInitWaiting = false; /// /// Indicates whether this instance has been initialized. /// protected bool hasInitDone = false; /// /// The initialization coroutine. /// protected IEnumerator initCoroutine; /// /// The orientation of the screen. /// protected ScreenOrientation screenOrientation; /// /// The width of the screen. /// protected int screenWidth; /// /// The height of the screen. /// protected int screenHeight; /// /// Indicates whether this instance avoids the front camera low light issue that occurs in only some Android devices (e.g. Google Pixel, Pixel2). /// Sets compulsorily the requestedFPS parameter to 15 (only when using the front camera), to avoid the problem of the WebCamTexture image becoming low light. /// https://forum.unity.com/threads/android-webcamtexture-in-low-light-only-some-models.520656/ /// https://forum.unity.com/threads/released-opencv-for-unity.277080/page-33#post-3445178 /// public bool avoidAndroidFrontCameraLowLightIssue = false; [System.Serializable] public enum ErrorCode : int { UNKNOWN = 0, CAMERA_DEVICE_NOT_EXIST = 1, TIMEOUT = 2, } [System.Serializable] public class ErrorUnityEvent : UnityEngine.Events.UnityEvent { } protected virtual void OnValidate () { _requestedWidth = (int)Mathf.Clamp (_requestedWidth, 0f, float.MaxValue); _requestedHeight = (int)Mathf.Clamp (_requestedHeight, 0f, float.MaxValue); _requestedFPS = Mathf.Clamp (_requestedFPS, -1f, float.MaxValue); _timeoutFrameCount = (int)Mathf.Clamp (_timeoutFrameCount, 0f, float.MaxValue); } // Update is called once per frame protected virtual void Update () { if (hasInitDone) { // Catch the orientation change of the screen and correct the mat image to the correct direction. if (screenOrientation != Screen.orientation && (screenWidth != Screen.width || screenHeight != Screen.height)) { if (onDisposed != null) onDisposed.Invoke (); if (frameMat != null) { frameMat.Dispose (); frameMat = null; } if (rotatedFrameMat != null) { rotatedFrameMat.Dispose (); rotatedFrameMat = null; } frameMat = new Mat (webCamTexture.height, webCamTexture.width, CvType.CV_8UC4, new Scalar (0, 0, 0, 255)); screenOrientation = Screen.orientation; screenWidth = Screen.width; screenHeight = Screen.height; bool isRotatedFrame = false; #if !UNITY_EDITOR && !(UNITY_STANDALONE || UNITY_WEBGL) if (screenOrientation == ScreenOrientation.Portrait || screenOrientation == ScreenOrientation.PortraitUpsideDown) { if (!rotate90Degree) isRotatedFrame = true; } else if (rotate90Degree) { isRotatedFrame = true; } #else if (rotate90Degree) isRotatedFrame = true; #endif if (isRotatedFrame) rotatedFrameMat = new Mat (webCamTexture.width, webCamTexture.height, CvType.CV_8UC4, new Scalar (0, 0, 0, 255)); if (onInitialized != null) onInitialized.Invoke (); } else { screenWidth = Screen.width; screenHeight = Screen.height; } } } /// /// Raises the destroy event. /// protected virtual void OnDestroy () { Dispose (); } /// /// Initializes this instance. /// public virtual void Initialize () { if (isInitWaiting) { CancelInitCoroutine (); ReleaseResources (); } if (onInitialized == null) onInitialized = new UnityEvent (); if (onDisposed == null) onDisposed = new UnityEvent (); if (onErrorOccurred == null) onErrorOccurred = new ErrorUnityEvent (); initCoroutine = _Initialize (); StartCoroutine (initCoroutine); } /// /// Initializes this instance. /// /// Requested width. /// Requested height. public virtual void Initialize (int requestedWidth, int requestedHeight) { if (isInitWaiting) { CancelInitCoroutine (); ReleaseResources (); } this._requestedWidth = requestedWidth; this._requestedHeight = requestedHeight; if (onInitialized == null) onInitialized = new UnityEvent (); if (onDisposed == null) onDisposed = new UnityEvent (); if (onErrorOccurred == null) onErrorOccurred = new ErrorUnityEvent (); initCoroutine = _Initialize (); StartCoroutine (initCoroutine); } /// /// Initializes this instance. /// /// If set to true requested to using the front camera. /// Requested FPS. /// If set to true requested to rotate camera frame 90 degrees. (clockwise) public virtual void Initialize (bool requestedIsFrontFacing, float requestedFPS = 30f, bool rotate90Degree = false) { if (isInitWaiting) { CancelInitCoroutine (); ReleaseResources (); } _requestedDeviceName = null; this._requestedIsFrontFacing = requestedIsFrontFacing; this._requestedFPS = requestedFPS; this._rotate90Degree = rotate90Degree; if (onInitialized == null) onInitialized = new UnityEvent (); if (onDisposed == null) onDisposed = new UnityEvent (); if (onErrorOccurred == null) onErrorOccurred = new ErrorUnityEvent (); initCoroutine = _Initialize (); StartCoroutine (initCoroutine); } /// /// Initializes this instance. /// /// Device name. /// Requested width. /// Requested height. /// If set to true requested to using the front camera. /// Requested FPS. /// If set to true requested to rotate camera frame 90 degrees. (clockwise) public virtual void Initialize (string deviceName, int requestedWidth, int requestedHeight, bool requestedIsFrontFacing = false, float requestedFPS = 30f, bool rotate90Degree = false) { if (isInitWaiting) { CancelInitCoroutine (); ReleaseResources (); } this._requestedDeviceName = deviceName; this._requestedWidth = requestedWidth; this._requestedHeight = requestedHeight; this._requestedIsFrontFacing = requestedIsFrontFacing; this._requestedFPS = requestedFPS; this._rotate90Degree = rotate90Degree; if (onInitialized == null) onInitialized = new UnityEvent (); if (onDisposed == null) onDisposed = new UnityEvent (); if (onErrorOccurred == null) onErrorOccurred = new ErrorUnityEvent (); initCoroutine = _Initialize (); StartCoroutine (initCoroutine); } /// /// Initializes this instance by coroutine. /// protected virtual IEnumerator _Initialize () { if (hasInitDone) { ReleaseResources (); if (onDisposed != null) onDisposed.Invoke (); } isInitWaiting = true; float requestedFPS = this.requestedFPS; // Creates the camera if (!String.IsNullOrEmpty (requestedDeviceName)) { int requestedDeviceIndex = -1; if (Int32.TryParse (requestedDeviceName, out requestedDeviceIndex)) { if (requestedDeviceIndex >= 0 && requestedDeviceIndex < WebCamTexture.devices.Length) { webCamDevice = WebCamTexture.devices [requestedDeviceIndex]; if (avoidAndroidFrontCameraLowLightIssue && webCamDevice.isFrontFacing == true) requestedFPS = 15f; if (requestedFPS < 0) { webCamTexture = new WebCamTexture (webCamDevice.name, requestedWidth, requestedHeight); } else { webCamTexture = new WebCamTexture (webCamDevice.name, requestedWidth, requestedHeight, (int)requestedFPS); } } } else { for (int cameraIndex = 0; cameraIndex < WebCamTexture.devices.Length; cameraIndex++) { if (WebCamTexture.devices [cameraIndex].name == requestedDeviceName) { webCamDevice = WebCamTexture.devices [cameraIndex]; if (avoidAndroidFrontCameraLowLightIssue && webCamDevice.isFrontFacing == true) requestedFPS = 15f; if (requestedFPS < 0) { webCamTexture = new WebCamTexture (webCamDevice.name, requestedWidth, requestedHeight); } else { webCamTexture = new WebCamTexture (webCamDevice.name, requestedWidth, requestedHeight, (int)requestedFPS); } break; } } } if (webCamTexture == null) Debug.Log ("Cannot find camera device " + requestedDeviceName + "."); } if (webCamTexture == null) { // Checks how many and which cameras are available on the device for (int cameraIndex = 0; cameraIndex < WebCamTexture.devices.Length; cameraIndex++) { if (WebCamTexture.devices [cameraIndex].isFrontFacing == requestedIsFrontFacing) { webCamDevice = WebCamTexture.devices [cameraIndex]; if (avoidAndroidFrontCameraLowLightIssue && webCamDevice.isFrontFacing == true) requestedFPS = 15f; if (requestedFPS < 0) { webCamTexture = new WebCamTexture (webCamDevice.name, requestedWidth, requestedHeight); } else { webCamTexture = new WebCamTexture (webCamDevice.name, requestedWidth, requestedHeight, (int)requestedFPS); } break; } } } if (webCamTexture == null) { if (WebCamTexture.devices.Length > 0) { webCamDevice = WebCamTexture.devices [0]; if (avoidAndroidFrontCameraLowLightIssue && webCamDevice.isFrontFacing == true) requestedFPS = 15f; if (requestedFPS < 0) { webCamTexture = new WebCamTexture (webCamDevice.name, requestedWidth, requestedHeight); } else { webCamTexture = new WebCamTexture (webCamDevice.name, requestedWidth, requestedHeight, (int)requestedFPS); } } else { isInitWaiting = false; if (onErrorOccurred != null) onErrorOccurred.Invoke (ErrorCode.CAMERA_DEVICE_NOT_EXIST); yield break; } } // Starts the camera webCamTexture.Play (); int initFrameCount = 0; bool isTimeout = false; while (true) { if (initFrameCount > timeoutFrameCount) { isTimeout = true; break; } // If you want to use webcamTexture.width and webcamTexture.height on iOS, you have to wait until webcamTexture.didUpdateThisFrame == 1, otherwise these two values will be equal to 16. (http://forum.unity3d.com/threads/webcamtexture-and-error-0x0502.123922/) #if UNITY_IOS && !UNITY_EDITOR && (UNITY_4_6_3 || UNITY_4_6_4 || UNITY_5_0_0 || UNITY_5_0_1) else if (webCamTexture.width > 16 && webCamTexture.height > 16) { #else else if (webCamTexture.didUpdateThisFrame) { #if UNITY_IOS && !UNITY_EDITOR && UNITY_5_2 while (webCamTexture.width <= 16) { if (initFrameCount > timeoutFrameCount) { isTimeout = true; break; }else { initFrameCount++; } webCamTexture.GetPixels32 (); yield return new WaitForEndOfFrame (); } if (isTimeout) break; #endif #endif Debug.Log ("WebCamTextureToMatHelper:: " + "devicename:" + webCamTexture.deviceName + " name:" + webCamTexture.name + " width:" + webCamTexture.width + " height:" + webCamTexture.height + " fps:" + webCamTexture.requestedFPS + " videoRotationAngle:" + webCamTexture.videoRotationAngle + " videoVerticallyMirrored:" + webCamTexture.videoVerticallyMirrored + " isFrongFacing:" + webCamDevice.isFrontFacing); if (colors == null || colors.Length != webCamTexture.width * webCamTexture.height) colors = new Color32[webCamTexture.width * webCamTexture.height]; frameMat = new Mat (webCamTexture.height, webCamTexture.width, CvType.CV_8UC4); screenOrientation = Screen.orientation; screenWidth = Screen.width; screenHeight = Screen.height; bool isRotatedFrame = false; #if !UNITY_EDITOR && !(UNITY_STANDALONE || UNITY_WEBGL) if (screenOrientation == ScreenOrientation.Portrait || screenOrientation == ScreenOrientation.PortraitUpsideDown) { if (!rotate90Degree) isRotatedFrame = true; } else if (rotate90Degree) { isRotatedFrame = true; } #else if (rotate90Degree) isRotatedFrame = true; #endif if (isRotatedFrame) rotatedFrameMat = new Mat (webCamTexture.width, webCamTexture.height, CvType.CV_8UC4); isInitWaiting = false; hasInitDone = true; initCoroutine = null; if (onInitialized != null) onInitialized.Invoke (); break; } else { initFrameCount++; yield return null; } } if (isTimeout) { webCamTexture.Stop (); webCamTexture = null; isInitWaiting = false; initCoroutine = null; if (onErrorOccurred != null) onErrorOccurred.Invoke (ErrorCode.TIMEOUT); } } /// /// Indicates whether this instance has been initialized. /// /// true, if this instance has been initialized, false otherwise. public virtual bool IsInitialized () { return hasInitDone; } /// /// Starts the camera. /// public virtual void Play () { if (hasInitDone) webCamTexture.Play (); } /// /// Pauses the active camera. /// public virtual void Pause () { if (hasInitDone) webCamTexture.Pause (); } /// /// Stops the active camera. /// public virtual void Stop () { if (hasInitDone) webCamTexture.Stop (); } /// /// Indicates whether the active camera is currently playing. /// /// true, if the active camera is playing, false otherwise. public virtual bool IsPlaying () { return hasInitDone ? webCamTexture.isPlaying : false; } /// /// Indicates whether the active camera device is currently front facng. /// /// true, if the active camera device is front facng, false otherwise. public virtual bool IsFrontFacing () { return hasInitDone ? webCamDevice.isFrontFacing : false; } /// /// Returns the active camera device name. /// /// The active camera device name. public virtual string GetDeviceName () { return hasInitDone ? webCamTexture.deviceName : ""; } /// /// Returns the active camera width. /// /// The active camera width. public virtual int GetWidth () { if (!hasInitDone) return -1; return (rotatedFrameMat != null) ? frameMat.height () : frameMat.width (); } /// /// Returns the active camera height. /// /// The active camera height. public virtual int GetHeight () { if (!hasInitDone) return -1; return (rotatedFrameMat != null) ? frameMat.width () : frameMat.height (); } /// /// Returns the active camera framerate. /// /// The active camera framerate. public virtual float GetFPS () { return hasInitDone ? webCamTexture.requestedFPS : -1f; } /// /// Returns the active WebcamTexture. /// /// The active WebcamTexture. public virtual WebCamTexture GetWebCamTexture () { return hasInitDone ? webCamTexture : null; } /// /// Returns the active WebcamDevice. /// /// The active WebcamDevice. public virtual WebCamDevice GetWebCamDevice () { return webCamDevice; } /// /// Returns the camera to world matrix. /// /// The camera to world matrix. public virtual Matrix4x4 GetCameraToWorldMatrix () { return Camera.main.cameraToWorldMatrix; } /// /// Returns the projection matrix matrix. /// /// The projection matrix. public virtual Matrix4x4 GetProjectionMatrix () { return Camera.main.projectionMatrix; } /// /// Indicates whether the video buffer of the frame has been updated. /// /// true, if the video buffer has been updated false otherwise. public virtual bool DidUpdateThisFrame () { if (!hasInitDone) return false; #if UNITY_IOS && !UNITY_EDITOR && (UNITY_4_6_3 || UNITY_4_6_4 || UNITY_5_0_0 || UNITY_5_0_1) if (webCamTexture.width > 16 && webCamTexture.height > 16) { return true; } else { return false; } #else return webCamTexture.didUpdateThisFrame; #endif } /// /// Gets the mat of the current frame. /// The Mat object's type is 'CV_8UC4' (RGBA). /// /// The mat of the current frame. public virtual Mat GetMat () { if (!hasInitDone || !webCamTexture.isPlaying) { return (rotatedFrameMat != null) ? rotatedFrameMat : frameMat; } Utils.webCamTextureToMat (webCamTexture, frameMat, colors, false); #if !UNITY_EDITOR && !(UNITY_STANDALONE || UNITY_WEBGL) if (rotatedFrameMat != null) { if (screenOrientation == ScreenOrientation.Portrait || screenOrientation == ScreenOrientation.PortraitUpsideDown) { // (Orientation is Portrait, rotate90Degree is false) if (webCamDevice.isFrontFacing) { FlipMat (frameMat, !flipHorizontal, !flipVertical); } else { FlipMat (frameMat, flipHorizontal, flipVertical); } } else { // (Orientation is Landscape, rotate90Degrees=true) FlipMat (frameMat, flipVertical, flipHorizontal); } Core.rotate (frameMat, rotatedFrameMat, Core.ROTATE_90_CLOCKWISE); return rotatedFrameMat; } else { if (screenOrientation == ScreenOrientation.Portrait || screenOrientation == ScreenOrientation.PortraitUpsideDown) { // (Orientation is Portrait, rotate90Degree is ture) if (webCamDevice.isFrontFacing) { FlipMat (frameMat, flipHorizontal, flipVertical); } else { FlipMat (frameMat, !flipHorizontal, !flipVertical); } } else { // (Orientation is Landscape, rotate90Degree is false) FlipMat (frameMat, flipVertical, flipHorizontal); } return frameMat; } #else FlipMat (frameMat, flipVertical, flipHorizontal); if (rotatedFrameMat != null) { Core.rotate (frameMat, rotatedFrameMat, Core.ROTATE_90_CLOCKWISE); return rotatedFrameMat; } else { return frameMat; } #endif } /// /// Flips the mat. /// /// Mat. protected virtual void FlipMat (Mat mat, bool flipVertical, bool flipHorizontal) { //Since the order of pixels of WebCamTexture and Mat is opposite, the initial value of flipCode is set to 0 (flipVertical). int flipCode = 0; if (webCamDevice.isFrontFacing) { if (webCamTexture.videoRotationAngle == 0) { flipCode = -1; } else if (webCamTexture.videoRotationAngle == 90) { flipCode = -1; } if (webCamTexture.videoRotationAngle == 180) { flipCode = int.MinValue; } else if (webCamTexture.videoRotationAngle == 270) { flipCode = int.MinValue; } } else { if (webCamTexture.videoRotationAngle == 180) { flipCode = 1; } else if (webCamTexture.videoRotationAngle == 270) { flipCode = 1; } } if (flipVertical) { if (flipCode == int.MinValue) { flipCode = 0; } else if (flipCode == 0) { flipCode = int.MinValue; } else if (flipCode == 1) { flipCode = -1; } else if (flipCode == -1) { flipCode = 1; } } if (flipHorizontal) { if (flipCode == int.MinValue) { flipCode = 1; } else if (flipCode == 0) { flipCode = -1; } else if (flipCode == 1) { flipCode = int.MinValue; } else if (flipCode == -1) { flipCode = 0; } } if (flipCode > int.MinValue) { Core.flip (mat, mat, flipCode); } } /// /// Gets the buffer colors. /// /// The buffer colors. public virtual Color32[] GetBufferColors () { return colors; } /// /// Cancel Init Coroutine. /// protected virtual void CancelInitCoroutine () { if (initCoroutine != null) { StopCoroutine (initCoroutine); ((IDisposable)initCoroutine).Dispose (); initCoroutine = null; } } /// /// To release the resources. /// protected virtual void ReleaseResources () { isInitWaiting = false; hasInitDone = false; if (webCamTexture != null) { webCamTexture.Stop (); WebCamTexture.Destroy (webCamTexture); webCamTexture = null; } if (frameMat != null) { frameMat.Dispose (); frameMat = null; } if (rotatedFrameMat != null) { rotatedFrameMat.Dispose (); rotatedFrameMat = null; } } /// /// Releases all resource used by the object. /// /// Call when you are finished using the . The /// method leaves the in an unusable state. After /// calling , you must release all references to the so /// the garbage collector can reclaim the memory that the was occupying. public virtual void Dispose () { if (colors != null) colors = null; if (isInitWaiting) { CancelInitCoroutine (); ReleaseResources (); } else if (hasInitDone) { ReleaseResources (); if (onDisposed != null) onDisposed.Invoke (); } } } }