using OpenCVForUnity.BgsegmModule; using OpenCVForUnity.CoreModule; using OpenCVForUnity.ImgprocModule; using OpenCVForUnity.UnityUtils; using OpenCVForUnity.UnityUtils.Helper; using OpenCVForUnity.VideoModule; using System; using UnityEngine; using UnityEngine.SceneManagement; using UnityEngine.UI; namespace OpenCVForUnityExample { /// /// Background Subtractor Example /// This example is intended for comparison of different background subtraction algorithms presented in OpenCV. /// [RequireComponent(typeof(VideoCaptureToMatHelper))] public class BackgroundSubtractorExample : MonoBehaviour { /// /// The background subtractor algorithm dropdown. /// public Dropdown backgroundSubtractorAlgorithmDropdown; /// /// The background subtractor algorithm. /// public BackgroundSubtractorAlgorithmPreset backgroundSubtractorAlgorithm = BackgroundSubtractorAlgorithmPreset.KNN; /// /// The enable MorphologyEx toggle. /// public Toggle enableMorphologyExToggle; /// /// The show background image toggle. /// public Toggle showBackgroundImageToggle; /// /// The texture. /// Texture2D texture; /// /// The video capture to mat helper. /// VideoCaptureToMatHelper sourceToMatHelper; /// /// The background substractor. /// BackgroundSubtractor backgroundSubstractor; /// /// The fgmask mat. /// Mat fgmaskMat; /// /// The kernel for morphologyEx method. /// Mat kernel; /// /// The stopwatch. /// System.Diagnostics.Stopwatch watch; /// /// The FPS monitor. /// FpsMonitor fpsMonitor; /// /// VIDEO_FILENAME /// protected static readonly string VIDEO_FILENAME = "OpenCVForUnity/768x576_mjpeg.mjpeg"; // Use this for initialization void Start() { fpsMonitor = GetComponent(); sourceToMatHelper = gameObject.GetComponent(); if (string.IsNullOrEmpty(sourceToMatHelper.requestedVideoFilePath)) sourceToMatHelper.requestedVideoFilePath = VIDEO_FILENAME; sourceToMatHelper.outputColorFormat = VideoCaptureToMatHelper.ColorFormat.RGB; // Background Subtractor API must handle 3 channels Mat image. sourceToMatHelper.Initialize(); // Update GUI state backgroundSubtractorAlgorithmDropdown.value = Array.IndexOf(System.Enum.GetNames(typeof(BackgroundSubtractorAlgorithmPreset)), backgroundSubtractorAlgorithm.ToString()); enableMorphologyExToggle.isOn = false; showBackgroundImageToggle.isOn = false; CreateBackgroundSubstractor(backgroundSubtractorAlgorithm); kernel = Imgproc.getStructuringElement(Imgproc.MORPH_ELLIPSE, new Size(3, 3)); watch = new System.Diagnostics.Stopwatch(); if (fpsMonitor != null) { fpsMonitor.Add("time: ", ""); } } protected void CreateBackgroundSubstractor(BackgroundSubtractorAlgorithmPreset algorithm) { if (backgroundSubstractor != null) backgroundSubstractor.Dispose(); switch (algorithm) { case BackgroundSubtractorAlgorithmPreset.KNN: backgroundSubstractor = Video.createBackgroundSubtractorKNN(); //BackgroundSubtractorKNN subtractorKNN = (BackgroundSubtractorKNN)backgroundSubstractor; //subtractorKNN.setDetectShadows(true); //subtractorKNN.setDist2Threshold(400); //subtractorKNN.setHistory(500); //subtractorKNN.setkNNSamples(2); //subtractorKNN.setNSamples(7); //subtractorKNN.setShadowThreshold(0.5); //subtractorKNN.setShadowValue(127); break; case BackgroundSubtractorAlgorithmPreset.MOG2: backgroundSubstractor = Video.createBackgroundSubtractorMOG2(); //BackgroundSubtractorMOG2 subtractorMOG2 = (BackgroundSubtractorMOG2)backgroundSubstractor; //subtractorMOG2.setBackgroundRatio(0.899999976158142); //subtractorMOG2.setComplexityReductionThreshold(0.0500000007450581); //subtractorMOG2.setDetectShadows(true); //subtractorMOG2.setHistory(500); //subtractorMOG2.setNMixtures(5); //subtractorMOG2.setShadowThreshold(0.5); //subtractorMOG2.setShadowValue(127); //subtractorMOG2.setVarInit(15); //subtractorMOG2.setVarMax(75); //subtractorMOG2.setVarMin(4); //subtractorMOG2.setVarThreshold(16); //subtractorMOG2.setVarThresholdGen(9); break; case BackgroundSubtractorAlgorithmPreset.CNT: backgroundSubstractor = Bgsegm.createBackgroundSubtractorCNT(); //BackgroundSubtractorCNT subtractorCNT = (BackgroundSubtractorCNT)backgroundSubstractor; //subtractorCNT.setIsParallel(true); //subtractorCNT.setMaxPixelStability(900) //subtractorCNT.setMinPixelStability(15); //subtractorCNT.setUseHistory(true); break; case BackgroundSubtractorAlgorithmPreset.GMG: backgroundSubstractor = Bgsegm.createBackgroundSubtractorGMG(); break; case BackgroundSubtractorAlgorithmPreset.GSOC: backgroundSubstractor = Bgsegm.createBackgroundSubtractorGSOC(); break; case BackgroundSubtractorAlgorithmPreset.LSBP: backgroundSubstractor = Bgsegm.createBackgroundSubtractorLSBP(); break; case BackgroundSubtractorAlgorithmPreset.MOG: backgroundSubstractor = Bgsegm.createBackgroundSubtractorMOG(); break; default: break; } } /// /// Raises the video capture to mat helper initialized event. /// public void OnVideoCaptureToMatHelperInitialized() { Debug.Log("OnVideoCaptureToMatHelperInitialized"); Mat rgbMat = sourceToMatHelper.GetMat(); texture = new Texture2D(rgbMat.cols(), rgbMat.rows(), TextureFormat.RGB24, false); Utils.matToTexture2D(rgbMat, texture); gameObject.GetComponent().material.mainTexture = texture; gameObject.transform.localScale = new Vector3(rgbMat.cols(), rgbMat.rows(), 1); Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation); float width = rgbMat.width(); float height = rgbMat.height(); float widthScale = (float)Screen.width / width; float heightScale = (float)Screen.height / height; if (widthScale < heightScale) { Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2; } else { Camera.main.orthographicSize = height / 2; } fgmaskMat = new Mat(rgbMat.rows(), rgbMat.cols(), CvType.CV_8UC1); } /// /// Raises the video capture to mat helper disposed event. /// public void OnVideoCaptureToMatHelperDisposed() { Debug.Log("OnVideoCaptureToMatHelperDisposed"); if (backgroundSubstractor != null) backgroundSubstractor.clear(); if (fgmaskMat != null) fgmaskMat.Dispose(); if (texture != null) { Texture2D.Destroy(texture); texture = null; } } /// /// Raises the video capture to mat helper error occurred event. /// /// Error code. public void OnVideoCaptureToMatHelperErrorOccurred(VideoCaptureToMatHelper.ErrorCode errorCode) { Debug.Log("OnVideoCaptureToMatHelperErrorOccurred " + errorCode); if (fpsMonitor != null) { fpsMonitor.consoleText = "ErrorCode: " + errorCode; } } // Update is called once per frame void Update() { if (sourceToMatHelper.IsPlaying() && sourceToMatHelper.DidUpdateThisFrame()) { Mat rgbMat = sourceToMatHelper.GetMat(); watch.Reset(); watch.Start(); backgroundSubstractor.apply(rgbMat, fgmaskMat); if (enableMorphologyExToggle.isOn) Imgproc.morphologyEx(fgmaskMat, fgmaskMat, Imgproc.MORPH_OPEN, kernel); watch.Stop(); if (showBackgroundImageToggle.isOn && backgroundSubtractorAlgorithm != BackgroundSubtractorAlgorithmPreset.GMG) { if (backgroundSubtractorAlgorithm == BackgroundSubtractorAlgorithmPreset.CNT) { backgroundSubstractor.getBackgroundImage(fgmaskMat); Imgproc.cvtColor(fgmaskMat, rgbMat, Imgproc.COLOR_GRAY2RGB); } else { backgroundSubstractor.getBackgroundImage(rgbMat); } } else { Imgproc.cvtColor(fgmaskMat, rgbMat, Imgproc.COLOR_GRAY2RGB); } Utils.matToTexture2D(rgbMat, texture); if (fpsMonitor != null) { fpsMonitor.Add("time: ", watch.ElapsedMilliseconds + " ms"); } } } /// /// Raises the destroy event. /// void OnDestroy() { if (backgroundSubstractor != null) backgroundSubstractor.Dispose(); if (sourceToMatHelper != null) sourceToMatHelper.Dispose(); } /// /// Raises the back button click event. /// public void OnBackButtonClick() { SceneManager.LoadScene("OpenCVForUnityExample"); } /// /// Raises the background subtractor algorithm dropdown value changed event. /// public void OnBackgroundSubtractorAlgorithmDropdownValueChanged(int result) { string[] enumNames = Enum.GetNames(typeof(BackgroundSubtractorAlgorithmPreset)); byte value = (byte)System.Enum.Parse(typeof(BackgroundSubtractorAlgorithmPreset), enumNames[result], true); if ((byte)backgroundSubtractorAlgorithm != value) { backgroundSubtractorAlgorithm = (BackgroundSubtractorAlgorithmPreset)value; } //Debug.Log((int)backgroundSubtractorAlgorithm); CreateBackgroundSubstractor(backgroundSubtractorAlgorithm); sourceToMatHelper.Initialize(); } /// /// Raises the enable MorphologyEx toggle value changed event. /// public void OnEnableMorphologyExToggleValueChanged() { // //Debug.Log(enableMorphologyExToggleToggle.isOn); } /// /// Raises the show background image toggle value changed event. /// public void OnShowBackgroundImageToggleValueChanged() { // //Debug.Log(showBackgroundImageToggle.isOn); } public enum BackgroundSubtractorAlgorithmPreset : byte { KNN = 0, MOG2, CNT, GMG, GSOC, LSBP, MOG, } } }