GHZMRSpaceXR/Assets/OpenCVForUnity/Examples/ContribModules/aruco/ArUcoExample/ArUcoCameraCalibrationExample.cs

using UnityEngine;
using UnityEngine.UI;
using UnityEngine.SceneManagement;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Xml.Serialization;
using System.IO;
using System.Linq;
using OpenCVForUnity.ArucoModule;
using OpenCVForUnity.CoreModule;
using OpenCVForUnity.ImgprocModule;
using OpenCVForUnity.Calib3dModule;
using OpenCVForUnity.UnityUtils;
using OpenCVForUnity.ImgcodecsModule;
using OpenCVForUnity.UnityUtils.Helper;

namespace OpenCVForUnityExample
{
    /// <summary>
    /// ArUco Camera Calibration Example
    /// An example of camera calibration using the aruco module.
    /// Referring to https://github.com/opencv/opencv_contrib/blob/master/modules/aruco/samples/calibrate_camera.cpp.
    /// https://github.com/opencv/opencv/blob/master/samples/cpp/tutorial_code/calib3d/camera_calibration/camera_calibration.cpp
    /// https://docs.opencv.org/3.2.0/da/d13/tutorial_aruco_calibration.html
    /// https://docs.opencv.org/3.4.0/d7/d21/tutorial_interactive_calibration.html
    /// </summary>
    [RequireComponent (typeof(WebCamTextureToMatHelper))]
    public class ArUcoCameraCalibrationExample : MonoBehaviour
    {
        /// <summary>
        /// The marker type.
        /// </summary>
        public MarkerType markerType = MarkerType.ChArUcoBoard;

        /// <summary>
        /// The marker type dropdown.
        /// </summary>
        public Dropdown markerTypeDropdown;

        /// <summary>
        /// The dictionary identifier.
        /// </summary>
        public ArUcoDictionary dictionaryId = ArUcoDictionary.DICT_6X6_250;

        /// <summary>
        /// The dictionary id dropdown.
        /// </summary>
        public Dropdown dictionaryIdDropdown;

        /// <summary>
        /// Number of squares in X direction.
        /// </summary>
        public NumberOfSquaresX squaresX = NumberOfSquaresX.X_5;

        /// <summary>
        /// The squares X dropdown.
        /// </summary>
        public Dropdown squaresXDropdown;

        /// <summary>
        /// Number of squares in X direction.
        /// </summary>
        public NumberOfSquaresY squaresY = NumberOfSquaresY.Y_7;

        /// <summary>
        /// The squares X dropdown.
        /// </summary>
        public Dropdown squaresYDropdown;

        /// <summary>
        /// The save path input field.
        /// </summary>
        public InputField savePathInputField;

        /// <summary>
        /// Determines if refine marker detection. (only valid for ArUco boards)
        /// </summary>
        public bool refineMarkerDetection = true;


        [Header ("Extra Option")]

        /// <summary>
        /// Determines if calibrates camera using the list of calibration images.
        /// </summary>
        [TooltipAttribute ("Determines if calibrates camera using the list of calibration images.")]
        public bool isImagesInputMode = false;

        /// <summary>
        /// The calibration images directory path.
        /// Set a relative directory path from the starting point of the "StreamingAssets" folder.  e.g. "calibration_images/".
        /// </summary>
        [TooltipAttribute ("Set a relative directory path from the starting point of the \"StreamingAssets\" folder.  e.g. \"calibration_images\"")]
        public string calibrationImagesDirectory = "calibration_images";

        /// <summary>
        /// The texture.
        /// </summary>
        Texture2D texture;
        
        /// <summary>
        /// The webcam texture to mat helper.
        /// </summary>
        WebCamTextureToMatHelper webCamTextureToMatHelper;

        /// <summary>
        /// The gray mat.
        /// </summary>
        Mat grayMat;

        /// <summary>
        /// The bgr mat.
        /// </summary>
        Mat bgrMat;

        /// <summary>
        /// The rgba mat.
        /// </summary>
        Mat rgbaMat;
        
        /// <summary>
        /// The cameraparam matrix.
        /// </summary>
        Mat camMatrix;
        
        /// <summary>
        /// The distortion coeffs.
        /// </summary>
        MatOfDouble distCoeffs;

        /// <summary>
        /// The identifiers.
        /// </summary>
        Mat ids;
        
        /// <summary>
        /// The corners.
        /// </summary>
        List<Mat> corners;
        
        /// <summary>
        /// The rejected corners.
        /// </summary>
        List<Mat> rejectedCorners;
        
        /// <summary>
        /// The rvecs.
        /// </summary>
        List<Mat> rvecs;
        
        /// <summary>
        /// The tvecs.
        /// </summary>
        List<Mat> tvecs;
        
        /// <summary>
        /// The detector parameters.
        /// </summary>
        DetectorParameters detectorParams;
        
        /// <summary>
        /// The dictionary.
        /// </summary>
        Dictionary dictionary;

        /// <summary>
        /// The recovered identifiers.
        /// </summary>
        Mat recoveredIdxs;

        const int calibrationFlags = 0;
        // Calib3d.CALIB_FIX_K3 | Calib3d.CALIB_FIX_K4 | Calib3d.CALIB_FIX_K5
        double repErr = 0;
        bool shouldCaptureFrame = false;

        // for ChArUcoBoard.
        // chessboard square side length (normally in meters)
        const float chArUcoBoradSquareLength = 0.04f;
        // marker side length (same unit than squareLength)
        const float chArUcoBoradMarkerLength = 0.02f;
        const int charucoMinMarkers = 2;
        Mat charucoCorners;
        Mat charucoIds;
        CharucoBoard charucoBoard;
        List<List<Mat>> allCorners;
        List<Mat> allIds;
        List<Mat> allImgs;

        // for OthearMarkers.
        // square size in some user-defined units (1 by default)
        const float squareSize = 1f;
        List<Mat> imagePoints;
        bool isInitialized = false;
        bool isCalibrating = false;

        // Use this for initialization
        IEnumerator Start ()
        {
            webCamTextureToMatHelper = gameObject.GetComponent<WebCamTextureToMatHelper> ();

            // fix the screen orientation.
            Screen.orientation = ScreenOrientation.LandscapeLeft;
            // wait for the screen orientation to change.
            yield return null;

            markerTypeDropdown.value = (int)markerType;
            dictionaryIdDropdown.value = (int)dictionaryId;
            squaresXDropdown.value = (int)squaresX - 1;
            squaresYDropdown.value = (int)squaresY - 1;
            dictionaryIdDropdown.interactable = (markerType == MarkerType.ChArUcoBoard);

            #if UNITY_WEBGL && !UNITY_EDITOR
            isImagesInputMode = false;
            #endif
            if (isImagesInputMode) {
                isImagesInputMode = InitializeImagesInputMode ();
            }

            if (!isImagesInputMode) {                
                #if UNITY_ANDROID && !UNITY_EDITOR
                // Avoids the front camera low light issue that occurs in only some Android devices (e.g. Google Pixel, Pixel2).
                webCamTextureToMatHelper.avoidAndroidFrontCameraLowLightIssue = true;
                #endif
                webCamTextureToMatHelper.Initialize ();
            }
        }

        /// <summary>
        /// Raises the webcam texture to mat helper initialized event.
        /// </summary>
        public void OnWebCamTextureToMatHelperInitialized ()
        {
            Debug.Log ("OnWebCamTextureToMatHelperInitialized");
            
            Mat webCamTextureMat = webCamTextureToMatHelper.GetMat ();
            
            InitializeCalibraton (webCamTextureMat);

            // if WebCamera is frontFaceing, flip Mat.
            if (webCamTextureToMatHelper.GetWebCamDevice ().isFrontFacing) {
                webCamTextureToMatHelper.flipHorizontal = true;
            }
        }

        /// <summary>
        /// Raises the webcam texture to mat helper disposed event.
        /// </summary>
        public void OnWebCamTextureToMatHelperDisposed ()
        {
            Debug.Log ("OnWebCamTextureToMatHelperDisposed");

            DisposeCalibraton ();
        }

        /// <summary>
        /// Raises the webcam texture to mat helper error occurred event.
        /// </summary>
        /// <param name="errorCode">Error code.</param>
        public void OnWebCamTextureToMatHelperErrorOccurred (WebCamTextureToMatHelper.ErrorCode errorCode)
        {
            Debug.Log ("OnWebCamTextureToMatHelperErrorOccurred " + errorCode);
        }
        
        // Update is called once per frame
        void Update ()
        {
            if (isImagesInputMode)
                return;

            if (webCamTextureToMatHelper.IsPlaying () && webCamTextureToMatHelper.DidUpdateThisFrame ()) {

                Mat rgbaMat = webCamTextureToMatHelper.GetMat ();

                Imgproc.cvtColor (rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);

                if (shouldCaptureFrame) {
                    shouldCaptureFrame = false;
                    Mat frameMat = grayMat.clone ();
                    double e = CaptureFrame (frameMat);
                    if (e > 0)
                        repErr = e;
                }

                DrawFrame (grayMat, bgrMat);
                Imgproc.cvtColor (bgrMat, rgbaMat, Imgproc.COLOR_BGR2RGBA);

                Utils.fastMatToTexture2D (rgbaMat, texture);
            }
        }

        private void InitializeCalibraton (Mat frameMat)
        {
            texture = new Texture2D (frameMat.cols (), frameMat.rows (), TextureFormat.RGBA32, false);

            gameObject.GetComponent<Renderer> ().material.mainTexture = texture;

            gameObject.transform.localScale = new Vector3 (frameMat.cols (), frameMat.rows (), 1);
            Debug.Log ("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);


            float width = frameMat.width ();
            float height = frameMat.height ();

            float imageSizeScale = 1.0f;
            float widthScale = (float)Screen.width / width;
            float heightScale = (float)Screen.height / height;
            if (widthScale < heightScale) {
                Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2;
                imageSizeScale = (float)Screen.height / (float)Screen.width;
            } else {
                Camera.main.orthographicSize = height / 2;
            }


            // set cameraparam.
            camMatrix = CreateCameraMatrix (width, height);
            Debug.Log ("camMatrix " + camMatrix.dump ());            

            distCoeffs = new MatOfDouble (0, 0, 0, 0, 0);
            Debug.Log ("distCoeffs " + distCoeffs.dump ());            

            // calibration camera.
            Size imageSize = new Size (width * imageSizeScale, height * imageSizeScale);
            double apertureWidth = 0;
            double apertureHeight = 0;
            double[] fovx = new double[1];
            double[] fovy = new double[1];
            double[] focalLength = new double[1];
            Point principalPoint = new Point (0, 0);
            double[] aspectratio = new double[1];

            Calib3d.calibrationMatrixValues (camMatrix, imageSize, apertureWidth, apertureHeight, fovx, fovy, focalLength, principalPoint, aspectratio);

            Debug.Log ("imageSize " + imageSize.ToString ());
            Debug.Log ("apertureWidth " + apertureWidth);
            Debug.Log ("apertureHeight " + apertureHeight);
            Debug.Log ("fovx " + fovx [0]);
            Debug.Log ("fovy " + fovy [0]);
            Debug.Log ("focalLength " + focalLength [0]);
            Debug.Log ("principalPoint " + principalPoint.ToString ());
            Debug.Log ("aspectratio " + aspectratio [0]);


            grayMat = new Mat (frameMat.rows (), frameMat.cols (), CvType.CV_8UC1);
            bgrMat = new Mat (frameMat.rows (), frameMat.cols (), CvType.CV_8UC3);
            rgbaMat = new Mat (frameMat.rows (), frameMat.cols (), CvType.CV_8UC4);
            ids = new Mat ();
            corners = new List<Mat> ();
            rejectedCorners = new List<Mat> ();
            rvecs = new List<Mat> ();
            tvecs = new List<Mat> ();

            detectorParams = DetectorParameters.create ();
            detectorParams.set_cornerRefinementMethod (1);// do cornerSubPix() of OpenCV.
            dictionary = Aruco.getPredefinedDictionary ((int)dictionaryId);

            recoveredIdxs = new Mat ();

            charucoCorners = new Mat ();
            charucoIds = new Mat ();
            charucoBoard = CharucoBoard.create ((int)squaresX, (int)squaresY, chArUcoBoradSquareLength, chArUcoBoradMarkerLength, dictionary);


            allCorners = new List<List<Mat>> ();
            allIds = new List<Mat> ();
            allImgs = new List<Mat> ();

            imagePoints = new List<Mat> ();

            isInitialized = true;
        }

        private void DisposeCalibraton ()
        {
            ResetCalibration ();

            if (grayMat != null)
                grayMat.Dispose ();
            if (bgrMat != null)
                bgrMat.Dispose ();
            if (rgbaMat != null)
                rgbaMat.Dispose ();
            
            if (texture != null) {
                Texture2D.Destroy (texture);
                texture = null;
            }

            if (ids != null)
                ids.Dispose ();
            foreach (var item in corners) {
                item.Dispose ();
            }
            corners.Clear ();
            foreach (var item in rejectedCorners) {
                item.Dispose ();
            }
            rejectedCorners.Clear ();
            foreach (var item in rvecs) {
                item.Dispose ();
            }
            rvecs.Clear ();
            foreach (var item in tvecs) {
                item.Dispose ();
            }
            tvecs.Clear ();

            if (recoveredIdxs != null)
                recoveredIdxs.Dispose (); 

            if (charucoCorners != null)
                charucoCorners.Dispose ();
            if (charucoIds != null)
                charucoIds.Dispose ();
            if (charucoBoard != null)
                charucoBoard.Dispose ();

            isInitialized = false;
        }

        private void DrawFrame (Mat grayMat, Mat bgrMat)
        {
            Imgproc.cvtColor (grayMat, bgrMat, Imgproc.COLOR_GRAY2BGR);

            switch (markerType) {
            default:
            case MarkerType.ChArUcoBoard:
                // detect markers.
                Aruco.detectMarkers (grayMat, dictionary, corners, ids, detectorParams, rejectedCorners, camMatrix, distCoeffs);

                // refine marker detection.
                if (refineMarkerDetection) {
                    Aruco.refineDetectedMarkers (grayMat, charucoBoard, corners, ids, rejectedCorners, camMatrix, distCoeffs, 10f, 3f, true, recoveredIdxs, detectorParams);
                }

                // if at least one marker detected
                if (ids.total () > 0) {
                    Aruco.interpolateCornersCharuco (corners, ids, grayMat, charucoBoard, charucoCorners, charucoIds, camMatrix, distCoeffs, charucoMinMarkers);

                    // draw markers.
                    Aruco.drawDetectedMarkers (bgrMat, corners, ids, new Scalar (0, 255, 0, 255));
                    // if at least one charuco corner detected
                    if (charucoIds.total () > 0) {
                        Aruco.drawDetectedCornersCharuco (bgrMat, charucoCorners, charucoIds, new Scalar (0, 0, 255, 255));
                    }
                }
                break;
            case MarkerType.ChessBoard:
            case MarkerType.CirclesGlid:
            case MarkerType.AsymmetricCirclesGlid:
                // detect markers.
                MatOfPoint2f points = new MatOfPoint2f ();
                bool found = false;

                switch (markerType) {
                default:
                case MarkerType.ChessBoard:
                    found = Calib3d.findChessboardCorners (grayMat, new Size ((int)squaresX, (int)squaresY), points, Calib3d.CALIB_CB_ADAPTIVE_THRESH | Calib3d.CALIB_CB_FAST_CHECK | Calib3d.CALIB_CB_NORMALIZE_IMAGE);
                    break;
                case MarkerType.CirclesGlid:
                    found = Calib3d.findCirclesGrid (grayMat, new Size ((int)squaresX, (int)squaresY), points, Calib3d.CALIB_CB_SYMMETRIC_GRID);
                    break;
                case MarkerType.AsymmetricCirclesGlid:
                    found = Calib3d.findCirclesGrid (grayMat, new Size ((int)squaresX, (int)squaresY), points, Calib3d.CALIB_CB_ASYMMETRIC_GRID);
                    break;
                }

                if (found) {
                    if (markerType == MarkerType.ChessBoard)
                        Imgproc.cornerSubPix (grayMat, points, new Size (5, 5), new Size (-1, -1), new TermCriteria (TermCriteria.EPS + TermCriteria.COUNT, 30, 0.1));

                    // draw markers.
                    Calib3d.drawChessboardCorners (bgrMat, new Size ((int)squaresX, (int)squaresY), points, found);
                }
                break;
            }

            double[] camMatrixArr = new double[(int)camMatrix.total ()];
            camMatrix.get (0, 0, camMatrixArr);
            double[] distCoeffsArr = new double[(int)distCoeffs.total ()];
            distCoeffs.get (0, 0, distCoeffsArr);

            int ff = Imgproc.FONT_HERSHEY_SIMPLEX;
            double fs = 0.4;
            Scalar c = new Scalar (255, 255, 255, 255);
            int t = 0;
            int lt = Imgproc.LINE_AA;
            bool blo = false;
            int frameCount = (markerType == MarkerType.ChArUcoBoard) ? allCorners.Count : imagePoints.Count;
            Imgproc.putText (bgrMat, frameCount + " FRAME CAPTURED", new Point (bgrMat.cols () - 300, 20), ff, fs, c, t, lt, blo);
            Imgproc.putText (bgrMat, "IMAGE_WIDTH: " + bgrMat.width (), new Point (bgrMat.cols () - 300, 40), ff, fs, c, t, lt, blo);
            Imgproc.putText (bgrMat, "IMAGE_HEIGHT: " + bgrMat.height (), new Point (bgrMat.cols () - 300, 60), ff, fs, c, t, lt, blo);
            Imgproc.putText (bgrMat, "CALIBRATION_FLAGS: " + calibrationFlags, new Point (bgrMat.cols () - 300, 80), ff, fs, c, t, lt, blo);

            Imgproc.putText (bgrMat, "CAMERA_MATRIX: ", new Point (bgrMat.cols () - 300, 100), ff, fs, c, t, lt, blo);
            for (int i = 0; i < camMatrixArr.Length; i = i + 3) {
                Imgproc.putText (bgrMat, "   " + camMatrixArr [i] + ", " + camMatrixArr [i + 1] + ", " + camMatrixArr [i + 2] + ",", new Point (bgrMat.cols () - 300, 120 + 20 * i / 3), ff, fs, c, t, lt, blo);
            }
            Imgproc.putText (bgrMat, "DISTORTION_COEFFICIENTS: ", new Point (bgrMat.cols () - 300, 180), ff, fs, c, t, lt, blo);
            for (int i = 0; i < distCoeffsArr.Length; ++i) {
                Imgproc.putText (bgrMat, "   " + distCoeffsArr [i] + ",", new Point (bgrMat.cols () - 300, 200 + 20 * i), ff, fs, c, t, lt, blo);
            }
            Imgproc.putText (bgrMat, "AVG_REPROJECTION_ERROR: " + repErr, new Point (bgrMat.cols () - 300, 300), ff, fs, c, t, lt, blo);

            if (frameCount == 0)
                Imgproc.putText (bgrMat, "Please press the capture button to start!", new Point (5, bgrMat.rows () - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 1, Imgproc.LINE_AA, false);
        }

        private double CaptureFrame (Mat frameMat)
        {
            double repErr = -1;

            switch (markerType) {
            default:
            case MarkerType.ChArUcoBoard:
                List<Mat> corners = new List<Mat> ();
                Mat ids = new Mat ();
                Aruco.detectMarkers (frameMat, dictionary, corners, ids, detectorParams, rejectedCorners, camMatrix, distCoeffs);

                if (refineMarkerDetection) {
                    Aruco.refineDetectedMarkers (frameMat, charucoBoard, corners, ids, rejectedCorners, camMatrix, distCoeffs, 10f, 3f, true, recoveredIdxs, detectorParams);
                }

                if (ids.total () > 0) {
                    Debug.Log ("Frame captured.");

                    allCorners.Add (corners);
                    allIds.Add (ids);
                    allImgs.Add (frameMat);
                } else {

                    Debug.Log ("Invalid frame.");

                    frameMat.Dispose ();
                    if (ids != null)
                        ids.Dispose ();
                    foreach (var item in corners) {
                        item.Dispose ();
                    }
                    corners.Clear ();

                    return -1;
                }

                // calibrate camera using aruco markers
                //double arucoRepErr = CalibrateCameraAruco (allCorners, allIds, charucoBoard, frameMat.size(), camMatrix, distCoeffs, rvecs, tvecs, calibrationFlags);
                //Debug.Log ("arucoRepErr: " + arucoRepErr);

                // calibrate camera using charuco
                repErr = CalibrateCameraCharuco (allCorners, allIds, charucoBoard, frameMat.size (), camMatrix, distCoeffs, rvecs, tvecs, calibrationFlags, calibrationFlags);

                break;
            case MarkerType.ChessBoard:
            case MarkerType.CirclesGlid:
            case MarkerType.AsymmetricCirclesGlid:
                
                MatOfPoint2f points = new MatOfPoint2f ();
                Size patternSize = new Size ((int)squaresX, (int)squaresY);

                bool found = false;
                switch (markerType) {
                default:
                case MarkerType.ChessBoard:
                    found = Calib3d.findChessboardCorners (frameMat, patternSize, points, Calib3d.CALIB_CB_ADAPTIVE_THRESH | Calib3d.CALIB_CB_FAST_CHECK | Calib3d.CALIB_CB_NORMALIZE_IMAGE);
                    break;
                case MarkerType.CirclesGlid:
                    found = Calib3d.findCirclesGrid (frameMat, patternSize, points, Calib3d.CALIB_CB_SYMMETRIC_GRID);
                    break;
                case MarkerType.AsymmetricCirclesGlid:
                    found = Calib3d.findCirclesGrid (frameMat, patternSize, points, Calib3d.CALIB_CB_ASYMMETRIC_GRID);
                    break;
                }

                if (found) {
                    Debug.Log ("Frame captured.");
                    if (markerType == MarkerType.ChessBoard)
                        Imgproc.cornerSubPix (frameMat, points, new Size (5, 5), new Size (-1, -1), new TermCriteria (TermCriteria.EPS + TermCriteria.COUNT, 30, 0.1));
                    
                    imagePoints.Add (points);
                    allImgs.Add (frameMat);
                } else {
                    Debug.Log ("Invalid frame.");
                    frameMat.Dispose ();
                    if (points != null)
                        points.Dispose ();
                    return -1;
                }
                    
                if (imagePoints.Count < 1) {
                    Debug.Log ("Not enough points for calibration.");
                    repErr = -1;
                } else {

                    MatOfPoint3f objectPoint = new MatOfPoint3f (new Mat (imagePoints [0].rows (), 1, CvType.CV_32FC3));
                    CalcChessboardCorners (patternSize, squareSize, objectPoint, markerType);

                    List<Mat> objectPoints = new List<Mat> ();
                    for (int i = 0; i < imagePoints.Count; ++i) {
                        objectPoints.Add (objectPoint);
                    }

                    repErr = Calib3d.calibrateCamera (objectPoints, imagePoints, frameMat.size (), camMatrix, distCoeffs, rvecs, tvecs, calibrationFlags);
                    objectPoint.Dispose ();
                }

                break;
            }

            Debug.Log ("repErr: " + repErr);
            Debug.Log ("camMatrix: " + camMatrix.dump ());
            Debug.Log ("distCoeffs: " + distCoeffs.dump ());
                
            return repErr;
        }

        private double CalibrateCameraAruco (List<List<Mat>> allCorners, List<Mat> allIds, CharucoBoard board, Size imageSize, Mat cameraMatrix, Mat distCoeffs, List<Mat> rvecs = null, List<Mat> tvecs = null, int calibrationFlags = 0)
        {
            // prepare data for calibration
            int nFrames = allCorners.Count;
            int allLen = 0;
            int[] markerCounterPerFrameArr = new int[allCorners.Count];
            for (int i = 0; i < nFrames; ++i) {
                markerCounterPerFrameArr [i] = allCorners [i].Count;
                allLen += allCorners [i].Count;
            }

            int[] allIdsConcatenatedArr = new int[allLen];
            int index = 0;
            for (int j = 0; j < allIds.Count; ++j) {
                int[] idsArr = new int[(int)allIds [j].total ()];
                allIds [j].get (0, 0, idsArr);

                for (int k = 0; k < idsArr.Length; ++k) {
                    allIdsConcatenatedArr [index + k] = (int)idsArr [k];
                }
                index += idsArr.Length;
            }

            using (Mat allIdsConcatenated = new Mat (1, allLen, CvType.CV_32SC1))
            using (Mat markerCounterPerFrame = new Mat (1, nFrames, CvType.CV_32SC1)) {

                List<Mat> allCornersConcatenated = new List<Mat> ();
                foreach (var c in allCorners) {
                    foreach (var m in c) {
                        allCornersConcatenated.Add (m);
                    }
                }

                allIdsConcatenated.put (0, 0, allIdsConcatenatedArr);
                markerCounterPerFrame.put (0, 0, markerCounterPerFrameArr);

                if (rvecs == null)
                    rvecs = new List<Mat> ();
                if (tvecs == null)
                    tvecs = new List<Mat> ();

                return Aruco.calibrateCameraAruco (allCornersConcatenated, allIdsConcatenated, markerCounterPerFrame, board, imageSize, cameraMatrix, distCoeffs, rvecs, tvecs, calibrationFlags);
            }
        }

        private double CalibrateCameraCharuco (List<List<Mat>> allCorners, List<Mat> allIds, CharucoBoard board, Size imageSize, Mat cameraMatrix, Mat distCoeffs, List<Mat> rvecs = null, List<Mat> tvecs = null, int calibrationFlags = 0, int minMarkers = 2)
        {
            // prepare data for charuco calibration
            int nFrames = allCorners.Count;
            List<Mat> allCharucoCorners = new List<Mat> ();
            List<Mat> allCharucoIds = new List<Mat> ();
            List<Mat> filteredImages = new List<Mat> ();

            for (int i = 0; i < nFrames; ++i) {
                // interpolate using camera parameters
                Mat currentCharucoCorners = new Mat ();
                Mat currentCharucoIds = new Mat ();

                Aruco.interpolateCornersCharuco (allCorners [i], allIds [i], allImgs [i], board, currentCharucoCorners, currentCharucoIds, cameraMatrix, distCoeffs, minMarkers);

                if (charucoIds.total () > 0) {
                    allCharucoCorners.Add (currentCharucoCorners);
                    allCharucoIds.Add (currentCharucoIds);
                    filteredImages.Add (allImgs [i]);
                } else {
                    currentCharucoCorners.Dispose ();
                    currentCharucoIds.Dispose ();
                }
            }

            if (allCharucoCorners.Count < 1) {
                Debug.Log ("Not enough corners for calibration.");
                return -1;
            }

            if (rvecs == null)
                rvecs = new List<Mat> ();
            if (tvecs == null)
                tvecs = new List<Mat> ();

            return Aruco.calibrateCameraCharuco (allCharucoCorners, allCharucoIds, board, imageSize, cameraMatrix, distCoeffs, rvecs, tvecs, calibrationFlags);
        }

        private void ResetCalibration ()
        {
            foreach (var corners in allCorners) {
                foreach (var item in corners) {
                    item.Dispose ();
                }
            }
            allCorners.Clear ();

            foreach (var item in allIds) {
                item.Dispose ();
            }
            allIds.Clear ();

            foreach (var item in allImgs) {
                item.Dispose ();
            }
            allImgs.Clear ();

            repErr = 0;
            camMatrix = CreateCameraMatrix (bgrMat.width (), bgrMat.height ());
            distCoeffs = new MatOfDouble (0, 0, 0, 0, 0);

            foreach (var item in imagePoints) {
                item.Dispose ();
            }
            imagePoints.Clear ();
        }

        private Mat CreateCameraMatrix (float width, float height)
        {
            int max_d = (int)Mathf.Max (width, height);
            double fx = max_d;
            double fy = max_d;
            double cx = width / 2.0f;
            double cy = height / 2.0f;

            Mat camMatrix = new Mat (3, 3, CvType.CV_64FC1);
            camMatrix.put (0, 0, fx);
            camMatrix.put (0, 1, 0);
            camMatrix.put (0, 2, cx);
            camMatrix.put (1, 0, 0);
            camMatrix.put (1, 1, fy);
            camMatrix.put (1, 2, cy);
            camMatrix.put (2, 0, 0);
            camMatrix.put (2, 1, 0);
            camMatrix.put (2, 2, 1.0f);

            return camMatrix;
        }

        private void CalcChessboardCorners (Size patternSize, float squareSize, MatOfPoint3f corners, MarkerType markerType)
        {
            if ((int)(patternSize.width * patternSize.height) != corners.rows ()) {
                Debug.Log ("Invalid corners size.");
                corners.create ((int)(patternSize.width * patternSize.height), 1, CvType.CV_32FC3);
            }

            const int cn = 3;
            float[] cornersArr = new float[corners.rows () * cn];
            int width = (int)patternSize.width;
            int height = (int)patternSize.height;

            switch (markerType) {
            default:
            case MarkerType.ChessBoard:
            case MarkerType.CirclesGlid:
                for (int i = 0; i < height; ++i) {
                    for (int j = 0; j < width; ++j) {
                        cornersArr [(i * width * cn) + (j * cn)] = j * squareSize;
                        cornersArr [(i * width * cn) + (j * cn) + 1] = i * squareSize;
                        cornersArr [(i * width * cn) + (j * cn) + 2] = 0;
                    }
                }
                corners.put (0, 0, cornersArr);

                break;
            case MarkerType.AsymmetricCirclesGlid:
                for (int i = 0; i < height; ++i) {
                    for (int j = 0; j < width; ++j) {
                        cornersArr [(i * width * cn) + (j * cn)] = (2 * j + i % 2) * squareSize;
                        cornersArr [(i * width * cn) + (j * cn) + 1] = i * squareSize;
                        cornersArr [(i * width * cn) + (j * cn) + 2] = 0;
                    }
                }
                corners.put (0, 0, cornersArr);

                break;
            }
        }

        private bool InitializeImagesInputMode ()
        {
            if (isInitialized)
                DisposeCalibraton ();

            if (String.IsNullOrEmpty (calibrationImagesDirectory)) {
                Debug.LogWarning ("When using the images input mode, please set a calibration images directory path.");
                return false;
            }

            string dirPath = Path.Combine (Application.streamingAssetsPath, calibrationImagesDirectory);
            if (!Directory.Exists (dirPath)) {
                Debug.LogWarning ("The directory does not exist.");
                return false;
            }
            string[] imageFiles = GetImageFilesInDirectory (dirPath);
            if (imageFiles.Length < 1) {
                Debug.LogWarning ("The image file does not exist.");
                return false;
            }                
                
            Uri rootPath = new Uri (Application.streamingAssetsPath + System.IO.Path.AltDirectorySeparatorChar);
            Uri fullPath = new Uri (imageFiles [0]);
            string relativePath = rootPath.MakeRelativeUri (fullPath).ToString ();

            using (Mat gray = Imgcodecs.imread (Utils.getFilePath (relativePath), Imgcodecs.IMREAD_GRAYSCALE)) {

                if (gray.total () == 0) {
                    Debug.LogWarning ("Invalid image file.");
                    return false;
                }

                using (Mat bgr = new Mat (gray.size (), CvType.CV_8UC3))
                using (Mat bgra = new Mat (gray.size (), CvType.CV_8UC4)) {
                    InitializeCalibraton (gray);

                    DrawFrame (gray, bgr);
                    Imgproc.cvtColor (bgr, bgra, Imgproc.COLOR_BGR2RGBA);
                    Utils.fastMatToTexture2D (bgra, texture);
                }
            }
            return true;
        }

        private IEnumerator CalibrateCameraUsingImages ()
        {
            string dirPath = Path.Combine (Application.streamingAssetsPath, calibrationImagesDirectory);
            string[] imageFiles = GetImageFilesInDirectory (dirPath);
            if (imageFiles.Length < 1)
                yield break;

            isCalibrating = true;
            markerTypeDropdown.interactable = dictionaryIdDropdown.interactable = squaresXDropdown.interactable = squaresYDropdown.interactable = false;

            Uri rootPath = new Uri (Application.streamingAssetsPath + System.IO.Path.AltDirectorySeparatorChar);

            foreach (var path in imageFiles) {

                Uri fullPath = new Uri (path);
                string relativePath = rootPath.MakeRelativeUri (fullPath).ToString ();

                using (Mat gray = Imgcodecs.imread (Utils.getFilePath (relativePath), Imgcodecs.IMREAD_GRAYSCALE)) {

                    if (gray.width () != bgrMat.width () || gray.height () != bgrMat.height ())
                        continue;

                    Mat frameMat = gray.clone ();
                    double e = CaptureFrame (frameMat);
                    if (e > 0)
                        repErr = e;

                    DrawFrame (gray, bgrMat);
                    Imgproc.cvtColor (bgrMat, rgbaMat, Imgproc.COLOR_BGR2RGBA);

                    Utils.matToTexture2D (rgbaMat, texture);
                }
                yield return new WaitForSeconds (0.5f);
            }

            isCalibrating = false;
            markerTypeDropdown.interactable = dictionaryIdDropdown.interactable = squaresXDropdown.interactable = squaresYDropdown.interactable = true;
        }

        private string[] GetImageFilesInDirectory (string dirPath)
        {
            if (Directory.Exists (dirPath)) {
                string[] files = Directory.GetFiles (dirPath, "*.jpg");
                files = files.Concat (Directory.GetFiles (dirPath, "*.jpeg")).ToArray ();
                files = files.Concat (Directory.GetFiles (dirPath, "*.png")).ToArray ();
                files = files.Concat (Directory.GetFiles (dirPath, "*.tiff")).ToArray ();
                files = files.Concat (Directory.GetFiles (dirPath, "*.tif")).ToArray ();
                return files;
            }
            return new string[0];
        }

        /// <summary>
        /// Raises the destroy event.
        /// </summary>
        void OnDestroy ()
        {
            if (isImagesInputMode) {
                DisposeCalibraton ();
            } else {
                webCamTextureToMatHelper.Dispose ();
            }
                
            Screen.orientation = ScreenOrientation.AutoRotation;
        }

        /// <summary>
        /// Raises the back button click event.
        /// </summary>
        public void OnBackButtonClick ()
        {
            SceneManager.LoadScene ("OpenCVForUnityExample");
        }

        /// <summary>
        /// Raises the play button click event.
        /// </summary>
        public void OnPlayButtonClick ()
        {
            if (isImagesInputMode)
                return;

            webCamTextureToMatHelper.Play ();
        }

        /// <summary>
        /// Raises the pause button click event.
        /// </summary>
        public void OnPauseButtonClick ()
        {
            if (isImagesInputMode)
                return;

            webCamTextureToMatHelper.Pause ();
        }

        /// <summary>
        /// Raises the stop button click event.
        /// </summary>
        public void OnStopButtonClick ()
        {
            if (isImagesInputMode)
                return;

            webCamTextureToMatHelper.Stop ();
        }

        /// <summary>
        /// Raises the change camera button click event.
        /// </summary>
        public void OnChangeCameraButtonClick ()
        {
            if (isImagesInputMode)
                return;

            webCamTextureToMatHelper.requestedIsFrontFacing = !webCamTextureToMatHelper.IsFrontFacing ();
        }

        /// <summary>
        /// Raises the marker type dropdown value changed event.
        /// </summary>
        public void OnMarkerTypeDropdownValueChanged (int result)
        {
            if ((int)markerType != result) {
                markerType = (MarkerType)result;

                dictionaryIdDropdown.interactable = (markerType == MarkerType.ChArUcoBoard);

                if (isImagesInputMode) {
                    InitializeImagesInputMode ();
                } else {
                    if (webCamTextureToMatHelper.IsInitialized ())
                        webCamTextureToMatHelper.Initialize ();
                }
            }
        }

        /// <summary>
        /// Raises the dictionary id dropdown value changed event.
        /// </summary>
        public void OnDictionaryIdDropdownValueChanged (int result)
        {
            if ((int)dictionaryId != result) {
                dictionaryId = (ArUcoDictionary)result;
                dictionary = Aruco.getPredefinedDictionary ((int)dictionaryId);

                if (isImagesInputMode) {
                    InitializeImagesInputMode ();
                } else {
                    if (webCamTextureToMatHelper.IsInitialized ())
                        webCamTextureToMatHelper.Initialize ();
                }
            }
        }

        /// <summary>
        /// Raises the squares X dropdown value changed event.
        /// </summary>
        public void OnSquaresXDropdownValueChanged (int result)
        {
            if ((int)squaresX != result + 1) {
                squaresX = (NumberOfSquaresX)(result + 1);

                if (isImagesInputMode) {
                    InitializeImagesInputMode ();
                } else {
                    if (webCamTextureToMatHelper.IsInitialized ())
                        webCamTextureToMatHelper.Initialize ();
                }
            }
        }

        /// <summary>
        /// Raises the squares Y dropdown value changed event.
        /// </summary>
        public void OnSquaresYDropdownValueChanged (int result)
        {
            if ((int)squaresY != result + 1) {
                squaresY = (NumberOfSquaresY)(result + 1);

                if (isImagesInputMode) {
                    InitializeImagesInputMode ();
                } else {
                    if (webCamTextureToMatHelper.IsInitialized ())
                        webCamTextureToMatHelper.Initialize ();
                }
            }
        }

        /// <summary>
        /// Raises the capture button click event.
        /// </summary>
        public void OnCaptureButtonClick ()
        {
            if (isImagesInputMode) {
                if (!isCalibrating)
                    InitializeImagesInputMode ();
                StartCoroutine ("CalibrateCameraUsingImages");
            } else {
                shouldCaptureFrame = true;
            }
        }

        /// <summary>
        /// Raises the reset button click event.
        /// </summary>
        public void OnResetButtonClick ()
        {
            if (isImagesInputMode) {
                if (!isCalibrating)
                    InitializeImagesInputMode ();
            } else {
                ResetCalibration ();
            }
        }

        /// <summary>
        /// Raises the save button click event.
        /// </summary>
        public void OnSaveButtonClick ()
        {
            string saveDirectoryPath = Path.Combine (Application.persistentDataPath, "ArUcoCameraCalibrationExample");

            if (!Directory.Exists (saveDirectoryPath)) {
                Directory.CreateDirectory (saveDirectoryPath);
            }

            string calibratonDirectoryName = "camera_parameters" + bgrMat.width () + "x" + bgrMat.height ();
            string saveCalibratonFileDirectoryPath = Path.Combine (saveDirectoryPath, calibratonDirectoryName);

            // Clean up old files.
            if (Directory.Exists (saveCalibratonFileDirectoryPath)) {
                DirectoryInfo directoryInfo = new DirectoryInfo (saveCalibratonFileDirectoryPath);
                foreach (FileInfo fileInfo in directoryInfo.GetFiles()) {
                    if ((fileInfo.Attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly) {
                        fileInfo.Attributes = FileAttributes.Normal;
                    }
                }
                if ((directoryInfo.Attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly) {
                    directoryInfo.Attributes = FileAttributes.Directory;
                }
                directoryInfo.Delete (true);
            }
            Directory.CreateDirectory (saveCalibratonFileDirectoryPath);

            // save the calibraton file.
            string savePath = Path.Combine (saveCalibratonFileDirectoryPath, calibratonDirectoryName + ".xml");
            int frameCount = (markerType == (int)MarkerType.ChArUcoBoard) ? allCorners.Count : imagePoints.Count;
            CameraParameters param = new CameraParameters (frameCount, bgrMat.width (), bgrMat.height (), calibrationFlags, camMatrix, distCoeffs, repErr);
            XmlSerializer serializer = new XmlSerializer (typeof(CameraParameters));
            using (var stream = new FileStream (savePath, FileMode.Create)) {
                serializer.Serialize (stream, param);
            }

            // save the calibration images.
            #if UNITY_WEBGL && !UNITY_EDITOR
            string format = "jpg";
            MatOfInt compressionParams = new MatOfInt(Imgcodecs.IMWRITE_JPEG_QUALITY, 100);
            #else
            string format = "png";
            MatOfInt compressionParams = new MatOfInt (Imgcodecs.IMWRITE_PNG_COMPRESSION, 0);
            #endif
            for (int i = 0; i < allImgs.Count; ++i) {
                Imgcodecs.imwrite (Path.Combine (saveCalibratonFileDirectoryPath, calibratonDirectoryName + "_" + i.ToString ("00") + "." + format), allImgs [i], compressionParams);
            }

            savePathInputField.text = savePath;
            Debug.Log ("Saved the CameraParameters to disk in XML file format.");
            Debug.Log ("savePath: " + savePath);
        }

        public enum MarkerType
        {
            ChArUcoBoard,
            ChessBoard,
            CirclesGlid,
            AsymmetricCirclesGlid
        }

        public enum ArUcoDictionary
        {
            DICT_4X4_50 = Aruco.DICT_4X4_50,
            DICT_4X4_100 = Aruco.DICT_4X4_100,
            DICT_4X4_250 = Aruco.DICT_4X4_250,
            DICT_4X4_1000 = Aruco.DICT_4X4_1000,
            DICT_5X5_50 = Aruco.DICT_5X5_50,
            DICT_5X5_100 = Aruco.DICT_5X5_100,
            DICT_5X5_250 = Aruco.DICT_5X5_250,
            DICT_5X5_1000 = Aruco.DICT_5X5_1000,
            DICT_6X6_50 = Aruco.DICT_6X6_50,
            DICT_6X6_100 = Aruco.DICT_6X6_100,
            DICT_6X6_250 = Aruco.DICT_6X6_250,
            DICT_6X6_1000 = Aruco.DICT_6X6_1000,
            DICT_7X7_50 = Aruco.DICT_7X7_50,
            DICT_7X7_100 = Aruco.DICT_7X7_100,
            DICT_7X7_250 = Aruco.DICT_7X7_250,
            DICT_7X7_1000 = Aruco.DICT_7X7_1000,
            DICT_ARUCO_ORIGINAL = Aruco.DICT_ARUCO_ORIGINAL,
        }

        public enum NumberOfSquaresX
        {
            X_1 = 1,
            X_2,
            X_3,
            X_4,
            X_5,
            X_6,
            X_7,
            X_8,
            X_9,
            X_10,
            X_11,
            X_12,
            X_13,
            X_14,
            X_15,
        }

        public enum NumberOfSquaresY
        {
            Y_1 = 1,
            Y_2,
            Y_3,
            Y_4,
            Y_5,
            Y_6,
            Y_7,
            Y_8,
            Y_9,
            Y_10,
            Y_11,
            Y_12,
            Y_13,
            Y_14,
            Y_15,
        }
    }
}