using UnityEngine;
using UnityEngine.UI;
using UnityEngine.SceneManagement;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Xml.Serialization;
using System.IO;
using System.Linq;
using OpenCVForUnity.CoreModule;
using OpenCVForUnity.ImgprocModule;
using OpenCVForUnity.Calib3dModule;
using OpenCVForUnity.UnityUtils;
using OpenCVForUnity.ImgcodecsModule;
using OpenCVForUnity.UnityUtils.Helper;
using OpenCVForUnity.ObjdetectModule;
using OpenCVForUnity.ArucoModule;
namespace OpenCVForUnityExample
{
///
/// ArUco Camera Calibration Example
/// An example of camera calibration using the objdetect module. (ChessBoard, CirclesGlid, AsymmetricCirclesGlid and ChArUcoBoard)
/// Referring to https://docs.opencv.org/master/d4/d94/tutorial_camera_calibration.html.
/// https://github.com/opencv/opencv/blob/master/samples/cpp/tutorial_code/calib3d/camera_calibration/camera_calibration.cpp
/// https://docs.opencv.org/3.4.0/d7/d21/tutorial_interactive_calibration.html
/// https://github.com/opencv/opencv/tree/master/apps/interactive-calibration
/// https://docs.opencv.org/3.2.0/da/d13/tutorial_aruco_calibration.html
/// https://github.com/opencv/opencv_contrib/blob/master/modules/aruco/samples/calibrate_camera_charuco.cpp
///
[RequireComponent(typeof(WebCamTextureToMatHelper))]
public class ArUcoCameraCalibrationExample : MonoBehaviour
{
///
/// The marker type used for calibration.
///
[Tooltip("The marker type used for calibration.")]
public MarkerType markerType = MarkerType.ChessBoard;
///
/// The marker type dropdown.
///
public Dropdown markerTypeDropdown;
///
/// Number of inner corners per a item column. (square, circle)
///
[Tooltip("Number of inner corners per a item column. (square, circle)")]
public NumberOfBoardSizeWidth boardSizeW = NumberOfBoardSizeWidth.W_9;
///
/// The board size W dropdown.
///
public Dropdown boardSizeWDropdown;
///
/// Number of inner corners per a item row. (square, circle)
///
[Tooltip("Number of inner corners per a item row. (square, circle)")]
public NumberOfBoardSizeHeight boardSizeH = NumberOfBoardSizeHeight.H_6;
///
/// The board size H dropdown.
///
public Dropdown boardSizeHDropdown;
///
/// The save path input field.
///
public InputField savePathInputField;
///
/// The show undistort image.
///
public bool showUndistortImage = true;
///
/// The show undistort image toggle.
///
public Toggle showUndistortImageToggle;
[Header("Normal Calibration Option")]
///
/// The normal calibration options group.
///
public GameObject normalCalibrationOptionsGroup;
///
/// The size of a square in some user defined metric system (pixel, millimeter)
///
[Tooltip("The size of a square in some user defined metric system (pixel, millimeter)")]
public float squareSize = 50f;
///
/// The square size input field.
///
public InputField squareSizeInputField;
///
/// If your calibration board is inaccurate, unmeasured, roughly planar targets
/// (Checkerboard patterns on paper using off-the-shelf printers are the most convenient calibration targets and most of them are not accurate enough.),
/// a method from [219] can be utilized to dramatically improve the accuracies of the estimated camera intrinsic parameters.
/// Need to set the measured values from the actual chess board to "squareSize" and "gridWidth".
/// https://docs.opencv.org/4.2.0/d9/d0c/group__calib3d.html#ga11eeb16e5a458e1ed382fb27f585b753
///
[Tooltip("If your calibration board is inaccurate, unmeasured, roughly planar targets (Checkerboard patterns on paper using off-the-shelf printers are the most convenient calibration targets and most of them are not accurate enough.), a method from [219] can be utilized to dramatically improve the accuracies of the estimated camera intrinsic parameters. Need to set the measured values from the actual chess board to \"squareSize\" and \"gridWidth\".")]
public bool useNewCalibrationMethod = true;
///
/// The use new calibration method toggle.
///
public Toggle useNewCalibrationMethodToggle;
///
/// The measured distance between top-left (0, 0, 0) and top-right (squareSize*(boardSizeW - 1), 0, 0) corners of the pattern grid points.
///
[Tooltip("The measured distance between top-left (0, 0, 0) and top-right (squareSize*(boardSizeW - 1), 0, 0) corners of the pattern grid points.")]
public float gridWidth = 400f;
///
/// The glid width input field.
///
public InputField gridWidthInputField;
///
/// Determines if use findChessboardCornersSB method. (More accurate than the findChessboardCorners and cornerSubPix methods)
/// https://docs.opencv.org/4.2.0/d9/d0c/group__calib3d.html#gad0e88e13cd3d410870a99927510d7f91
///
[Tooltip("Determines if use findChessboardCornersSB method. (More accurate than the findChessboardCorners and cornerSubPix methods)")]
public bool useFindChessboardCornersSBMethod = true;
///
/// Determines if enable CornerSubPix method. (Improve the found corners' coordinate accuracy for chessboard)
///
[Tooltip("Determines if enable CornerSubPix method. (Improve the found corners' coordinate accuracy for chessboard)")]
public bool enableCornerSubPix = true;
[Header("ArUco Calibration Option")]
///
/// The arUco calibration options group.
///
public GameObject arUcoCalibrationOptionsGroup;
///
/// The dictionary identifier used for ArUco marker detection.
///
[Tooltip("The dictionary identifier used for ArUco marker detection.")]
public ArUcoDictionary dictionaryId = ArUcoDictionary.DICT_6X6_250;
///
/// The dictionary id dropdown.
///
public Dropdown dictionaryIdDropdown;
///
/// Determines if refine marker detection. (only valid for ArUco boards)
///
[Tooltip("Determines if refine marker detection. (only valid for ArUco boards)")]
public bool refineMarkerDetection = true;
[Header("Image Input Option")]
///
/// Determines if calibrates camera using the list of calibration images.
///
[Tooltip("Determines if calibrates camera using the list of calibration images.")]
public bool isImagesInputMode = false;
///
/// The calibration images directory path.
/// Set a relative directory path from the starting point of the "StreamingAssets" folder. e.g. "objdetect/calibration_images/".
///
[Tooltip("Set a relative directory path from the starting point of the \"StreamingAssets\" folder. e.g. \"OpenCVForUnity/objdetect/calibration_images\"")]
public string calibrationImagesDirectory = "OpenCVForUnity/objdetect/calibration_images";
///
/// The texture.
///
Texture2D texture;
///
/// The webcam texture to mat helper.
///
WebCamTextureToMatHelper webCamTextureToMatHelper;
///
/// The gray mat.
///
Mat grayMat;
///
/// The bgr mat.
///
Mat bgrMat;
///
/// The undistorted bgr mat.
///
Mat undistortedBgrMat;
///
/// The rgba mat.
///
Mat rgbaMat;
///
/// The cameraparam matrix.
///
Mat camMatrix;
///
/// The distortion coeffs.
///
MatOfDouble distCoeffs;
///
/// The rvecs.
///
List rvecs;
///
/// The tvecs.
///
List tvecs;
List imagePoints;
List allImgs;
bool isInitialized = false;
bool isCalibrating = false;
double repErr = 0;
bool shouldCaptureFrame = false;
const int findChessboardCornersFlags =
Calib3d.CALIB_CB_ADAPTIVE_THRESH |
Calib3d.CALIB_CB_NORMALIZE_IMAGE |
//Calib3d.CALIB_CB_FILTER_QUADS |
Calib3d.CALIB_CB_FAST_CHECK |
0;
const int findChessboardCornersSBFlags =
Calib3d.CALIB_CB_NORMALIZE_IMAGE |
Calib3d.CALIB_CB_EXHAUSTIVE |
Calib3d.CALIB_CB_ACCURACY |
0;
const int findCirclesGridFlags =
//Calib3d.CALIB_CB_CLUSTERING |
0;
const int calibrationFlags =
//Calib3d.CALIB_USE_INTRINSIC_GUESS |
//Calib3d.CALIB_FIX_PRINCIPAL_POINT |
//Calib3d.CALIB_FIX_ASPECT_RATIO |
//Calib3d.CALIB_ZERO_TANGENT_DIST |
//Calib3d.CALIB_FIX_K1 |
//Calib3d.CALIB_FIX_K2 |
//Calib3d.CALIB_FIX_K3 |
//Calib3d.CALIB_FIX_K4 |
//Calib3d.CALIB_FIX_K5 |
Calib3d.CALIB_USE_LU |
0;
/*
// for ChArUcoBoard.
// chessboard square side length (normally in meters)
const float chArUcoBoradSquareLength = 0.04f;
// marker side length (same unit than squareLength)
const float chArUcoBoradMarkerLength = 0.02f;
const int charucoMinMarkers = 2;
Mat ids;
List corners;
List rejectedCorners;
Mat recoveredIdxs;
Mat charucoCorners;
Mat charucoIds;
CharucoBoard charucoBoard;
ArucoDetector arucoDetector;
CharucoDetector charucoDetector;
*/
Dictionary dictionary;
List> allCorners;
List allIds;
// Use this for initialization
IEnumerator Start()
{
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode(true);
webCamTextureToMatHelper = gameObject.GetComponent();
// fix the screen orientation.
Screen.orientation = ScreenOrientation.LandscapeLeft;
// wait for the screen orientation to change.
yield return null;
markerTypeDropdown.value = (int)markerType;
boardSizeWDropdown.value = (int)boardSizeW - 1;
boardSizeHDropdown.value = (int)boardSizeH - 1;
showUndistortImageToggle.isOn = showUndistortImage;
squareSizeInputField.text = squareSize.ToString();
useNewCalibrationMethodToggle.isOn = useNewCalibrationMethod;
gridWidthInputField.text = gridWidth.ToString();
dictionaryIdDropdown.value = (int)dictionaryId;
bool arUcoCalibMode = markerType == MarkerType.ChArUcoBoard;
normalCalibrationOptionsGroup.gameObject.SetActive(!arUcoCalibMode);
arUcoCalibrationOptionsGroup.gameObject.SetActive(arUcoCalibMode);
#if UNITY_WEBGL && !UNITY_EDITOR
isImagesInputMode = false;
#endif
if (isImagesInputMode)
{
isImagesInputMode = InitializeImagesInputMode();
}
if (!isImagesInputMode)
{
#if UNITY_ANDROID && !UNITY_EDITOR
// Avoids the front camera low light issue that occurs in only some Android devices (e.g. Google Pixel, Pixel2).
webCamTextureToMatHelper.avoidAndroidFrontCameraLowLightIssue = true;
#endif
webCamTextureToMatHelper.Initialize();
}
}
///
/// Raises the webcam texture to mat helper initialized event.
///
public void OnWebCamTextureToMatHelperInitialized()
{
Debug.Log("OnWebCamTextureToMatHelperInitialized");
Mat webCamTextureMat = webCamTextureToMatHelper.GetMat();
InitializeCalibraton(webCamTextureMat);
// If the WebCam is front facing, flip the Mat horizontally. Required for successful detection of AR markers.
if (webCamTextureToMatHelper.IsFrontFacing() && !webCamTextureToMatHelper.flipHorizontal)
{
webCamTextureToMatHelper.flipHorizontal = true;
}
else if (!webCamTextureToMatHelper.IsFrontFacing() && webCamTextureToMatHelper.flipHorizontal)
{
webCamTextureToMatHelper.flipHorizontal = false;
}
}
///
/// Raises the webcam texture to mat helper disposed event.
///
public void OnWebCamTextureToMatHelperDisposed()
{
Debug.Log("OnWebCamTextureToMatHelperDisposed");
DisposeCalibraton();
}
///
/// Raises the webcam texture to mat helper error occurred event.
///
/// Error code.
public void OnWebCamTextureToMatHelperErrorOccurred(WebCamTextureToMatHelper.ErrorCode errorCode)
{
Debug.Log("OnWebCamTextureToMatHelperErrorOccurred " + errorCode);
}
// Update is called once per frame
void Update()
{
if (isImagesInputMode)
return;
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
if (shouldCaptureFrame)
{
shouldCaptureFrame = false;
Mat frameMat = grayMat.clone();
double e = CaptureFrame(frameMat);
if (e > 0)
repErr = e;
}
DrawFrame(grayMat, bgrMat);
if (showUndistortImage)
{
Calib3d.undistort(bgrMat, undistortedBgrMat, camMatrix, distCoeffs);
DrawCalibrationResult(undistortedBgrMat);
Imgproc.cvtColor(undistortedBgrMat, rgbaMat, Imgproc.COLOR_BGR2RGBA);
}
else
{
DrawCalibrationResult(bgrMat);
Imgproc.cvtColor(bgrMat, rgbaMat, Imgproc.COLOR_BGR2RGBA);
}
Utils.matToTexture2D(rgbaMat, texture);
}
}
private void InitializeCalibraton(Mat frameMat)
{
texture = new Texture2D(frameMat.cols(), frameMat.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(frameMat, texture);
gameObject.GetComponent().material.mainTexture = texture;
gameObject.transform.localScale = new Vector3(frameMat.cols(), frameMat.rows(), 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float width = frameMat.width();
float height = frameMat.height();
float imageSizeScale = 1.0f;
float widthScale = (float)Screen.width / width;
float heightScale = (float)Screen.height / height;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2;
imageSizeScale = (float)Screen.height / (float)Screen.width;
}
else
{
Camera.main.orthographicSize = height / 2;
}
// set cameraparam.
camMatrix = CreateCameraMatrix(width, height);
Debug.Log("camMatrix " + camMatrix.dump());
distCoeffs = new MatOfDouble(0, 0, 0, 0, 0);
Debug.Log("distCoeffs " + distCoeffs.dump());
// calibration camera.
Size imageSize = new Size(width * imageSizeScale, height * imageSizeScale);
double apertureWidth = 0;
double apertureHeight = 0;
double[] fovx = new double[1];
double[] fovy = new double[1];
double[] focalLength = new double[1];
Point principalPoint = new Point(0, 0);
double[] aspectratio = new double[1];
Calib3d.calibrationMatrixValues(camMatrix, imageSize, apertureWidth, apertureHeight, fovx, fovy, focalLength, principalPoint, aspectratio);
Debug.Log("imageSize " + imageSize.ToString());
Debug.Log("apertureWidth " + apertureWidth);
Debug.Log("apertureHeight " + apertureHeight);
Debug.Log("fovx " + fovx[0]);
Debug.Log("fovy " + fovy[0]);
Debug.Log("focalLength " + focalLength[0]);
Debug.Log("principalPoint " + principalPoint.ToString());
Debug.Log("aspectratio " + aspectratio[0]);
grayMat = new Mat(frameMat.rows(), frameMat.cols(), CvType.CV_8UC1);
bgrMat = new Mat(frameMat.rows(), frameMat.cols(), CvType.CV_8UC3);
undistortedBgrMat = new Mat();
rgbaMat = new Mat(frameMat.rows(), frameMat.cols(), CvType.CV_8UC4);
rvecs = new List();
tvecs = new List();
imagePoints = new List();
allImgs = new List();
/*
ids = new Mat();
corners = new List();
rejectedCorners = new List();
recoveredIdxs = new Mat();
DetectorParameters detectorParams = new DetectorParameters();
detectorParams.set_minDistanceToBorder(3);
detectorParams.set_useAruco3Detection(true);
detectorParams.set_cornerRefinementMethod(Objdetect.CORNER_REFINE_SUBPIX);
detectorParams.set_minSideLengthCanonicalImg(16);
detectorParams.set_errorCorrectionRate(0.8);
dictionary = Objdetect.getPredefinedDictionary((int)dictionaryId);
RefineParameters refineParameters = new RefineParameters(10f, 3f, true);
arucoDetector = new ArucoDetector(dictionary, detectorParams, refineParameters);
charucoCorners = new Mat();
charucoIds = new Mat();
charucoBoard = new CharucoBoard( new Size((int)boardSizeW, (int)boardSizeH), chArUcoBoradSquareLength, chArUcoBoradMarkerLength, dictionary);
charucoDetector = new CharucoDetector(charucoBoard);
CharucoParameters charucoParameters = charucoDetector.getCharucoParameters();
charucoParameters.set_cameraMatrix(camMatrix);
charucoParameters.set_distCoeffs(distCoeffs);
charucoParameters.set_minMarkers(charucoMinMarkers);
charucoDetector.setCharucoParameters(charucoParameters);
charucoDetector.setDetectorParameters(detectorParams);
charucoDetector.setRefineParameters(refineParameters);
*/
allIds = new List();
allCorners = new List>();
isInitialized = true;
}
private void DisposeCalibraton()
{
ResetCalibration();
if (grayMat != null)
grayMat.Dispose();
if (bgrMat != null)
bgrMat.Dispose();
if (undistortedBgrMat != null)
undistortedBgrMat.Dispose();
if (rgbaMat != null)
rgbaMat.Dispose();
if (texture != null)
{
Texture2D.Destroy(texture);
texture = null;
}
foreach (var item in rvecs)
{
item.Dispose();
}
rvecs.Clear();
foreach (var item in tvecs)
{
item.Dispose();
}
tvecs.Clear();
/*
if (ids != null)
ids.Dispose();
foreach (var item in corners)
{
item.Dispose();
}
corners.Clear();
foreach (var item in rejectedCorners)
{
item.Dispose();
}
rejectedCorners.Clear();
if (recoveredIdxs != null)
recoveredIdxs.Dispose();
if (charucoCorners != null)
charucoCorners.Dispose();
if (charucoIds != null)
charucoIds.Dispose();
if (charucoBoard != null)
charucoBoard.Dispose();
if (arucoDetector != null)
arucoDetector.Dispose();
if (charucoDetector != null)
charucoDetector.Dispose();
*/
isInitialized = false;
}
private void DrawFrame(Mat grayMat, Mat bgrMat)
{
Imgproc.cvtColor(grayMat, bgrMat, Imgproc.COLOR_GRAY2BGR);
switch (markerType)
{
default:
case MarkerType.ChessBoard:
case MarkerType.CirclesGlid:
case MarkerType.AsymmetricCirclesGlid:
// detect markers.
MatOfPoint2f points = new MatOfPoint2f();
bool found = false;
switch (markerType)
{
default:
case MarkerType.ChessBoard:
if (useFindChessboardCornersSBMethod)
{
found = Calib3d.findChessboardCornersSB(grayMat, new Size((int)boardSizeW, (int)boardSizeH), points, findChessboardCornersSBFlags);
}
else
{
found = Calib3d.findChessboardCorners(grayMat, new Size((int)boardSizeW, (int)boardSizeH), points, findChessboardCornersFlags);
}
break;
case MarkerType.CirclesGlid:
found = Calib3d.findCirclesGrid(grayMat, new Size((int)boardSizeW, (int)boardSizeH), points, findCirclesGridFlags | Calib3d.CALIB_CB_SYMMETRIC_GRID);
break;
case MarkerType.AsymmetricCirclesGlid:
found = Calib3d.findCirclesGrid(grayMat, new Size((int)boardSizeW, (int)boardSizeH), points, findCirclesGridFlags | Calib3d.CALIB_CB_ASYMMETRIC_GRID);
break;
}
if (found)
{
// draw markers.
Calib3d.drawChessboardCorners(bgrMat, new Size((int)boardSizeW, (int)boardSizeH), points, found);
}
break;
case MarkerType.ChArUcoBoard:
/*
// detect markers.
arucoDetector.detectMarkers(grayMat, corners, ids, rejectedCorners);
// refine marker detection.
if (refineMarkerDetection)
{
// https://github.com/opencv/opencv/blob/377be68d923e40900ac5526242bcf221e3f355e5/modules/objdetect/src/aruco/charuco_detector.cpp#L310
arucoDetector.refineDetectedMarkers(grayMat, charucoBoard, corners, ids, rejectedCorners);
}
// if at least one marker detected
if (ids.total() > 0)
{
charucoDetector.detectBoard(grayMat, charucoCorners, charucoIds, corners, ids);
// draw markers.
if (corners.Count == ids.total() || ids.total() == 0)
Objdetect.drawDetectedMarkers(bgrMat, corners, ids, new Scalar(0, 255, 0, 255));
// if at least one charuco corner detected
if (charucoCorners.total() == charucoIds.total() || charucoIds.total() == 0)
{
Objdetect.drawDetectedCornersCharuco(bgrMat, charucoCorners, charucoIds, new Scalar(0, 0, 255, 255));
}
}
*/
break;
}
}
private void DrawCalibrationResult(Mat bgrMat)
{
double[] camMatrixArr = new double[(int)camMatrix.total()];
camMatrix.get(0, 0, camMatrixArr);
double[] distCoeffsArr = new double[(int)distCoeffs.total()];
distCoeffs.get(0, 0, distCoeffsArr);
int textLeft = 320;
int ff = Imgproc.FONT_HERSHEY_SIMPLEX;
double fs = 0.4;
Scalar c = new Scalar(255, 255, 255, 255);
int t = 0;
int lt = Imgproc.LINE_AA;
bool blo = false;
int frameCount = (markerType == MarkerType.ChArUcoBoard) ? allCorners.Count : imagePoints.Count;
Imgproc.putText(bgrMat, frameCount + " FRAME CAPTURED", new Point(bgrMat.cols() - textLeft, 20), ff, fs, c, t, lt, blo);
Imgproc.putText(bgrMat, "IMAGE_WIDTH: " + bgrMat.width(), new Point(bgrMat.cols() - textLeft, 40), ff, fs, c, t, lt, blo);
Imgproc.putText(bgrMat, "IMAGE_HEIGHT: " + bgrMat.height(), new Point(bgrMat.cols() - textLeft, 60), ff, fs, c, t, lt, blo);
Imgproc.putText(bgrMat, "CALIBRATION_FLAGS: " + calibrationFlags, new Point(bgrMat.cols() - textLeft, 80), ff, fs, c, t, lt, blo);
Imgproc.putText(bgrMat, "CAMERA_MATRIX: ", new Point(bgrMat.cols() - 310, 100), ff, fs, c, t, lt, blo);
for (int i = 0; i < camMatrixArr.Length; i = i + 3)
{
Imgproc.putText(bgrMat, " " + camMatrixArr[i] + ", " + camMatrixArr[i + 1] + ", " + camMatrixArr[i + 2] + ",", new Point(bgrMat.cols() - textLeft, 120 + 20 * i / 3), ff, fs, c, t, lt, blo);
}
Imgproc.putText(bgrMat, "DISTORTION_COEFFICIENTS: ", new Point(bgrMat.cols() - textLeft, 180), ff, fs, c, t, lt, blo);
for (int i = 0; i < distCoeffsArr.Length; ++i)
{
Imgproc.putText(bgrMat, " " + distCoeffsArr[i] + ",", new Point(bgrMat.cols() - textLeft, 200 + 20 * i), ff, fs, c, t, lt, blo);
}
Imgproc.putText(bgrMat, "AVG_REPROJECTION_ERROR: " + repErr, new Point(bgrMat.cols() - textLeft, 300), ff, fs, c, t, lt, blo);
if (frameCount == 0)
Imgproc.putText(bgrMat, "Please press the capture button to start!", new Point(5, bgrMat.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 255, 255, 255), 1, Imgproc.LINE_AA, false);
}
private double CaptureFrame(Mat frameMat)
{
double repErr = -1;
switch (markerType)
{
default:
case MarkerType.ChessBoard:
case MarkerType.CirclesGlid:
case MarkerType.AsymmetricCirclesGlid:
MatOfPoint2f points = new MatOfPoint2f();
Size patternSize = new Size((int)boardSizeW, (int)boardSizeH);
bool found = false;
switch (markerType)
{
default:
case MarkerType.ChessBoard:
if (useFindChessboardCornersSBMethod)
{
found = Calib3d.findChessboardCornersSB(frameMat, patternSize, points, findChessboardCornersSBFlags);
}
else
{
found = Calib3d.findChessboardCorners(frameMat, patternSize, points, findChessboardCornersFlags);
}
break;
case MarkerType.CirclesGlid:
found = Calib3d.findCirclesGrid(frameMat, patternSize, points, findCirclesGridFlags | Calib3d.CALIB_CB_SYMMETRIC_GRID);
break;
case MarkerType.AsymmetricCirclesGlid:
found = Calib3d.findCirclesGrid(frameMat, patternSize, points, findCirclesGridFlags | Calib3d.CALIB_CB_ASYMMETRIC_GRID);
break;
}
if (found)
{
if (markerType == MarkerType.ChessBoard && !useFindChessboardCornersSBMethod && enableCornerSubPix)
{
int winSize = 11;
Imgproc.cornerSubPix(frameMat, points, new Size(winSize, winSize), new Size(-1, -1), new TermCriteria(TermCriteria.EPS + TermCriteria.COUNT, 30, 0.0001));
}
imagePoints.Add(points);
allImgs.Add(frameMat);
Debug.Log(imagePoints.Count + " Frame captured.");
}
else
{
Debug.Log("Invalid frame.");
frameMat.Dispose();
if (points != null)
points.Dispose();
return -1;
}
if (imagePoints.Count < 1)
{
Debug.Log("Not enough points for calibration.");
repErr = -1;
}
else
{
MatOfPoint3f objectPoint = new MatOfPoint3f(new Mat(imagePoints[0].rows(), 1, CvType.CV_32FC3));
CalcChessboardCorners(patternSize, squareSize, objectPoint, markerType);
float grid_width = squareSize * ((int)patternSize.width - 1);
bool release_object = false;
if (useNewCalibrationMethod)
{
grid_width = gridWidth;
release_object = true;
}
float[] tlPt = new float[3]; // top-left point
objectPoint.get(0, 0, tlPt);
float[] trPt = new float[3]; // top-right point
objectPoint.get((int)patternSize.width - 1, 0, trPt);
trPt[0] = tlPt[0] + grid_width;
objectPoint.put((int)patternSize.width - 1, 0, trPt);
Mat newObjPoints = objectPoint.clone();
List objectPoints = new List();
for (int i = 0; i < imagePoints.Count; ++i)
{
objectPoints.Add(objectPoint.clone());
}
int iFixedPoint = -1;
if (release_object)
iFixedPoint = (int)patternSize.width - 1;
repErr = Calib3d.calibrateCameraRO(
objectPoints,
imagePoints,
frameMat.size(),
iFixedPoint,
camMatrix,
distCoeffs,
rvecs,
tvecs,
newObjPoints,
calibrationFlags
);
//if (release_object)
//{
// Debug.Log("New board corners: ");
// Point3[] newPoints = new MatOfPoint3f(newObjPoints).toArray();
// Debug.Log(newPoints[0]);
// Debug.Log(newPoints[(int)patternSize.width - 1]);
// Debug.Log(newPoints[(int)patternSize.width * ((int)patternSize.height - 1)]);
// Debug.Log(newPoints[newPoints.Length - 1]);
//}
objectPoint.Dispose();
}
break;
case MarkerType.ChArUcoBoard:
/*
List corners = new List();
Mat ids = new Mat();
arucoDetector.detectMarkers(frameMat, corners, ids, rejectedCorners);
if (refineMarkerDetection)
{
// https://github.com/opencv/opencv/blob/377be68d923e40900ac5526242bcf221e3f355e5/modules/objdetect/src/aruco/charuco_detector.cpp#L310
arucoDetector.refineDetectedMarkers(frameMat, charucoBoard, corners, ids, rejectedCorners);
}
if (ids.total() > 0)
{
allCorners.Add(corners);
allIds.Add(ids);
allImgs.Add(frameMat);
Debug.Log(allCorners.Count + " Frame captured.");
}
else
{
Debug.Log("Invalid frame.");
frameMat.Dispose();
if (ids != null)
ids.Dispose();
foreach (var item in corners)
{
item.Dispose();
}
corners.Clear();
return -1;
}
// calibrate camera using charuco boards
repErr = CalibrateCameraCharuco(allCorners, allIds, charucoBoard, frameMat.size(), camMatrix, distCoeffs, rvecs, tvecs, calibrationFlags, calibrationFlags);
*/
break;
}
Debug.Log("repErr: " + repErr);
Debug.Log("camMatrix: " + camMatrix.dump());
Debug.Log("distCoeffs: " + distCoeffs.dump());
return repErr;
}
/*
private double CalibrateCameraCharuco(List> allCorners, List allIds, CharucoBoard board, Size imageSize, Mat cameraMatrix, Mat distCoeffs, List rvecs = null, List tvecs = null, int calibrationFlags = 0, int minMarkers = 2)
{
// prepare data for charuco calibration
int nFrames = allCorners.Count;
List allCharucoCorners = new List();
List allCharucoIds = new List();
List filteredImages = new List();
for (int i = 0; i < nFrames; ++i)
{
// interpolate using camera parameters
Mat currentCharucoCorners = new Mat();
Mat currentCharucoIds = new Mat();
charucoDetector.detectBoard(allImgs[i], currentCharucoCorners, currentCharucoIds, allCorners[i], allIds[i]);
//if (currentCharucoIds.total() > 0)
if (currentCharucoIds.total() > 0 && currentCharucoCorners.total() == currentCharucoIds.total())
{
allCharucoCorners.Add(currentCharucoCorners);
allCharucoIds.Add(currentCharucoIds);
filteredImages.Add(allImgs[i]);
}
else
{
currentCharucoCorners.Dispose();
currentCharucoIds.Dispose();
}
}
if (allCharucoCorners.Count < 1)
{
Debug.Log("Not enough corners for calibration.");
return -1;
}
if (rvecs == null)
rvecs = new List();
if (tvecs == null)
tvecs = new List();
return Aruco.calibrateCameraCharuco(allCharucoCorners, allCharucoIds, board, imageSize, cameraMatrix, distCoeffs, rvecs, tvecs, calibrationFlags); // error
}
*/
private void ResetCalibration()
{
foreach (var item in allImgs)
{
item.Dispose();
}
allImgs.Clear();
repErr = 0;
camMatrix = CreateCameraMatrix(bgrMat.width(), bgrMat.height());
distCoeffs = new MatOfDouble(0, 0, 0, 0, 0);
foreach (var item in imagePoints)
{
item.Dispose();
}
imagePoints.Clear();
foreach (var corners in allCorners)
{
foreach (var item in corners)
{
item.Dispose();
}
}
allCorners.Clear();
foreach (var item in allIds)
{
item.Dispose();
}
allIds.Clear();
}
private Mat CreateCameraMatrix(float width, float height)
{
int max_d = (int)Mathf.Max(width, height);
double fx = max_d;
double fy = max_d;
double cx = width / 2.0f;
double cy = height / 2.0f;
Mat camMatrix = new Mat(3, 3, CvType.CV_64FC1);
camMatrix.put(0, 0, fx);
camMatrix.put(0, 1, 0);
camMatrix.put(0, 2, cx);
camMatrix.put(1, 0, 0);
camMatrix.put(1, 1, fy);
camMatrix.put(1, 2, cy);
camMatrix.put(2, 0, 0);
camMatrix.put(2, 1, 0);
camMatrix.put(2, 2, 1.0f);
return camMatrix;
}
private void CalcChessboardCorners(Size patternSize, float squareSize, MatOfPoint3f corners, MarkerType markerType)
{
if ((int)(patternSize.width * patternSize.height) != corners.rows())
{
Debug.Log("Invalid corners size.");
corners.create((int)(patternSize.width * patternSize.height), 1, CvType.CV_32FC3);
}
int width = (int)patternSize.width;
int height = (int)patternSize.height;
switch (markerType)
{
default:
case MarkerType.ChessBoard:
case MarkerType.CirclesGlid:
for (int i = 0; i < height; ++i)
{
for (int j = 0; j < width; ++j)
{
corners.put(width * i + j, 0, new float[] { j * squareSize, i * squareSize, 0f });
}
}
break;
case MarkerType.AsymmetricCirclesGlid:
for (int i = 0; i < height; ++i)
{
for (int j = 0; j < width; ++j)
{
corners.put(width * i + j, 0, new float[] { (2 * j + i % 2) * squareSize, i * squareSize, 0f });
}
}
break;
}
}
private bool InitializeImagesInputMode()
{
if (isInitialized)
DisposeCalibraton();
if (String.IsNullOrEmpty(calibrationImagesDirectory))
{
Debug.LogWarning("When using the images input mode, please set a calibration images directory path.");
return false;
}
string dirPath = Path.Combine(Application.streamingAssetsPath, calibrationImagesDirectory);
if (!Directory.Exists(dirPath))
{
Debug.LogWarning("The directory does not exist.");
return false;
}
string[] imageFiles = GetImageFilesInDirectory(dirPath);
if (imageFiles.Length < 1)
{
Debug.LogWarning("The image file does not exist.");
return false;
}
Uri rootPath = new Uri(Application.streamingAssetsPath + System.IO.Path.AltDirectorySeparatorChar);
Uri fullPath = new Uri(imageFiles[0]);
string relativePath = rootPath.MakeRelativeUri(fullPath).ToString();
using (Mat gray = Imgcodecs.imread(Utils.getFilePath(relativePath), Imgcodecs.IMREAD_GRAYSCALE))
{
if (gray.total() == 0)
{
Debug.LogWarning("Invalid image file.");
return false;
}
using (Mat bgr = new Mat(gray.size(), CvType.CV_8UC3))
using (Mat rgba = new Mat(gray.size(), CvType.CV_8UC4))
{
Imgproc.cvtColor(gray, rgba, Imgproc.COLOR_GRAY2RGBA);
InitializeCalibraton(rgba);
DrawFrame(gray, bgr);
DrawCalibrationResult(bgr);
Imgproc.cvtColor(bgr, rgba, Imgproc.COLOR_BGR2RGBA);
Utils.matToTexture2D(rgba, texture);
}
}
return true;
}
private IEnumerator CalibrateCameraUsingImages()
{
string dirPath = Path.Combine(Application.streamingAssetsPath, calibrationImagesDirectory);
string[] imageFiles = GetImageFilesInDirectory(dirPath);
if (imageFiles.Length < 1)
yield break;
isCalibrating = true;
markerTypeDropdown.interactable = boardSizeWDropdown.interactable = boardSizeHDropdown.interactable = false;
normalCalibrationOptionsGroup.gameObject.SetActive(false);
arUcoCalibrationOptionsGroup.gameObject.SetActive(false);
Uri rootPath = new Uri(Application.streamingAssetsPath + System.IO.Path.AltDirectorySeparatorChar);
foreach (var path in imageFiles)
{
Uri fullPath = new Uri(path);
string relativePath = rootPath.MakeRelativeUri(fullPath).ToString();
using (Mat gray = Imgcodecs.imread(Utils.getFilePath(relativePath), Imgcodecs.IMREAD_GRAYSCALE))
{
if (gray.width() != bgrMat.width() || gray.height() != bgrMat.height())
continue;
Mat frameMat = gray.clone();
double e = CaptureFrame(frameMat);
if (e > 0)
repErr = e;
DrawFrame(gray, bgrMat);
DrawCalibrationResult(bgrMat);
Imgproc.cvtColor(bgrMat, rgbaMat, Imgproc.COLOR_BGR2RGBA);
Utils.matToTexture2D(rgbaMat, texture);
}
yield return new WaitForSeconds(0.5f);
}
isCalibrating = false;
markerTypeDropdown.interactable = boardSizeWDropdown.interactable = boardSizeHDropdown.interactable = true;
bool arUcoCalibMode = markerType == MarkerType.ChArUcoBoard;
normalCalibrationOptionsGroup.gameObject.SetActive(!arUcoCalibMode);
arUcoCalibrationOptionsGroup.gameObject.SetActive(arUcoCalibMode);
}
private string[] GetImageFilesInDirectory(string dirPath)
{
if (Directory.Exists(dirPath))
{
string[] files = Directory.GetFiles(dirPath, "*.jpg");
files = files.Concat(Directory.GetFiles(dirPath, "*.jpeg")).ToArray();
files = files.Concat(Directory.GetFiles(dirPath, "*.png")).ToArray();
files = files.Concat(Directory.GetFiles(dirPath, "*.tiff")).ToArray();
files = files.Concat(Directory.GetFiles(dirPath, "*.tif")).ToArray();
return files;
}
return new string[0];
}
///
/// Raises the destroy event.
///
void OnDestroy()
{
if (isImagesInputMode)
{
DisposeCalibraton();
}
else
{
webCamTextureToMatHelper.Dispose();
}
Screen.orientation = ScreenOrientation.AutoRotation;
Utils.setDebugMode(false);
}
///
/// Raises the back button click event.
///
public void OnBackButtonClick()
{
SceneManager.LoadScene("OpenCVForUnityExample");
}
///
/// Raises the play button click event.
///
public void OnPlayButtonClick()
{
if (isImagesInputMode)
return;
webCamTextureToMatHelper.Play();
}
///
/// Raises the pause button click event.
///
public void OnPauseButtonClick()
{
if (isImagesInputMode)
return;
webCamTextureToMatHelper.Pause();
}
///
/// Raises the stop button click event.
///
public void OnStopButtonClick()
{
if (isImagesInputMode)
return;
webCamTextureToMatHelper.Stop();
}
///
/// Raises the change camera button click event.
///
public void OnChangeCameraButtonClick()
{
if (isImagesInputMode)
return;
webCamTextureToMatHelper.requestedIsFrontFacing = !webCamTextureToMatHelper.requestedIsFrontFacing;
}
///
/// Raises the marker type dropdown value changed event.
///
public void OnMarkerTypeDropdownValueChanged(int result)
{
if ((int)markerType != result)
{
markerType = (MarkerType)result;
bool arUcoCalibMode = markerType == MarkerType.ChArUcoBoard;
normalCalibrationOptionsGroup.gameObject.SetActive(!arUcoCalibMode);
arUcoCalibrationOptionsGroup.gameObject.SetActive(arUcoCalibMode);
if (isImagesInputMode)
{
InitializeImagesInputMode();
}
else
{
if (webCamTextureToMatHelper.IsInitialized())
webCamTextureToMatHelper.Initialize();
}
}
}
///
/// Raises the board size W dropdown value changed event.
///
public void OnBoardSizeWDropdownValueChanged(int result)
{
if ((int)boardSizeW != result + 1)
{
boardSizeW = (NumberOfBoardSizeWidth)(result + 1);
gridWidth = squareSize * ((int)boardSizeW - 1);
gridWidthInputField.text = gridWidth.ToString();
if (isImagesInputMode)
{
InitializeImagesInputMode();
}
else
{
if (webCamTextureToMatHelper.IsInitialized())
webCamTextureToMatHelper.Initialize();
}
}
}
///
/// Raises the board size H dropdown value changed event.
///
public void OnBoardSizeHDropdownValueChanged(int result)
{
if ((int)boardSizeH != result + 1)
{
boardSizeH = (NumberOfBoardSizeHeight)(result + 1);
if (isImagesInputMode)
{
InitializeImagesInputMode();
}
else
{
if (webCamTextureToMatHelper.IsInitialized())
webCamTextureToMatHelper.Initialize();
}
}
}
///
/// Raises the show undistort image toggle value changed event.
///
public void OnShowUndistortImageToggleValueChanged()
{
if (showUndistortImage != showUndistortImageToggle.isOn)
{
showUndistortImage = showUndistortImageToggle.isOn;
}
}
///
/// Raises the square size input field end edit event.
///
public void OnSquareSizeInputFieldEndEdit()
{
float f;
bool result = float.TryParse(squareSizeInputField.text, out f);
if (result)
{
squareSize = f;
squareSizeInputField.text = f.ToString();
}
else
{
squareSize = 1f;
squareSizeInputField.text = squareSize.ToString();
}
}
///
/// Raises the use new calibration method toggle value changed event.
///
public void OnUseNewCalibrationMethodToggleValueChanged()
{
if (useNewCalibrationMethod != useNewCalibrationMethodToggle.isOn)
{
useNewCalibrationMethod = useNewCalibrationMethodToggle.isOn;
}
}
///
/// Raises the grid width input field end edit event.
///
public void OnGridWidthInputFieldEndEdit()
{
float f;
bool result = float.TryParse(gridWidthInputField.text, out f);
if (result)
{
gridWidth = f;
gridWidthInputField.text = f.ToString();
}
else
{
gridWidth = squareSize * ((int)boardSizeW - 1);
gridWidthInputField.text = gridWidth.ToString();
}
}
///
/// Raises the dictionary id dropdown value changed event.
///
public void OnDictionaryIdDropdownValueChanged(int result)
{
if ((int)dictionaryId != result)
{
dictionaryId = (ArUcoDictionary)result;
dictionary = Objdetect.getPredefinedDictionary((int)dictionaryId);
if (isImagesInputMode)
{
InitializeImagesInputMode();
}
else
{
if (webCamTextureToMatHelper.IsInitialized())
webCamTextureToMatHelper.Initialize();
}
}
}
///
/// Raises the capture button click event.
///
public void OnCaptureButtonClick()
{
if (isImagesInputMode)
{
if (!isCalibrating)
InitializeImagesInputMode();
StartCoroutine("CalibrateCameraUsingImages");
}
else
{
shouldCaptureFrame = true;
}
}
///
/// Raises the reset button click event.
///
public void OnResetButtonClick()
{
if (isImagesInputMode)
{
if (!isCalibrating)
InitializeImagesInputMode();
}
else
{
ResetCalibration();
}
}
///
/// Raises the save button click event.
///
public void OnSaveButtonClick()
{
string saveDirectoryPath = Path.Combine(Application.persistentDataPath, "ArUcoCameraCalibrationExample");
if (!Directory.Exists(saveDirectoryPath))
{
Directory.CreateDirectory(saveDirectoryPath);
}
string calibratonDirectoryName = "camera_parameters" + bgrMat.width() + "x" + bgrMat.height();
string saveCalibratonFileDirectoryPath = Path.Combine(saveDirectoryPath, calibratonDirectoryName);
// Clean up old files.
if (Directory.Exists(saveCalibratonFileDirectoryPath))
{
DirectoryInfo directoryInfo = new DirectoryInfo(saveCalibratonFileDirectoryPath);
foreach (FileInfo fileInfo in directoryInfo.GetFiles())
{
if ((fileInfo.Attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly)
{
fileInfo.Attributes = FileAttributes.Normal;
}
}
if ((directoryInfo.Attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly)
{
directoryInfo.Attributes = FileAttributes.Directory;
}
directoryInfo.Delete(true);
}
Directory.CreateDirectory(saveCalibratonFileDirectoryPath);
// save the calibraton file.
string savePath = Path.Combine(saveCalibratonFileDirectoryPath, calibratonDirectoryName + ".xml");
int frameCount = (markerType == MarkerType.ChArUcoBoard) ? allCorners.Count : imagePoints.Count;
CameraParameters param = new CameraParameters(frameCount, bgrMat.width(), bgrMat.height(), calibrationFlags, camMatrix, distCoeffs, repErr);
XmlSerializer serializer = new XmlSerializer(typeof(CameraParameters));
using (var stream = new FileStream(savePath, FileMode.Create))
{
serializer.Serialize(stream, param);
}
// save the calibration images.
#if UNITY_WEBGL && !UNITY_EDITOR
string format = "jpg";
MatOfInt compressionParams = new MatOfInt(Imgcodecs.IMWRITE_JPEG_QUALITY, 100);
#else
string format = "png";
MatOfInt compressionParams = new MatOfInt(Imgcodecs.IMWRITE_PNG_COMPRESSION, 0);
#endif
for (int i = 0; i < allImgs.Count; ++i)
{
Imgcodecs.imwrite(Path.Combine(saveCalibratonFileDirectoryPath, calibratonDirectoryName + "_" + i.ToString("00") + "." + format), allImgs[i], compressionParams);
}
savePathInputField.text = savePath;
Debug.Log("Saved the CameraParameters to disk in XML file format.");
Debug.Log("savePath: " + savePath);
}
public enum MarkerType
{
ChessBoard,
CirclesGlid,
AsymmetricCirclesGlid,
ChArUcoBoard,
}
public enum NumberOfBoardSizeWidth
{
W_1 = 1,
W_2,
W_3,
W_4,
W_5,
W_6,
W_7,
W_8,
W_9,
W_10,
W_11,
W_12,
W_13,
W_14,
W_15,
}
public enum NumberOfBoardSizeHeight
{
H_1 = 1,
H_2,
H_3,
H_4,
H_5,
H_6,
H_7,
H_8,
H_9,
H_10,
H_11,
H_12,
H_13,
H_14,
H_15,
}
public enum ArUcoDictionary
{
DICT_4X4_50 = Objdetect.DICT_4X4_50,
DICT_4X4_100 = Objdetect.DICT_4X4_100,
DICT_4X4_250 = Objdetect.DICT_4X4_250,
DICT_4X4_1000 = Objdetect.DICT_4X4_1000,
DICT_5X5_50 = Objdetect.DICT_5X5_50,
DICT_5X5_100 = Objdetect.DICT_5X5_100,
DICT_5X5_250 = Objdetect.DICT_5X5_250,
DICT_5X5_1000 = Objdetect.DICT_5X5_1000,
DICT_6X6_50 = Objdetect.DICT_6X6_50,
DICT_6X6_100 = Objdetect.DICT_6X6_100,
DICT_6X6_250 = Objdetect.DICT_6X6_250,
DICT_6X6_1000 = Objdetect.DICT_6X6_1000,
DICT_7X7_50 = Objdetect.DICT_7X7_50,
DICT_7X7_100 = Objdetect.DICT_7X7_100,
DICT_7X7_250 = Objdetect.DICT_7X7_250,
DICT_7X7_1000 = Objdetect.DICT_7X7_1000,
DICT_ARUCO_ORIGINAL = Objdetect.DICT_ARUCO_ORIGINAL,
}
}
}