#if !UNITY_WSA_10_0 using OpenCVForUnity.CoreModule; using OpenCVForUnity.ImgcodecsModule; using OpenCVForUnity.ImgprocModule; using OpenCVForUnity.ObjdetectModule; using OpenCVForUnity.UnityUtils; using System; using System.Collections; using UnityEngine; using UnityEngine.SceneManagement; using Rect = OpenCVForUnity.CoreModule.Rect; namespace OpenCVForUnityExample { /// /// FaceRecognizerSF Example /// An example of human face recognition using the FaceRecognizerSF class. /// https://github.com/opencv/opencv/blob/master/samples/dnn/face_detect.cpp /// https://docs.opencv.org/4.5.4/d0/dd4/tutorial_dnn_face.html /// public class FaceRecognizerSFExample : MonoBehaviour { /// /// Filter out faces of score < score_threshold. /// float scoreThreshold = 0.9f; /// /// Suppress bounding boxes of iou >= nms_threshold /// float nmsThreshold = 0.3f; /// /// Keep top_k bounding boxes before NMS. /// int topK = 5000; /// /// FD_MODEL_FILENAME /// protected static readonly string FD_MODEL_FILENAME = "OpenCVForUnity/objdetect/face_detection_yunet_2023mar.onnx"; /// /// The fd model filepath. /// string fd_model_filepath; /// /// The cosine similar thresh. /// double cosine_similar_thresh = 0.363; /// /// The l2norm similar thresh. /// double l2norm_similar_thresh = 1.128; /// /// SF_MODEL_FILENAME /// protected static readonly string SF_MODEL_FILENAME = "OpenCVForUnity/objdetect/face_recognition_sface_2021dec.onnx"; /// /// The sf model filepath. /// string sf_model_filepath; /// /// IMAGE_0_FILENAME /// protected static readonly string IMAGE_0_FILENAME = "OpenCVForUnity/face/facerec_0.bmp"; /// /// The image 0 filepath. /// string image_0_filepath; /// /// IMAGE_1_FILENAME /// protected static readonly string IMAGE_1_FILENAME = "OpenCVForUnity/face/facerec_1.bmp"; /// /// The image 1 filepath. /// string image_1_filepath; /// /// SAMPLE_IMAGE_FILENAME /// protected static readonly string SAMPLE_IMAGE_FILENAME = "OpenCVForUnity/face/facerec_sample.bmp"; /// /// The sample image filepath. /// string sample_image_filepath; #if UNITY_WEBGL IEnumerator getFilePath_Coroutine; #endif // Use this for initialization void Start() { #if UNITY_WEBGL getFilePath_Coroutine = GetFilePath (); StartCoroutine (getFilePath_Coroutine); #else fd_model_filepath = Utils.getFilePath(FD_MODEL_FILENAME); sf_model_filepath = Utils.getFilePath(SF_MODEL_FILENAME); image_0_filepath = Utils.getFilePath(IMAGE_0_FILENAME); image_1_filepath = Utils.getFilePath(IMAGE_1_FILENAME); sample_image_filepath = Utils.getFilePath(SAMPLE_IMAGE_FILENAME); Run(); #endif } #if UNITY_WEBGL private IEnumerator GetFilePath() { var getFilePathAsync_fd_Coroutine = Utils.getFilePathAsync(FD_MODEL_FILENAME, (result) => { fd_model_filepath = result; }); yield return getFilePathAsync_fd_Coroutine; var getFilePathAsync_sf_Coroutine = Utils.getFilePathAsync(SF_MODEL_FILENAME, (result) => { sf_model_filepath = result; }); yield return getFilePathAsync_sf_Coroutine; var getFilePathAsync_0_Coroutine = Utils.getFilePathAsync (IMAGE_0_FILENAME, (result) => { image_0_filepath = result; }); yield return getFilePathAsync_0_Coroutine; var getFilePathAsync_1_Coroutine = Utils.getFilePathAsync (IMAGE_1_FILENAME, (result) => { image_1_filepath = result; }); yield return getFilePathAsync_1_Coroutine; var getFilePathAsync_sample_Coroutine = Utils.getFilePathAsync (SAMPLE_IMAGE_FILENAME, (result) => { sample_image_filepath = result; }); yield return getFilePathAsync_sample_Coroutine; getFilePath_Coroutine = null; Run (); } #endif private void Run() { //if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console. Utils.setDebugMode(true); if (string.IsNullOrEmpty(image_0_filepath) || string.IsNullOrEmpty(image_1_filepath) || string.IsNullOrEmpty(sample_image_filepath)) { Debug.LogError(IMAGE_0_FILENAME + " or " + IMAGE_1_FILENAME + " or " + SAMPLE_IMAGE_FILENAME + " is not loaded. Please move from “OpenCVForUnity/StreamingAssets/OpenCVForUnity/” to “Assets/StreamingAssets/OpenCVForUnity/” folder."); } Mat testSampleMat = Imgcodecs.imread(sample_image_filepath, Imgcodecs.IMREAD_COLOR); Mat image0Mat = Imgcodecs.imread(image_0_filepath, Imgcodecs.IMREAD_COLOR); Mat image1Mat = Imgcodecs.imread(image_1_filepath, Imgcodecs.IMREAD_COLOR); int imageSizeW = 112; int imageSizeH = 112; Mat resultMat = new Mat(imageSizeH * 2, imageSizeW * 2, CvType.CV_8UC3, new Scalar(127, 127, 127, 255)); if (string.IsNullOrEmpty(fd_model_filepath) || string.IsNullOrEmpty(sf_model_filepath)) { Debug.LogError(FD_MODEL_FILENAME + " or " + SF_MODEL_FILENAME + " is not loaded. Please read “StreamingAssets/OpenCVForUnity/objdetect/setup_objdetect_module.pdf” to make the necessary setup."); Imgproc.putText(resultMat, "model file is not loaded.", new Point(5, resultMat.rows() - 30), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false); Imgproc.putText(resultMat, "Please read console message.", new Point(5, resultMat.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false); } else { FaceDetectorYN faceDetector = FaceDetectorYN.create(fd_model_filepath, "", new Size(imageSizeW, imageSizeH), scoreThreshold, nmsThreshold, topK); FaceRecognizerSF faceRecognizer = FaceRecognizerSF.create(sf_model_filepath, ""); // Detect faces. faceDetector.setInputSize(testSampleMat.size()); Mat faces_sample = new Mat(); faceDetector.detect(testSampleMat, faces_sample); if (faces_sample.rows() < 1) { Debug.Log("Cannot find a face in " + SAMPLE_IMAGE_FILENAME + "."); } faceDetector.setInputSize(image0Mat.size()); Mat faces_0 = new Mat(); faceDetector.detect(image0Mat, faces_0); if (faces_0.rows() < 1) { Debug.Log("Cannot find a face in " + IMAGE_0_FILENAME + "."); } faceDetector.setInputSize(image1Mat.size()); Mat faces_1 = new Mat(); faceDetector.detect(image1Mat, faces_1); if (faces_1.rows() < 1) { Debug.Log("Cannot find a face in " + IMAGE_1_FILENAME + "."); } /* // Draw results on the input image. DrawDetection(faces_sample.row(0),testSampleMat); DrawDetection(faces_0.row(0), image0Mat); DrawDetection(faces_1.row(0), image1Mat); */ // Aligning and cropping facial image through the first face of faces detected. Mat aligned_face_sample = new Mat(); faceRecognizer.alignCrop(testSampleMat, faces_sample.row(0), aligned_face_sample); Mat aligned_face_0 = new Mat(); faceRecognizer.alignCrop(image0Mat, faces_0.row(0), aligned_face_0); Mat aligned_face_1 = new Mat(); faceRecognizer.alignCrop(image1Mat, faces_1.row(0), aligned_face_1); // Run feature extraction with given aligned_face. Mat feature_sample = new Mat(); faceRecognizer.feature(aligned_face_sample, feature_sample); feature_sample = feature_sample.clone(); Mat feature_0 = new Mat(); faceRecognizer.feature(aligned_face_0, feature_0); feature_0 = feature_0.clone(); Mat feature_1 = new Mat(); faceRecognizer.feature(aligned_face_1, feature_1); feature_1 = feature_1.clone(); // Match the face features. bool sample_0_match = Match(faceRecognizer, feature_sample, feature_0); bool sample_1_match = Match(faceRecognizer, feature_sample, feature_1); Debug.Log("Match(" + SAMPLE_IMAGE_FILENAME + ", " + IMAGE_0_FILENAME + "): " + sample_0_match); Debug.Log("Match(" + SAMPLE_IMAGE_FILENAME + ", " + IMAGE_1_FILENAME + "): " + sample_1_match); // Draw the recognition result. aligned_face_sample.copyTo(resultMat.submat(new Rect(new Point(imageSizeW / 2, 0), aligned_face_sample.size()))); aligned_face_0.copyTo(resultMat.submat(new Rect(new Point(0, imageSizeH), aligned_face_0.size()))); aligned_face_1.copyTo(resultMat.submat(new Rect(new Point(imageSizeW, imageSizeH), aligned_face_1.size()))); Imgproc.putText(resultMat, "TestSample", new Point(imageSizeW / 2 + 5, 15), Imgproc.FONT_HERSHEY_SIMPLEX, 0.4, new Scalar(255, 255, 255, 255), 1, Imgproc.LINE_AA, false); Imgproc.putText(resultMat, "Image0", new Point(5, imageSizeH + 15), Imgproc.FONT_HERSHEY_SIMPLEX, 0.4, new Scalar(255, 255, 255, 255), 1, Imgproc.LINE_AA, false); Imgproc.putText(resultMat, "Image1", new Point(imageSizeW + 5, imageSizeH + 15), Imgproc.FONT_HERSHEY_SIMPLEX, 0.4, new Scalar(255, 255, 255, 255), 1, Imgproc.LINE_AA, false); if (sample_0_match) Imgproc.rectangle(resultMat, new Rect(0, imageSizeH, imageSizeW, imageSizeH), new Scalar(255, 0, 0, 255), 2); if (sample_1_match) Imgproc.rectangle(resultMat, new Rect(imageSizeW, imageSizeH, imageSizeW, imageSizeH), new Scalar(255, 0, 0, 255), 2); } Texture2D texture = new Texture2D(resultMat.cols(), resultMat.rows(), TextureFormat.RGB24, false); Utils.matToTexture2D(resultMat, texture); gameObject.GetComponent().material.mainTexture = texture; Utils.setDebugMode(false); } // Update is called once per frame void Update() { } /// /// Raises the destroy event. /// void OnDestroy() { #if UNITY_WEBGL if (getFilePath_Coroutine != null) { StopCoroutine (getFilePath_Coroutine); ((IDisposable)getFilePath_Coroutine).Dispose (); } #endif } /// /// Raises the back button click event. /// public void OnBackButtonClick() { SceneManager.LoadScene("OpenCVForUnityExample"); } protected void DrawDetection(Mat d, Mat frame) { float[] buf = new float[15]; d.get(0, 0, buf); Scalar color = new Scalar(255, 255, 0, 255); Imgproc.rectangle(frame, new Point(buf[0], buf[1]), new Point(buf[0] + buf[2], buf[1] + buf[3]), color, 2); Imgproc.circle(frame, new Point(buf[4], buf[5]), 2, color, 2); Imgproc.circle(frame, new Point(buf[6], buf[7]), 2, color, 2); Imgproc.circle(frame, new Point(buf[8], buf[9]), 2, color, 2); Imgproc.circle(frame, new Point(buf[10], buf[11]), 2, color, 2); Imgproc.circle(frame, new Point(buf[12], buf[13]), 2, color, 2); } protected bool Match(FaceRecognizerSF faceRecognizer, Mat feature1, Mat feature2) { double cos_score = faceRecognizer.match(feature1, feature2, FaceRecognizerSF.FR_COSINE); bool cos_match = (cos_score >= cosine_similar_thresh); Debug.Log((cos_match ? "They have the same identity;" : "They have different identities;") + "\n" + " Cosine Similarity: " + cos_score + ", threshold: " + cosine_similar_thresh + ". (higher value means higher similarity, max 1.0)"); double L2_score = faceRecognizer.match(feature1, feature2, FaceRecognizerSF.FR_NORM_L2); bool L2_match = (L2_score <= l2norm_similar_thresh); Debug.Log((L2_match ? "They have the same identity;" : "They have different identities;") + "\n" + " NormL2 Distance: " + L2_score + ", threshold: " + l2norm_similar_thresh + ". (lower value means higher similarity, min 0.0)"); return cos_match && L2_match; } } } #endif