#if !UNITY_WSA_10_0
using OpenCVForUnity.CoreModule;
using OpenCVForUnity.ImgcodecsModule;
using OpenCVForUnity.ImgprocModule;
using OpenCVForUnity.ObjdetectModule;
using OpenCVForUnity.UnityUtils;
using System;
using System.Collections;
using UnityEngine;
using UnityEngine.SceneManagement;
using Rect = OpenCVForUnity.CoreModule.Rect;
namespace OpenCVForUnityExample
{
///
/// FaceRecognizerSF Example
/// An example of human face recognition using the FaceRecognizerSF class.
/// https://github.com/opencv/opencv/blob/master/samples/dnn/face_detect.cpp
/// https://docs.opencv.org/4.5.4/d0/dd4/tutorial_dnn_face.html
///
public class FaceRecognizerSFExample : MonoBehaviour
{
///
/// Filter out faces of score < score_threshold.
///
float scoreThreshold = 0.9f;
///
/// Suppress bounding boxes of iou >= nms_threshold
///
float nmsThreshold = 0.3f;
///
/// Keep top_k bounding boxes before NMS.
///
int topK = 5000;
///
/// FD_MODEL_FILENAME
///
protected static readonly string FD_MODEL_FILENAME = "OpenCVForUnity/objdetect/face_detection_yunet_2023mar.onnx";
///
/// The fd model filepath.
///
string fd_model_filepath;
///
/// The cosine similar thresh.
///
double cosine_similar_thresh = 0.363;
///
/// The l2norm similar thresh.
///
double l2norm_similar_thresh = 1.128;
///
/// SF_MODEL_FILENAME
///
protected static readonly string SF_MODEL_FILENAME = "OpenCVForUnity/objdetect/face_recognition_sface_2021dec.onnx";
///
/// The sf model filepath.
///
string sf_model_filepath;
///
/// IMAGE_0_FILENAME
///
protected static readonly string IMAGE_0_FILENAME = "OpenCVForUnity/face/facerec_0.bmp";
///
/// The image 0 filepath.
///
string image_0_filepath;
///
/// IMAGE_1_FILENAME
///
protected static readonly string IMAGE_1_FILENAME = "OpenCVForUnity/face/facerec_1.bmp";
///
/// The image 1 filepath.
///
string image_1_filepath;
///
/// SAMPLE_IMAGE_FILENAME
///
protected static readonly string SAMPLE_IMAGE_FILENAME = "OpenCVForUnity/face/facerec_sample.bmp";
///
/// The sample image filepath.
///
string sample_image_filepath;
#if UNITY_WEBGL
IEnumerator getFilePath_Coroutine;
#endif
// Use this for initialization
void Start()
{
#if UNITY_WEBGL
getFilePath_Coroutine = GetFilePath ();
StartCoroutine (getFilePath_Coroutine);
#else
fd_model_filepath = Utils.getFilePath(FD_MODEL_FILENAME);
sf_model_filepath = Utils.getFilePath(SF_MODEL_FILENAME);
image_0_filepath = Utils.getFilePath(IMAGE_0_FILENAME);
image_1_filepath = Utils.getFilePath(IMAGE_1_FILENAME);
sample_image_filepath = Utils.getFilePath(SAMPLE_IMAGE_FILENAME);
Run();
#endif
}
#if UNITY_WEBGL
private IEnumerator GetFilePath()
{
var getFilePathAsync_fd_Coroutine = Utils.getFilePathAsync(FD_MODEL_FILENAME, (result) => {
fd_model_filepath = result;
});
yield return getFilePathAsync_fd_Coroutine;
var getFilePathAsync_sf_Coroutine = Utils.getFilePathAsync(SF_MODEL_FILENAME, (result) => {
sf_model_filepath = result;
});
yield return getFilePathAsync_sf_Coroutine;
var getFilePathAsync_0_Coroutine = Utils.getFilePathAsync (IMAGE_0_FILENAME, (result) => {
image_0_filepath = result;
});
yield return getFilePathAsync_0_Coroutine;
var getFilePathAsync_1_Coroutine = Utils.getFilePathAsync (IMAGE_1_FILENAME, (result) => {
image_1_filepath = result;
});
yield return getFilePathAsync_1_Coroutine;
var getFilePathAsync_sample_Coroutine = Utils.getFilePathAsync (SAMPLE_IMAGE_FILENAME, (result) => {
sample_image_filepath = result;
});
yield return getFilePathAsync_sample_Coroutine;
getFilePath_Coroutine = null;
Run ();
}
#endif
private void Run()
{
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode(true);
if (string.IsNullOrEmpty(image_0_filepath) || string.IsNullOrEmpty(image_1_filepath) || string.IsNullOrEmpty(sample_image_filepath))
{
Debug.LogError(IMAGE_0_FILENAME + " or " + IMAGE_1_FILENAME + " or " + SAMPLE_IMAGE_FILENAME + " is not loaded. Please move from “OpenCVForUnity/StreamingAssets/OpenCVForUnity/” to “Assets/StreamingAssets/OpenCVForUnity/” folder.");
}
Mat testSampleMat = Imgcodecs.imread(sample_image_filepath, Imgcodecs.IMREAD_COLOR);
Mat image0Mat = Imgcodecs.imread(image_0_filepath, Imgcodecs.IMREAD_COLOR);
Mat image1Mat = Imgcodecs.imread(image_1_filepath, Imgcodecs.IMREAD_COLOR);
int imageSizeW = 112;
int imageSizeH = 112;
Mat resultMat = new Mat(imageSizeH * 2, imageSizeW * 2, CvType.CV_8UC3, new Scalar(127, 127, 127, 255));
if (string.IsNullOrEmpty(fd_model_filepath) || string.IsNullOrEmpty(sf_model_filepath))
{
Debug.LogError(FD_MODEL_FILENAME + " or " + SF_MODEL_FILENAME + " is not loaded. Please read “StreamingAssets/OpenCVForUnity/objdetect/setup_objdetect_module.pdf” to make the necessary setup.");
Imgproc.putText(resultMat, "model file is not loaded.", new Point(5, resultMat.rows() - 30), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(resultMat, "Please read console message.", new Point(5, resultMat.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
FaceDetectorYN faceDetector = FaceDetectorYN.create(fd_model_filepath, "", new Size(imageSizeW, imageSizeH), scoreThreshold, nmsThreshold, topK);
FaceRecognizerSF faceRecognizer = FaceRecognizerSF.create(sf_model_filepath, "");
// Detect faces.
faceDetector.setInputSize(testSampleMat.size());
Mat faces_sample = new Mat();
faceDetector.detect(testSampleMat, faces_sample);
if (faces_sample.rows() < 1)
{
Debug.Log("Cannot find a face in " + SAMPLE_IMAGE_FILENAME + ".");
}
faceDetector.setInputSize(image0Mat.size());
Mat faces_0 = new Mat();
faceDetector.detect(image0Mat, faces_0);
if (faces_0.rows() < 1)
{
Debug.Log("Cannot find a face in " + IMAGE_0_FILENAME + ".");
}
faceDetector.setInputSize(image1Mat.size());
Mat faces_1 = new Mat();
faceDetector.detect(image1Mat, faces_1);
if (faces_1.rows() < 1)
{
Debug.Log("Cannot find a face in " + IMAGE_1_FILENAME + ".");
}
/*
// Draw results on the input image.
DrawDetection(faces_sample.row(0),testSampleMat);
DrawDetection(faces_0.row(0), image0Mat);
DrawDetection(faces_1.row(0), image1Mat);
*/
// Aligning and cropping facial image through the first face of faces detected.
Mat aligned_face_sample = new Mat();
faceRecognizer.alignCrop(testSampleMat, faces_sample.row(0), aligned_face_sample);
Mat aligned_face_0 = new Mat();
faceRecognizer.alignCrop(image0Mat, faces_0.row(0), aligned_face_0);
Mat aligned_face_1 = new Mat();
faceRecognizer.alignCrop(image1Mat, faces_1.row(0), aligned_face_1);
// Run feature extraction with given aligned_face.
Mat feature_sample = new Mat();
faceRecognizer.feature(aligned_face_sample, feature_sample);
feature_sample = feature_sample.clone();
Mat feature_0 = new Mat();
faceRecognizer.feature(aligned_face_0, feature_0);
feature_0 = feature_0.clone();
Mat feature_1 = new Mat();
faceRecognizer.feature(aligned_face_1, feature_1);
feature_1 = feature_1.clone();
// Match the face features.
bool sample_0_match = Match(faceRecognizer, feature_sample, feature_0);
bool sample_1_match = Match(faceRecognizer, feature_sample, feature_1);
Debug.Log("Match(" + SAMPLE_IMAGE_FILENAME + ", " + IMAGE_0_FILENAME + "): " + sample_0_match);
Debug.Log("Match(" + SAMPLE_IMAGE_FILENAME + ", " + IMAGE_1_FILENAME + "): " + sample_1_match);
// Draw the recognition result.
aligned_face_sample.copyTo(resultMat.submat(new Rect(new Point(imageSizeW / 2, 0), aligned_face_sample.size())));
aligned_face_0.copyTo(resultMat.submat(new Rect(new Point(0, imageSizeH), aligned_face_0.size())));
aligned_face_1.copyTo(resultMat.submat(new Rect(new Point(imageSizeW, imageSizeH), aligned_face_1.size())));
Imgproc.putText(resultMat, "TestSample", new Point(imageSizeW / 2 + 5, 15), Imgproc.FONT_HERSHEY_SIMPLEX, 0.4, new Scalar(255, 255, 255, 255), 1, Imgproc.LINE_AA, false);
Imgproc.putText(resultMat, "Image0", new Point(5, imageSizeH + 15), Imgproc.FONT_HERSHEY_SIMPLEX, 0.4, new Scalar(255, 255, 255, 255), 1, Imgproc.LINE_AA, false);
Imgproc.putText(resultMat, "Image1", new Point(imageSizeW + 5, imageSizeH + 15), Imgproc.FONT_HERSHEY_SIMPLEX, 0.4, new Scalar(255, 255, 255, 255), 1, Imgproc.LINE_AA, false);
if (sample_0_match)
Imgproc.rectangle(resultMat, new Rect(0, imageSizeH, imageSizeW, imageSizeH), new Scalar(255, 0, 0, 255), 2);
if (sample_1_match)
Imgproc.rectangle(resultMat, new Rect(imageSizeW, imageSizeH, imageSizeW, imageSizeH), new Scalar(255, 0, 0, 255), 2);
}
Texture2D texture = new Texture2D(resultMat.cols(), resultMat.rows(), TextureFormat.RGB24, false);
Utils.matToTexture2D(resultMat, texture);
gameObject.GetComponent().material.mainTexture = texture;
Utils.setDebugMode(false);
}
// Update is called once per frame
void Update()
{
}
///
/// Raises the destroy event.
///
void OnDestroy()
{
#if UNITY_WEBGL
if (getFilePath_Coroutine != null) {
StopCoroutine (getFilePath_Coroutine);
((IDisposable)getFilePath_Coroutine).Dispose ();
}
#endif
}
///
/// Raises the back button click event.
///
public void OnBackButtonClick()
{
SceneManager.LoadScene("OpenCVForUnityExample");
}
protected void DrawDetection(Mat d, Mat frame)
{
float[] buf = new float[15];
d.get(0, 0, buf);
Scalar color = new Scalar(255, 255, 0, 255);
Imgproc.rectangle(frame, new Point(buf[0], buf[1]), new Point(buf[0] + buf[2], buf[1] + buf[3]), color, 2);
Imgproc.circle(frame, new Point(buf[4], buf[5]), 2, color, 2);
Imgproc.circle(frame, new Point(buf[6], buf[7]), 2, color, 2);
Imgproc.circle(frame, new Point(buf[8], buf[9]), 2, color, 2);
Imgproc.circle(frame, new Point(buf[10], buf[11]), 2, color, 2);
Imgproc.circle(frame, new Point(buf[12], buf[13]), 2, color, 2);
}
protected bool Match(FaceRecognizerSF faceRecognizer, Mat feature1, Mat feature2)
{
double cos_score = faceRecognizer.match(feature1, feature2, FaceRecognizerSF.FR_COSINE);
bool cos_match = (cos_score >= cosine_similar_thresh);
Debug.Log((cos_match ? "They have the same identity;" : "They have different identities;") + "\n"
+ " Cosine Similarity: " + cos_score + ", threshold: " + cosine_similar_thresh + ". (higher value means higher similarity, max 1.0)");
double L2_score = faceRecognizer.match(feature1, feature2, FaceRecognizerSF.FR_NORM_L2);
bool L2_match = (L2_score <= l2norm_similar_thresh);
Debug.Log((L2_match ? "They have the same identity;" : "They have different identities;") + "\n"
+ " NormL2 Distance: " + L2_score + ", threshold: " + l2norm_similar_thresh + ". (lower value means higher similarity, min 0.0)");
return cos_match && L2_match;
}
}
}
#endif