#if !UNITY_WSA_10_0
using System;
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.UI;
using UnityEngine.SceneManagement;
using OpenCVForUnity.CoreModule;
using OpenCVForUnity.DnnModule;
using OpenCVForUnity.ImgprocModule;
using OpenCVForUnity.UnityUtils;
using OpenCVForUnity.UnityUtils.Helper;
namespace OpenCVForUnityExample
{
///
/// TensorFlow WebCamTexture Example
/// An example of using OpenCV dnn module with Tensorflow Inception model.
/// Referring to https://github.com/opencv/opencv/blob/master/samples/dnn/tf_inception.cpp.
///
[RequireComponent (typeof(WebCamTextureToMatHelper))]
public class TensorFlowWebCamTextureExample : MonoBehaviour
{
///
/// The texture.
///
Texture2D texture;
///
/// The webcam texture to mat helper.
///
WebCamTextureToMatHelper webCamTextureToMatHelper;
///
/// The bgr mat.
///
Mat bgrMat;
///
/// The BLOB.
///
Mat blob;
///
/// The net.
///
Net net;
///
/// The classes.
///
List classes;
///
/// The FPS monitor.
///
FpsMonitor fpsMonitor;
string tensorflow_inception_graph_pb_filepath;
string imagenet_comp_graph_label_strings_txt_filepath;
#if UNITY_WEBGL && !UNITY_EDITOR
IEnumerator getFilePath_Coroutine;
#endif
// Use this for initialization
void Start ()
{
fpsMonitor = GetComponent ();
webCamTextureToMatHelper = gameObject.GetComponent ();
#if UNITY_WEBGL && !UNITY_EDITOR
getFilePath_Coroutine = GetFilePath ();
StartCoroutine (getFilePath_Coroutine);
#else
tensorflow_inception_graph_pb_filepath = Utils.getFilePath ("dnn/tensorflow_inception_graph.pb");
imagenet_comp_graph_label_strings_txt_filepath = Utils.getFilePath ("dnn/imagenet_comp_graph_label_strings.txt");
Run ();
#endif
}
#if UNITY_WEBGL && !UNITY_EDITOR
private IEnumerator GetFilePath()
{
var getFilePathAsync_0_Coroutine = Utils.getFilePathAsync ("dnn/tensorflow_inception_graph.pb", (result) => {
tensorflow_inception_graph_pb_filepath = result;
});
yield return getFilePathAsync_0_Coroutine;
var getFilePathAsync_1_Coroutine = Utils.getFilePathAsync ("dnn/imagenet_comp_graph_label_strings.txt", (result) => {
imagenet_comp_graph_label_strings_txt_filepath = result;
});
yield return getFilePathAsync_1_Coroutine;
getFilePath_Coroutine = null;
Run ();
}
#endif
// Use this for initialization
void Run ()
{
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode (true);
net = Dnn.readNetFromTensorflow (tensorflow_inception_graph_pb_filepath);
#if !UNITY_WSA_10_0
if (net.empty ()) {
Debug.LogError ("model file is not loaded. The model and class names list can be downloaded here: \"https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip\". Please copy to “Assets/StreamingAssets/dnn/” folder. ");
}
#endif
classes = readClassNames (imagenet_comp_graph_label_strings_txt_filepath);
#if !UNITY_WSA_10_0
if (classes == null) {
Debug.LogError ("class names list file is not loaded. The model and class names list can be downloaded here: \"https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip\". Please copy to “Assets/StreamingAssets/dnn/” folder. ");
}
#endif
#if UNITY_ANDROID && !UNITY_EDITOR
// Avoids the front camera low light issue that occurs in only some Android devices (e.g. Google Pixel, Pixel2).
webCamTextureToMatHelper.avoidAndroidFrontCameraLowLightIssue = true;
#endif
webCamTextureToMatHelper.Initialize ();
}
///
/// Raises the webcam texture to mat helper initialized event.
///
public void OnWebCamTextureToMatHelperInitialized ()
{
Debug.Log ("OnWebCamTextureToMatHelperInitialized");
Mat webCamTextureMat = webCamTextureToMatHelper.GetMat ();
texture = new Texture2D (webCamTextureMat.cols (), webCamTextureMat.rows (), TextureFormat.RGBA32, false);
gameObject.GetComponent ().material.mainTexture = texture;
gameObject.transform.localScale = new Vector3 (webCamTextureMat.cols (), webCamTextureMat.rows (), 1);
Debug.Log ("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
if (fpsMonitor != null) {
fpsMonitor.Add ("width", webCamTextureMat.width ().ToString ());
fpsMonitor.Add ("height", webCamTextureMat.height ().ToString ());
fpsMonitor.Add ("orientation", Screen.orientation.ToString ());
}
float width = webCamTextureMat.width ();
float height = webCamTextureMat.height ();
float widthScale = (float)Screen.width / width;
float heightScale = (float)Screen.height / height;
if (widthScale < heightScale) {
Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2;
} else {
Camera.main.orthographicSize = height / 2;
}
bgrMat = new Mat (webCamTextureMat.rows (), webCamTextureMat.cols (), CvType.CV_8UC3);
}
///
/// Raises the webcam texture to mat helper disposed event.
///
public void OnWebCamTextureToMatHelperDisposed ()
{
Debug.Log ("OnWebCamTextureToMatHelperDisposed");
if (bgrMat != null)
bgrMat.Dispose ();
if (texture != null) {
Texture2D.Destroy (texture);
texture = null;
}
}
///
/// Raises the webcam texture to mat helper error occurred event.
///
/// Error code.
public void OnWebCamTextureToMatHelperErrorOccurred (WebCamTextureToMatHelper.ErrorCode errorCode)
{
Debug.Log ("OnWebCamTextureToMatHelperErrorOccurred " + errorCode);
}
// Update is called once per frame
void Update ()
{
if (webCamTextureToMatHelper.IsPlaying () && webCamTextureToMatHelper.DidUpdateThisFrame ()) {
Mat rgbaMat = webCamTextureToMatHelper.GetMat ();
if (net.empty () || classes == null) {
Imgproc.putText (rgbaMat, "model file is not loaded.", new Point (5, rgbaMat.rows () - 30), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText (rgbaMat, "Please read console message.", new Point (5, rgbaMat.rows () - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
} else {
Imgproc.cvtColor (rgbaMat, bgrMat, Imgproc.COLOR_RGBA2BGR);
blob = Dnn.blobFromImage (bgrMat, 1, new Size (224, 224), new Scalar (104, 117, 123), false, true);
net.setInput (blob);
Mat prob = net.forward ();
Core.MinMaxLocResult minmax = Core.minMaxLoc (prob.reshape (1, 1));
// Debug.Log ("Best match " + (int)minmax.maxLoc.x);
// Debug.Log ("Best match class " + classes [(int)minmax.maxLoc.x]);
// Debug.Log ("Probability: " + minmax.maxVal * 100 + "%");
prob.Dispose ();
//Imgproc.putText (rgbaMat, "Best match class " + classes [(int)minmax.maxLoc.x], new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
if (fpsMonitor != null) {
fpsMonitor.consoleText = "Best match class " + classes [(int)minmax.maxLoc.x];
}
}
Utils.fastMatToTexture2D (rgbaMat, texture);
}
}
///
/// Raises the disable event.
///
void OnDisable ()
{
webCamTextureToMatHelper.Dispose ();
if (blob != null)
blob.Dispose ();
if (net != null)
net.Dispose ();
Utils.setDebugMode (false);
#if UNITY_WEBGL && !UNITY_EDITOR
if (getFilePath_Coroutine != null) {
StopCoroutine (getFilePath_Coroutine);
((IDisposable)getFilePath_Coroutine).Dispose ();
}
#endif
}
///
/// Raises the back button click event.
///
public void OnBackButtonClick ()
{
SceneManager.LoadScene ("OpenCVForUnityExample");
}
///
/// Raises the play button click event.
///
public void OnPlayButtonClick ()
{
webCamTextureToMatHelper.Play ();
}
///
/// Raises the pause button click event.
///
public void OnPauseButtonClick ()
{
webCamTextureToMatHelper.Pause ();
}
///
/// Raises the stop button click event.
///
public void OnStopButtonClick ()
{
webCamTextureToMatHelper.Stop ();
}
///
/// Raises the change camera button click event.
///
public void OnChangeCameraButtonClick ()
{
webCamTextureToMatHelper.requestedIsFrontFacing = !webCamTextureToMatHelper.IsFrontFacing ();
}
private List readClassNames (string filename)
{
List classNames = new List ();
System.IO.StreamReader cReader = null;
try {
cReader = new System.IO.StreamReader (filename, System.Text.Encoding.Default);
while (cReader.Peek () >= 0) {
string name = cReader.ReadLine ();
classNames.Add (name);
}
} catch (System.Exception ex) {
Debug.LogError (ex.Message);
return null;
} finally {
if (cReader != null)
cReader.Close ();
}
return classNames;
}
}
}
#endif