#if !UNITY_WSA_10_0
using UnityEngine;
using UnityEngine.SceneManagement;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Xml;
using OpenCVForUnity.CoreModule;
using OpenCVForUnity.ImgcodecsModule;
using OpenCVForUnity.TextModule;
using OpenCVForUnity.ImgprocModule;
using OpenCVForUnity.UnityUtils;
namespace OpenCVForUnityExample
{
///
/// Text Detection Example
/// A demo script of the Extremal Region Filter algorithm described in:Neumann L., Matas J.: Real-Time Scene Text Localization and Recognition, CVPR 2012.
/// Referring to https://github.com/opencv/opencv_contrib/blob/master/modules/text/samples/textdetection.py.
///
public class TextRecognitionExample : MonoBehaviour
{
///
/// IMAGE_FILENAME
///
protected static readonly string IMAGE_FILENAME = "OpenCVForUnity/text/test_text.jpg";
///
/// The image filepath.
///
string image_filepath;
///
/// TRAINED_CLASSIFIER_NM_1_FILENAME
///
protected static readonly string TRAINED_CLASSIFIER_NM_1_FILENAME = "OpenCVForUnity/text/trained_classifierNM1.xml";
///
/// The trained classifierNM1 filepath.
///
string trained_classifierNM1_filepath;
///
/// TRAINED_CLASSIFIER_NM_2_FILENAME
///
protected static readonly string TRAINED_CLASSIFIER_NM_2_FILENAME = "OpenCVForUnity/text/trained_classifierNM2.xml";
///
/// The trained classifierNM2 filepath.
///
string trained_classifierNM2_filepath;
///
/// CLASSIFIER_NM_2_FILENAME
///
protected static readonly string OCRHMM_TRANSITIONS_TABLE_FILENAME = "OpenCVForUnity/text/OCRHMM_transitions_table.xml";
///
/// The OCRHMM transitions table filepath.
///
string OCRHMM_transitions_table_filepath;
///
/// CLASSIFIER_NM_2_FILENAME
///
/// https://stackoverflow.com/questions/4666098/why-does-android-aapt-remove-gz-file-extension-of-assets
#if UNITY_ANDROID && !UNITY_EDITOR
protected static readonly string OCRHMM_KNN_MODEL_FILENAME = "OpenCVForUnity/text/OCRHMM_knn_model_data.xml";
#else
protected static readonly string OCRHMM_KNN_MODEL_FILENAME = "OpenCVForUnity/text/OCRHMM_knn_model_data.xml.gz";
#endif
///
/// The OCRHMM knn model data filepath.
///
string OCRHMM_knn_model_data_filepath;
#if UNITY_WEBGL
IEnumerator getFilePath_Coroutine;
#endif
// Use this for initialization
void Start()
{
#if UNITY_WEBGL
getFilePath_Coroutine = GetFilePath ();
StartCoroutine (getFilePath_Coroutine);
#else
image_filepath = Utils.getFilePath(IMAGE_FILENAME);
trained_classifierNM1_filepath = Utils.getFilePath(TRAINED_CLASSIFIER_NM_1_FILENAME);
trained_classifierNM2_filepath = Utils.getFilePath(TRAINED_CLASSIFIER_NM_2_FILENAME);
OCRHMM_transitions_table_filepath = Utils.getFilePath(OCRHMM_TRANSITIONS_TABLE_FILENAME);
OCRHMM_knn_model_data_filepath = Utils.getFilePath(OCRHMM_KNN_MODEL_FILENAME);
Run();
#endif
}
#if UNITY_WEBGL
private IEnumerator GetFilePath ()
{
var getFilePathAsync_0_Coroutine = Utils.getFilePathAsync (IMAGE_FILENAME, (result) => {
image_filepath = result;
});
yield return getFilePathAsync_0_Coroutine;
var getFilePathAsync_1_Coroutine = Utils.getFilePathAsync (TRAINED_CLASSIFIER_NM_1_FILENAME, (result) => {
trained_classifierNM1_filepath = result;
});
yield return getFilePathAsync_1_Coroutine;
var getFilePathAsync_2_Coroutine = Utils.getFilePathAsync (TRAINED_CLASSIFIER_NM_2_FILENAME, (result) => {
trained_classifierNM2_filepath = result;
});
yield return getFilePathAsync_2_Coroutine;
var getFilePathAsync_3_Coroutine = Utils.getFilePathAsync (OCRHMM_TRANSITIONS_TABLE_FILENAME, (result) => {
OCRHMM_transitions_table_filepath = result;
});
yield return getFilePathAsync_3_Coroutine;
var getFilePathAsync_4_Coroutine = Utils.getFilePathAsync (OCRHMM_KNN_MODEL_FILENAME, (result) => {
OCRHMM_knn_model_data_filepath = result;
});
yield return getFilePathAsync_4_Coroutine;
getFilePath_Coroutine = null;
Run ();
}
#endif
private void Run()
{
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode(true);
Mat frame = Imgcodecs.imread(image_filepath);
if (frame.empty())
{
Debug.LogError(IMAGE_FILENAME + " is not loaded. Please move from “OpenCVForUnity/StreamingAssets/OpenCVForUnity/” to “Assets/StreamingAssets/OpenCVForUnity/” folder.");
}
if (string.IsNullOrEmpty(trained_classifierNM1_filepath) || string.IsNullOrEmpty(trained_classifierNM2_filepath))
{
Debug.LogError(TRAINED_CLASSIFIER_NM_1_FILENAME + " or " + TRAINED_CLASSIFIER_NM_2_FILENAME + " is not loaded. Please move from “OpenCVForUnity/StreamingAssets/OpenCVForUnity/” to “Assets/StreamingAssets/OpenCVForUnity/” folder.");
}
if (string.IsNullOrEmpty(OCRHMM_transitions_table_filepath) || string.IsNullOrEmpty(OCRHMM_knn_model_data_filepath))
{
Debug.LogError(OCRHMM_TRANSITIONS_TABLE_FILENAME + " or " + OCRHMM_KNN_MODEL_FILENAME + " is not loaded. Please move from “OpenCVForUnity/StreamingAssets/OpenCVForUnity/” to “Assets/StreamingAssets/OpenCVForUnity/” folder.");
}
Mat binaryMat = new Mat();
Mat maskMat = new Mat();
List regions = new List();
ERFilter er_filter1 = Text.createERFilterNM1(trained_classifierNM1_filepath, 8, 0.00015f, 0.13f, 0.2f, true, 0.1f);
ERFilter er_filter2 = Text.createERFilterNM2(trained_classifierNM2_filepath, 0.5f);
Mat transition_p = new Mat(62, 62, CvType.CV_64FC1);
//string filename = "OCRHMM_transitions_table.xml";
//FileStorage fs(filename, FileStorage::READ);
//fs["transition_probabilities"] >> transition_p;
//fs.release();
//Load TransitionProbabilitiesData.
transition_p.put(0, 0, GetTransitionProbabilitiesData(OCRHMM_transitions_table_filepath));
Mat emission_p = Mat.eye(62, 62, CvType.CV_64FC1);
string voc = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
OCRHMMDecoder decoder = OCRHMMDecoder.create(
OCRHMM_knn_model_data_filepath,
voc, transition_p, emission_p);
//Text Detection
Imgproc.cvtColor(frame, frame, Imgproc.COLOR_BGR2RGB);
Imgproc.cvtColor(frame, binaryMat, Imgproc.COLOR_RGB2GRAY);
Imgproc.threshold(binaryMat, binaryMat, 0, 255, Imgproc.THRESH_BINARY | Imgproc.THRESH_OTSU);
Core.absdiff(binaryMat, new Scalar(255), maskMat);
Text.detectRegions(binaryMat, er_filter1, er_filter2, regions);
Debug.Log("regions.Count " + regions.Count);
MatOfRect groups_rects = new MatOfRect();
List rects = new List();
Text.erGrouping(frame, binaryMat, regions, groups_rects);
for (int i = 0; i < regions.Count; i++)
{
regions[i].Dispose();
}
regions.Clear();
rects.AddRange(groups_rects.toList());
groups_rects.Dispose();
//Text Recognition (OCR)
List detections = new List();
for (int i = 0; i < (int)rects.Count; i++)
{
Mat group_img = new Mat();
maskMat.submat(rects[i]).copyTo(group_img);
Core.copyMakeBorder(group_img, group_img, 15, 15, 15, 15, Core.BORDER_CONSTANT, new Scalar(0));
detections.Add(group_img);
}
Debug.Log("detections.Count " + detections.Count);
//#Visualization
for (int i = 0; i < rects.Count; i++)
{
Imgproc.rectangle(frame, new Point(rects[i].x, rects[i].y), new Point(rects[i].x + rects[i].width, rects[i].y + rects[i].height), new Scalar(255, 0, 0), 2);
Imgproc.rectangle(frame, new Point(rects[i].x, rects[i].y), new Point(rects[i].x + rects[i].width, rects[i].y + rects[i].height), new Scalar(255, 255, 255), 1);
string output = decoder.run(detections[i], 0);
if (!string.IsNullOrEmpty(output))
{
Debug.Log("output " + output);
Imgproc.putText(frame, output, new Point(rects[i].x, rects[i].y), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(0, 0, 255), 1, Imgproc.LINE_AA, false);
}
}
Texture2D texture = new Texture2D(frame.cols(), frame.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(frame, texture);
//Texture2D texture = new Texture2D (detections [0].cols (), detections [0].rows (), TextureFormat.RGBA32, false);
//
//Utils.matToTexture2D (detections [0], texture);
gameObject.GetComponent().material.mainTexture = texture;
for (int i = 0; i < detections.Count; i++)
{
detections[i].Dispose();
}
binaryMat.Dispose();
maskMat.Dispose();
Utils.setDebugMode(false);
}
// Update is called once per frame
void Update()
{
}
///
/// Gets the transition probabilities data.
///
/// The transition probabilities data.
/// File path.
double[] GetTransitionProbabilitiesData(string filePath)
{
XmlDocument xmlDoc = new XmlDocument();
xmlDoc.Load(filePath);
XmlNode dataNode = xmlDoc.GetElementsByTagName("data").Item(0);
//Debug.Log ("dataNode.InnerText " + dataNode.InnerText);
string[] dataString = dataNode.InnerText.Split(new string[] {
" ",
"\r\n", "\n"
}, StringSplitOptions.RemoveEmptyEntries);
//Debug.Log ("dataString.Length " + dataString.Length);
double[] data = new double[dataString.Length];
for (int i = 0; i < data.Length; i++)
{
try
{
data[i] = Convert.ToDouble(dataString[i]);
}
catch (FormatException)
{
Debug.Log("Unable to convert '{" + dataString[i] + "}' to a Double.");
}
catch (OverflowException)
{
Debug.Log("'{" + dataString[i] + "}' is outside the range of a Double.");
}
}
return data;
}
///
/// Raises the destroy event.
///
void OnDestroy()
{
#if UNITY_WEBGL
if (getFilePath_Coroutine != null) {
StopCoroutine (getFilePath_Coroutine);
((IDisposable)getFilePath_Coroutine).Dispose ();
}
#endif
}
///
/// Raises the back button click event.
///
public void OnBackButtonClick()
{
SceneManager.LoadScene("OpenCVForUnityExample");
}
}
}
#endif