/**************************************************************************** * Copyright 2019 Nreal Techonology Limited. All rights reserved. * * This file is part of NRSDK. * * https://www.nreal.ai/ * *****************************************************************************/ namespace NRKernal.Record { using UnityEngine; using NRKernal; using System; using System.Collections.Generic; /// /// Records a audio from the MR images directly to disk. MR images comes from rgb camera or rgb /// camera image and virtual image blending. The final audio recording will be stored on the file /// system in the MP4 format. public class NRAudioCapture : IDisposable { /// /// Indicates whether or not the AudioCapture instance is currently recording audio. /// True if this object is recording, false if not. public bool IsRecording { get; private set; } /// Encoder for the capture. private AudioEncoder m_AudioEncoder; private AudioFilterStream m_AudioFilterStream; public event AudioDataCallBack OnAudioData; public int BytesPerSample { get { return NativeConstants.RECORD_AUDIO_BYTES_PER_SAMPLE; }} public int Channels { get { return NativeConstants.RECORD_AUDIO_CHANNEL; }} public int SamplesPerSec { get { return NativeConstants.RECORD_AUDIO_SAMPLERATE_DEFAULT; }} /// Create a NRAudioCapture object. public static NRAudioCapture Create() { NRAudioCapture capture = new NRAudioCapture(); return capture; } /// Default constructor. public NRAudioCapture() { IsRecording = false; } /// Finalizer. ~NRAudioCapture() { } /// Dispose must be called to shutdown the AudioCapture instance. public void Dispose() { if (m_AudioEncoder != null) { m_AudioEncoder.Release(); m_AudioEncoder = null; } if (m_AudioFilterStream != null) { m_AudioFilterStream.Dispose(); m_AudioFilterStream = null; } } /// Starts recording asynchronous. /// Filename of the file. /// The on started recording video callback. public void StartRecordingAsync(string filename, OnStartedRecordingAudioCallback onStartedRecordingAudioCallback) { float volumeFactorMic = 1.0f; float volumeFactorApp = 1.0f; if (NRDevice.Subsystem.GetDeviceType() == NRDeviceType.NrealLight) { volumeFactorMic = NativeConstants.RECORD_VOLUME_MIC; volumeFactorApp = NativeConstants.RECORD_VOLUME_APP; } StartRecordingAsync(filename, onStartedRecordingAudioCallback, volumeFactorMic, volumeFactorApp); } /// Starts recording asynchronous. /// Filename of the file. /// The on started recording audio callback. public void StartRecordingAsync(string filename, OnStartedRecordingAudioCallback onStartedRecordingAudioCallback, float volumeFactorMic, float volumeFactorApp) { NRDebugger.Info("[AudioCapture] StartRecordingAsync: IsRecording={0}, volFactorMic={1}, volFactorApp={2}", IsRecording, volumeFactorMic, volumeFactorApp); var result = new AudioCaptureResult(); if (IsRecording) { result.resultType = CaptureResultType.UnknownError; onStartedRecordingAudioCallback?.Invoke(result); } else { try { if (m_AudioFilterStream == null) { m_AudioFilterStream = new AudioFilterStream(); } m_AudioEncoder.AdjustVolume(RecorderIndex.REC_MIC, volumeFactorMic); m_AudioEncoder.AdjustVolume(RecorderIndex.REC_APP, volumeFactorApp); m_AudioEncoder.EncodeConfig.SetOutPutPath(filename); m_AudioEncoder.Start(OnAudioDataCallback); IsRecording = true; result.resultType = CaptureResultType.Success; onStartedRecordingAudioCallback?.Invoke(result); } catch (Exception ex) { NRDebugger.Warning("[AudioCapture] StartRecordingAsync: {0}\n{1}", ex.Message, ex.StackTrace); result.resultType = CaptureResultType.UnknownError; onStartedRecordingAudioCallback?.Invoke(result); throw; } } } private void OnAudioDataCallback(IntPtr data, UInt32 size) { // NRDebugger.Warning("[AudioCapture] OnAudioDataCallback: size={0}, data={1}", size, data); if (m_AudioFilterStream != null) { m_AudioFilterStream.OnAudioDataRead(data, size); } OnAudioData?.Invoke(data, size); } public bool FlushAudioData(ref byte[] outBytesData, ref int samples) { if (m_AudioFilterStream != null) { if (m_AudioFilterStream.Flush(ref outBytesData)) { samples = outBytesData.Length / Channels / BytesPerSample; return true; } } return false; } /// Starts audio mode asynchronous. /// Options for controlling the setup. /// The on audio mode started callback. /// Auto adaption for BlendMode based on supported feature on current device. public void StartAudioModeAsync(CameraParameters setupParams, OnAudioModeStartedCallback onAudioModeStartedCallback) { if (Application.isEditor || Application.platform != RuntimePlatform.Android) { StartAudioMode(setupParams, onAudioModeStartedCallback); return; } bool recordMic = setupParams.CaptureAudioMic; bool recordApp = setupParams.CaptureAudioApplication; if (recordApp) { NRAndroidPermissionsManager.GetInstance().RequestAndroidPermission("android.permission.RECORD_AUDIO").ThenAction((requestResult) => { if (requestResult.IsAllGranted) { NRAndroidPermissionsManager.GetInstance().RequestScreenCapture().ThenAction((AndroidJavaObject mediaProjection) => { if (mediaProjection != null) { setupParams.mediaProjection = mediaProjection; StartAudioMode(setupParams, onAudioModeStartedCallback); } else { NRDebugger.Error("[AudioCapture] Screen capture is denied by user."); var result = new AudioCaptureResult(); result.resultType = CaptureResultType.UnknownError; onAudioModeStartedCallback?.Invoke(result); NRSessionManager.Instance.OprateInitException(new NRPermissionDenyError(NativeConstants.ScreenCaptureDenyErrorTip)); } }); } else { NRDebugger.Error("[AudioCapture] Record audio need the permission of 'android.permission.RECORD_AUDIO'."); var result = new AudioCaptureResult(); result.resultType = CaptureResultType.UnknownError; onAudioModeStartedCallback?.Invoke(result); NRSessionManager.Instance.OprateInitException(new NRPermissionDenyError(NativeConstants.AudioPermissionDenyErrorTip)); } }); } else if (recordMic) { NRAndroidPermissionsManager.GetInstance().RequestAndroidPermission("android.permission.RECORD_AUDIO").ThenAction((requestResult) => { if (requestResult.IsAllGranted) { StartAudioMode(setupParams, onAudioModeStartedCallback); } else { NRDebugger.Error("[AudioCapture] Record audio need the permission of 'android.permission.RECORD_AUDIO'."); var result = new AudioCaptureResult(); result.resultType = CaptureResultType.UnknownError; onAudioModeStartedCallback?.Invoke(result); NRSessionManager.Instance.OprateInitException(new NRPermissionDenyError(NativeConstants.AudioPermissionDenyErrorTip)); } }); } else { StartAudioMode(setupParams, onAudioModeStartedCallback); } } private void StartAudioMode(CameraParameters setupParams, OnAudioModeStartedCallback onAudioModeStartedCallback) { setupParams.camMode = CamMode.None; if (setupParams.frameRate <= 0) NRDebugger.Warning("[AudioCapture] frameRate need to be bigger than zero"); m_AudioEncoder = new AudioEncoder(); m_AudioEncoder.Config(setupParams); var result = new AudioCaptureResult(); result.resultType = CaptureResultType.Success; onAudioModeStartedCallback?.Invoke(result); } /// Stops recording asynchronous. /// The on stopped recording audio callback. public void StopRecordingAsync(OnStoppedRecordingAudioCallback onStoppedRecordingAudioCallback) { var result = new AudioCaptureResult(); if (!IsRecording) { result.resultType = CaptureResultType.UnknownError; onStoppedRecordingAudioCallback?.Invoke(result); } else { try { m_AudioEncoder.Stop(); IsRecording = false; result.resultType = CaptureResultType.Success; onStoppedRecordingAudioCallback?.Invoke(result); } catch (Exception) { result.resultType = CaptureResultType.UnknownError; onStoppedRecordingAudioCallback?.Invoke(result); throw; } } } /// Stops video mode asynchronous. /// The on video mode stopped callback. public void StopAudioModeAsync(OnAudioModeStoppedCallback onAudioModeStoppedCallback) { m_AudioEncoder?.Release(); m_AudioEncoder = null; var result = new AudioCaptureResult(); result.resultType = CaptureResultType.Success; onAudioModeStoppedCallback?.Invoke(result); } /// Contains the result of the capture request. public enum CaptureResultType { /// /// Specifies that the desired operation was successful. /// Success = 0, /// /// Specifies that an unknown error occurred. /// UnknownError = 1 } /// /// A data container that contains the result information of a audio recording operation. public struct AudioCaptureResult { /// /// A generic result that indicates whether or not the AudioCapture operation succeeded. public CaptureResultType resultType; /// The specific HResult value. public long hResult; /// Indicates whether or not the operation was successful. /// True if success, false if not. public bool success { get { return resultType == CaptureResultType.Success; } } } /// Called when the web camera begins recording the audio. /// Indicates whether or not audio recording started successfully. public delegate void OnStartedRecordingAudioCallback(AudioCaptureResult result); /// Called when audio mode has been started. /// Indicates whether or not audio mode was successfully activated. public delegate void OnAudioModeStartedCallback(AudioCaptureResult result); /// Called when audio mode has been stopped. /// Indicates whether or not audio mode was successfully deactivated. public delegate void OnAudioModeStoppedCallback(AudioCaptureResult result); /// Called when the audio recording has been saved to the file system. /// Indicates whether or not audio recording was saved successfully to the /// file system. public delegate void OnStoppedRecordingAudioCallback(AudioCaptureResult result); } }