/****************************************************************************
* Copyright 2019 Nreal Techonology Limited. All rights reserved.
*
* This file is part of NRSDK.
*
* https://www.nreal.ai/
*
*****************************************************************************/
namespace NRKernal.Record
{
using UnityEngine;
using NRKernal;
using System;
using System.Collections.Generic;
///
/// Records a audio from the MR images directly to disk. MR images comes from rgb camera or rgb
/// camera image and virtual image blending. The final audio recording will be stored on the file
/// system in the MP4 format.
public class NRAudioCapture : IDisposable
{
///
/// Indicates whether or not the AudioCapture instance is currently recording audio.
/// True if this object is recording, false if not.
public bool IsRecording { get; private set; }
/// Encoder for the capture.
private AudioEncoder m_AudioEncoder;
private AudioFilterStream m_AudioFilterStream;
public event AudioDataCallBack OnAudioData;
public int BytesPerSample { get { return NativeConstants.RECORD_AUDIO_BYTES_PER_SAMPLE; }}
public int Channels { get { return NativeConstants.RECORD_AUDIO_CHANNEL; }}
public int SamplesPerSec { get { return NativeConstants.RECORD_AUDIO_SAMPLERATE_DEFAULT; }}
/// Create a NRAudioCapture object.
public static NRAudioCapture Create()
{
NRAudioCapture capture = new NRAudioCapture();
return capture;
}
/// Default constructor.
public NRAudioCapture()
{
IsRecording = false;
}
/// Finalizer.
~NRAudioCapture()
{
}
/// Dispose must be called to shutdown the AudioCapture instance.
public void Dispose()
{
if (m_AudioEncoder != null)
{
m_AudioEncoder.Release();
m_AudioEncoder = null;
}
if (m_AudioFilterStream != null)
{
m_AudioFilterStream.Dispose();
m_AudioFilterStream = null;
}
}
/// Starts recording asynchronous.
/// Filename of the file.
/// The on started recording video callback.
public void StartRecordingAsync(string filename, OnStartedRecordingAudioCallback onStartedRecordingAudioCallback)
{
float volumeFactorMic = 1.0f;
float volumeFactorApp = 1.0f;
if (NRDevice.Subsystem.GetDeviceType() == NRDeviceType.NrealLight)
{
volumeFactorMic = NativeConstants.RECORD_VOLUME_MIC;
volumeFactorApp = NativeConstants.RECORD_VOLUME_APP;
}
StartRecordingAsync(filename, onStartedRecordingAudioCallback, volumeFactorMic, volumeFactorApp);
}
/// Starts recording asynchronous.
/// Filename of the file.
/// The on started recording audio callback.
public void StartRecordingAsync(string filename, OnStartedRecordingAudioCallback onStartedRecordingAudioCallback, float volumeFactorMic, float volumeFactorApp)
{
NRDebugger.Info("[AudioCapture] StartRecordingAsync: IsRecording={0}, volFactorMic={1}, volFactorApp={2}", IsRecording, volumeFactorMic, volumeFactorApp);
var result = new AudioCaptureResult();
if (IsRecording)
{
result.resultType = CaptureResultType.UnknownError;
onStartedRecordingAudioCallback?.Invoke(result);
}
else
{
try
{
if (m_AudioFilterStream == null)
{
m_AudioFilterStream = new AudioFilterStream();
}
m_AudioEncoder.AdjustVolume(RecorderIndex.REC_MIC, volumeFactorMic);
m_AudioEncoder.AdjustVolume(RecorderIndex.REC_APP, volumeFactorApp);
m_AudioEncoder.EncodeConfig.SetOutPutPath(filename);
m_AudioEncoder.Start(OnAudioDataCallback);
IsRecording = true;
result.resultType = CaptureResultType.Success;
onStartedRecordingAudioCallback?.Invoke(result);
}
catch (Exception ex)
{
NRDebugger.Warning("[AudioCapture] StartRecordingAsync: {0}\n{1}", ex.Message, ex.StackTrace);
result.resultType = CaptureResultType.UnknownError;
onStartedRecordingAudioCallback?.Invoke(result);
throw;
}
}
}
private void OnAudioDataCallback(IntPtr data, UInt32 size)
{
// NRDebugger.Warning("[AudioCapture] OnAudioDataCallback: size={0}, data={1}", size, data);
if (m_AudioFilterStream != null)
{
m_AudioFilterStream.OnAudioDataRead(data, size);
}
OnAudioData?.Invoke(data, size);
}
public bool FlushAudioData(ref byte[] outBytesData, ref int samples)
{
if (m_AudioFilterStream != null)
{
if (m_AudioFilterStream.Flush(ref outBytesData))
{
samples = outBytesData.Length / Channels / BytesPerSample;
return true;
}
}
return false;
}
/// Starts audio mode asynchronous.
/// Options for controlling the setup.
/// The on audio mode started callback.
/// Auto adaption for BlendMode based on supported feature on current device.
public void StartAudioModeAsync(CameraParameters setupParams, OnAudioModeStartedCallback onAudioModeStartedCallback)
{
if (Application.isEditor || Application.platform != RuntimePlatform.Android)
{
StartAudioMode(setupParams, onAudioModeStartedCallback);
return;
}
bool recordMic = setupParams.CaptureAudioMic;
bool recordApp = setupParams.CaptureAudioApplication;
if (recordApp)
{
NRAndroidPermissionsManager.GetInstance().RequestAndroidPermission("android.permission.RECORD_AUDIO").ThenAction((requestResult) =>
{
if (requestResult.IsAllGranted)
{
NRAndroidPermissionsManager.GetInstance().RequestScreenCapture().ThenAction((AndroidJavaObject mediaProjection) =>
{
if (mediaProjection != null)
{
setupParams.mediaProjection = mediaProjection;
StartAudioMode(setupParams, onAudioModeStartedCallback);
}
else
{
NRDebugger.Error("[AudioCapture] Screen capture is denied by user.");
var result = new AudioCaptureResult();
result.resultType = CaptureResultType.UnknownError;
onAudioModeStartedCallback?.Invoke(result);
NRSessionManager.Instance.OprateInitException(new NRPermissionDenyError(NativeConstants.ScreenCaptureDenyErrorTip));
}
});
}
else {
NRDebugger.Error("[AudioCapture] Record audio need the permission of 'android.permission.RECORD_AUDIO'.");
var result = new AudioCaptureResult();
result.resultType = CaptureResultType.UnknownError;
onAudioModeStartedCallback?.Invoke(result);
NRSessionManager.Instance.OprateInitException(new NRPermissionDenyError(NativeConstants.AudioPermissionDenyErrorTip));
}
});
}
else if (recordMic)
{
NRAndroidPermissionsManager.GetInstance().RequestAndroidPermission("android.permission.RECORD_AUDIO").ThenAction((requestResult) =>
{
if (requestResult.IsAllGranted)
{
StartAudioMode(setupParams, onAudioModeStartedCallback);
}
else {
NRDebugger.Error("[AudioCapture] Record audio need the permission of 'android.permission.RECORD_AUDIO'.");
var result = new AudioCaptureResult();
result.resultType = CaptureResultType.UnknownError;
onAudioModeStartedCallback?.Invoke(result);
NRSessionManager.Instance.OprateInitException(new NRPermissionDenyError(NativeConstants.AudioPermissionDenyErrorTip));
}
});
}
else
{
StartAudioMode(setupParams, onAudioModeStartedCallback);
}
}
private void StartAudioMode(CameraParameters setupParams, OnAudioModeStartedCallback onAudioModeStartedCallback)
{
setupParams.camMode = CamMode.None;
if (setupParams.frameRate <= 0)
NRDebugger.Warning("[AudioCapture] frameRate need to be bigger than zero");
m_AudioEncoder = new AudioEncoder();
m_AudioEncoder.Config(setupParams);
var result = new AudioCaptureResult();
result.resultType = CaptureResultType.Success;
onAudioModeStartedCallback?.Invoke(result);
}
/// Stops recording asynchronous.
/// The on stopped recording audio callback.
public void StopRecordingAsync(OnStoppedRecordingAudioCallback onStoppedRecordingAudioCallback)
{
var result = new AudioCaptureResult();
if (!IsRecording)
{
result.resultType = CaptureResultType.UnknownError;
onStoppedRecordingAudioCallback?.Invoke(result);
}
else
{
try
{
m_AudioEncoder.Stop();
IsRecording = false;
result.resultType = CaptureResultType.Success;
onStoppedRecordingAudioCallback?.Invoke(result);
}
catch (Exception)
{
result.resultType = CaptureResultType.UnknownError;
onStoppedRecordingAudioCallback?.Invoke(result);
throw;
}
}
}
/// Stops video mode asynchronous.
/// The on video mode stopped callback.
public void StopAudioModeAsync(OnAudioModeStoppedCallback onAudioModeStoppedCallback)
{
m_AudioEncoder?.Release();
m_AudioEncoder = null;
var result = new AudioCaptureResult();
result.resultType = CaptureResultType.Success;
onAudioModeStoppedCallback?.Invoke(result);
}
/// Contains the result of the capture request.
public enum CaptureResultType
{
///
/// Specifies that the desired operation was successful.
///
Success = 0,
///
/// Specifies that an unknown error occurred.
///
UnknownError = 1
}
///
/// A data container that contains the result information of a audio recording operation.
public struct AudioCaptureResult
{
///
/// A generic result that indicates whether or not the AudioCapture operation succeeded.
public CaptureResultType resultType;
/// The specific HResult value.
public long hResult;
/// Indicates whether or not the operation was successful.
/// True if success, false if not.
public bool success
{
get
{
return resultType == CaptureResultType.Success;
}
}
}
/// Called when the web camera begins recording the audio.
/// Indicates whether or not audio recording started successfully.
public delegate void OnStartedRecordingAudioCallback(AudioCaptureResult result);
/// Called when audio mode has been started.
/// Indicates whether or not audio mode was successfully activated.
public delegate void OnAudioModeStartedCallback(AudioCaptureResult result);
/// Called when audio mode has been stopped.
/// Indicates whether or not audio mode was successfully deactivated.
public delegate void OnAudioModeStoppedCallback(AudioCaptureResult result);
/// Called when the audio recording has been saved to the file system.
/// Indicates whether or not audio recording was saved successfully to the
/// file system.
public delegate void OnStoppedRecordingAudioCallback(AudioCaptureResult result);
}
}