#include "pch.h" #include #include #include #include #include "NvCodecUtils.h" #include "NvDecoder/NvDecoder.h" #include "NvDecoderImpl.h" #include "ProfilerMarkerFactory.h" #include "ScopedProfiler.h" namespace unity { namespace webrtc { using namespace ::webrtc; ColorSpace ExtractH264ColorSpace(const CUVIDEOFORMAT& format) { return ColorSpace( static_cast(format.video_signal_description.color_primaries), static_cast(format.video_signal_description.transfer_characteristics), static_cast(format.video_signal_description.matrix_coefficients), static_cast(format.video_signal_description.video_full_range_flag)); } NvDecoderImpl::NvDecoderImpl(CUcontext context, ProfilerMarkerFactory* profiler) : m_context(context) , m_decoder(nullptr) , m_isConfiguredDecoder(false) , m_decodedCompleteCallback(nullptr) , m_buffer_pool(false) , m_profiler(profiler) { if (profiler) m_marker = profiler->CreateMarker( "NvDecoderImpl.ConvertNV12ToI420", kUnityProfilerCategoryOther, kUnityProfilerMarkerFlagDefault, 0); } NvDecoderImpl::~NvDecoderImpl() { Release(); } VideoDecoder::DecoderInfo NvDecoderImpl::GetDecoderInfo() const { VideoDecoder::DecoderInfo info; info.implementation_name = "NvCodec"; info.is_hardware_accelerated = true; return info; } int NvDecoderImpl::InitDecode(const VideoCodec* codec_settings, int32_t number_of_cores) { if (codec_settings == nullptr) { RTC_LOG(LS_ERROR) << "initialization failed on codec_settings is null "; return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } if (codec_settings->codecType != kVideoCodecH264) { RTC_LOG(LS_ERROR) << "initialization failed on codectype is not kVideoCodecH264"; return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } if (codec_settings->width < 1 || codec_settings->height < 1) { RTC_LOG(LS_ERROR) << "initialization failed on codec_settings width < 0 or height < 0"; return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } m_codec = *codec_settings; const CUresult result = cuCtxSetCurrent(m_context); if (!ck(result)) { RTC_LOG(LS_ERROR) << "initialization failed on cuCtxSetCurrent result" << result; return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE; } // todo(kazuki): Max resolution is differred each architecture. // Refer to the table in Video Decoder Capabilities. // https://docs.nvidia.com/video-technologies/video-codec-sdk/nvdec-video-decoder-api-prog-guide int maxWidth = 4096; int maxHeight = 4096; // bUseDeviceFrame: allocate in memory or cuda device memory m_decoder = std::make_unique( m_context, false, cudaVideoCodec_H264, true, false, nullptr, nullptr, maxWidth, maxHeight); return WEBRTC_VIDEO_CODEC_OK; } int32_t NvDecoderImpl::RegisterDecodeCompleteCallback(DecodedImageCallback* callback) { this->m_decodedCompleteCallback = callback; return WEBRTC_VIDEO_CODEC_OK; } int32_t NvDecoderImpl::Release() { m_buffer_pool.Release(); return WEBRTC_VIDEO_CODEC_OK; } int32_t NvDecoderImpl::Decode(const EncodedImage& input_image, bool missing_frames, int64_t render_time_ms) { CUcontext current; if (!ck(cuCtxGetCurrent(¤t))) { RTC_LOG(LS_ERROR) << "decode failed on cuCtxGetCurrent is failed"; return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } if (current != m_context) { RTC_LOG(LS_ERROR) << "decode failed on not match current context and hold context"; return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } if (m_decodedCompleteCallback == nullptr) { RTC_LOG(LS_ERROR) << "decode failed on not set m_decodedCompleteCallback"; return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } if (!input_image.data() || !input_image.size()) { RTC_LOG(LS_ERROR) << "decode failed on input image is null"; return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } m_h264_bitstream_parser.ParseBitstream(input_image); absl::optional qp = m_h264_bitstream_parser.GetLastSliceQp(); absl::optional sps = m_h264_bitstream_parser.sps(); if (m_isConfiguredDecoder) { if (!sps || sps.value().width != static_cast(m_decoder->GetWidth()) || sps.value().height != static_cast(m_decoder->GetHeight())) { m_decoder->setReconfigParams(nullptr, nullptr); } } int nFrameReturnd = 0; do { nFrameReturnd = m_decoder->Decode( input_image.data(), static_cast(input_image.size()), CUVID_PKT_TIMESTAMP, input_image.Timestamp()); } while (nFrameReturnd == 0); m_isConfiguredDecoder = true; // todo: support other output format // Chromium's H264 Encoder is output on NV12, so currently only NV12 is supported. if (m_decoder->GetOutputFormat() != cudaVideoSurfaceFormat_NV12) { RTC_LOG(LS_ERROR) << "not supported this format: " << m_decoder->GetOutputFormat(); return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } // Pass on color space from input frame if explicitly specified. const ColorSpace& color_space = input_image.ColorSpace() ? *input_image.ColorSpace() : ExtractH264ColorSpace(m_decoder->GetVideoFormatInfo()); for (int i = 0; i < nFrameReturnd; i++) { int64_t timeStamp; uint8_t* pFrame = m_decoder->GetFrame(&timeStamp); rtc::scoped_refptr i420_buffer = m_buffer_pool.CreateI420Buffer(m_decoder->GetWidth(), m_decoder->GetHeight()); int result; { std::unique_ptr profiler; if (m_profiler) profiler = m_profiler->CreateScopedProfiler(*m_marker); result = libyuv::NV12ToI420( pFrame, m_decoder->GetDeviceFramePitch(), pFrame + m_decoder->GetHeight() * m_decoder->GetDeviceFramePitch(), m_decoder->GetDeviceFramePitch(), i420_buffer->MutableDataY(), i420_buffer->StrideY(), i420_buffer->MutableDataU(), i420_buffer->StrideU(), i420_buffer->MutableDataV(), i420_buffer->StrideV(), m_decoder->GetWidth(), m_decoder->GetHeight()); } if (result) { RTC_LOG(LS_INFO) << "libyuv::NV12ToI420 failed. error:" << result; } VideoFrame decoded_frame = VideoFrame::Builder() .set_video_frame_buffer(i420_buffer) .set_timestamp_rtp(static_cast(timeStamp)) .set_color_space(color_space) .build(); // todo: measurement decoding time absl::optional decodetime; m_decodedCompleteCallback->Decoded(decoded_frame, decodetime, qp); } return WEBRTC_VIDEO_CODEC_OK; } } // end namespace webrtc } // end namespace unity