Download modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.cpp from tgollogly/test: direct link, hf CLI and curl.
- Browser
- Download file 23.4 kB
-
https://huggingface.co/tgollogly/test/resolve/main/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.cpp
- Command line
-
hf download hf://tgollogly/test/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.cpp
-
curl -L -o preproc_engine.cpp https://huggingface.co/tgollogly/test/resolve/main/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.cpp
23.4 kB
| // This file is part of OpenCV project. | |
| // It is subject to the license terms in the LICENSE file found in the top-level directory | |
| // of this distribution and at http://opencv.org/license.html. | |
| // | |
| // Copyright (C) 2022 Intel Corporation | |
| namespace cv { | |
| namespace gapi { | |
| namespace wip { | |
| namespace onevpl { | |
| bool FrameInfoComparator::operator()(const mfxFrameInfo& lhs, const mfxFrameInfo& rhs) const { | |
| return lhs < rhs; | |
| } | |
| bool FrameInfoComparator::equal_to(const mfxFrameInfo& lhs, const mfxFrameInfo& rhs) { | |
| return lhs == rhs; | |
| } | |
| static void apply_roi(mfxFrameSurface1* surface_handle, | |
| const cv::util::optional<cv::Rect> &opt_roi) { | |
| if (opt_roi.has_value()) { | |
| const cv::Rect &roi = opt_roi.value(); | |
| surface_handle->Info.CropX = static_cast<mfxU16>(roi.x); | |
| surface_handle->Info.CropY = static_cast<mfxU16>(roi.y); | |
| surface_handle->Info.CropW = static_cast<mfxU16>(roi.width); | |
| surface_handle->Info.CropH = static_cast<mfxU16>(roi.height); | |
| GAPI_LOG_DEBUG(nullptr, "applied ROI {" << surface_handle->Info.CropX << | |
| ", " << surface_handle->Info.CropY << "}, " | |
| "{ " << surface_handle->Info.CropX + surface_handle->Info.CropW << | |
| ", " << surface_handle->Info.CropY + surface_handle->Info.CropH << "}"); | |
| } | |
| } | |
| VPPPreprocEngine::VPPPreprocEngine(std::unique_ptr<VPLAccelerationPolicy>&& accel) : | |
| ProcessingEngineBase(std::move(accel)) { | |
| GAPI_LOG_DEBUG(nullptr, "Create VPP preprocessing engine"); | |
| preprocessed_frames_count = 0; | |
| create_pipeline( | |
| // 0) preproc decoded surface with VPP params | |
| [this] (EngineSession& sess) -> ExecutionStatus | |
| { | |
| session_type &my_sess = static_cast<session_type&>(sess); | |
| while (!my_sess.sync_in_queue.empty()) { | |
| do { | |
| if (!my_sess.processing_surface_ptr.expired()) { | |
| session_type::incoming_task pending_op = my_sess.sync_in_queue.front(); | |
| GAPI_LOG_DEBUG(nullptr, "pending IN operations count: " << | |
| my_sess.sync_in_queue.size() << | |
| ", sync id: " << | |
| pending_op.sync_handle << | |
| ", surface: " << | |
| pending_op.decoded_surface_ptr); | |
| my_sess.sync_in_queue.pop(); | |
| auto *vpp_suface = my_sess.processing_surface_ptr.lock()->get_handle(); | |
| apply_roi(pending_op.decoded_surface_ptr, pending_op.roi); | |
| mfxSyncPoint vpp_sync_handle{}; | |
| my_sess.last_status = MFXVideoVPP_RunFrameVPPAsync(my_sess.session, | |
| pending_op.decoded_surface_ptr, | |
| vpp_suface, | |
| nullptr, &vpp_sync_handle); | |
| session_type::outgoing_task vpp_pending_op {vpp_sync_handle, | |
| vpp_suface, | |
| std::move(pending_op) }; | |
| GAPI_LOG_DEBUG(nullptr, "Got VPP async operation" << | |
| ", sync id: " << | |
| vpp_pending_op.sync_handle << | |
| ", dec surface: " << | |
| vpp_pending_op.original_surface_ptr << | |
| ", trans surface: " << | |
| vpp_pending_op.vpp_surface_ptr << | |
| ", status: " << | |
| mfxstatus_to_string(my_sess.last_status)); | |
| // NB: process status | |
| if (my_sess.last_status == MFX_ERR_MORE_SURFACE || | |
| my_sess.last_status == MFX_ERR_NONE) { | |
| vpp_pending_op.vpp_surface_ptr->Data.Locked++; // TODO -S- workaround | |
| my_sess.vpp_out_queue.emplace(vpp_pending_op); | |
| } | |
| } | |
| try { | |
| my_sess.swap_surface(*this); | |
| } catch (const std::runtime_error& ex) { | |
| // NB: not an error, yield CPU ticks to check | |
| // surface availability at a next phase. | |
| // But print WARNING to notify user about pipeline stuck | |
| GAPI_LOG_WARNING(nullptr, "[" << my_sess.session << | |
| "] has no VPP surface, reason: " << | |
| ex.what()); | |
| my_sess.processing_surface_ptr.reset(); | |
| break; | |
| } | |
| } while(my_sess.last_status == MFX_ERR_MORE_SURFACE); | |
| if (my_sess.processing_surface_ptr.expired()) { | |
| // TODO break main loop | |
| break; | |
| } | |
| } | |
| return ExecutionStatus::Continue; | |
| }, | |
| // 1) Wait for ASYNC decode result | |
| [this] (EngineSession& sess) -> ExecutionStatus | |
| { | |
| session_type& my_sess = static_cast<session_type&>(sess); | |
| do { | |
| if (!my_sess.vpp_out_queue.empty()) { // FIFO: check the oldest async operation complete | |
| session_type::outgoing_task& pending_op = my_sess.vpp_out_queue.front(); | |
| sess.last_status = MFXVideoCORE_SyncOperation(sess.session, pending_op.sync_handle, 0); | |
| GAPI_LOG_DEBUG(nullptr, "pending VPP operations count: " << | |
| my_sess.vpp_out_queue.size() << | |
| ", sync id: " << | |
| pending_op.sync_handle << | |
| ", surface: " << | |
| pending_op.vpp_surface_ptr << | |
| ", status: " << | |
| mfxstatus_to_string(my_sess.last_status)); | |
| // put frames in ready queue on success | |
| if (MFX_ERR_NONE == sess.last_status) { | |
| pending_op.release_frame(); | |
| on_frame_ready(my_sess, pending_op.vpp_surface_ptr); | |
| } | |
| } | |
| } while (MFX_ERR_NONE == sess.last_status && !my_sess.vpp_out_queue.empty()); | |
| return ExecutionStatus::Continue; | |
| }, | |
| // 2) Falls back on generic status processing | |
| [this] (EngineSession& sess) -> ExecutionStatus | |
| { | |
| return this->process_error(sess.last_status, static_cast<session_type&>(sess)); | |
| } | |
| ); | |
| } | |
| cv::util::optional<pp_params> VPPPreprocEngine::is_applicable(const cv::MediaFrame& in_frame) { | |
| // TODO consider something smarter than RTI | |
| cv::util::optional<pp_params> ret; | |
| BaseFrameAdapter *vpl_adapter = in_frame.get<BaseFrameAdapter>(); | |
| GAPI_LOG_DEBUG(nullptr, "validate VPP preprocessing is applicable for frame"); | |
| if (vpl_adapter) { | |
| ret = cv::util::make_optional<pp_params>( | |
| pp_params::create<vpp_pp_params>(vpl_adapter->get_session_handle(), | |
| vpl_adapter->get_surface()->get_info(), | |
| vpl_adapter)); | |
| GAPI_LOG_DEBUG(nullptr, "VPP preprocessing applicable, session [" << | |
| vpl_adapter->get_session_handle() << "]"); | |
| } | |
| return ret; | |
| } | |
| pp_session VPPPreprocEngine::initialize_preproc(const pp_params& initial_frame_param, | |
| const GFrameDesc& required_frame_descr) { | |
| const vpp_pp_params ¶ms = initial_frame_param.get<vpp_pp_params>(); | |
| // adjust preprocessing settings | |
| mfxVideoParam mfxVPPParams{}; | |
| memset(&mfxVPPParams, 0, sizeof(mfxVideoParam)); | |
| // NB: IN params for VPP session must be equal to decoded surface params | |
| mfxVPPParams.vpp.In = params.info; | |
| // NB: OUT params must refer to IN params of a network | |
| GAPI_LOG_DEBUG(nullptr, "network input size: " << required_frame_descr.size.width << | |
| "x" << required_frame_descr.size.height); | |
| mfxVPPParams.vpp.Out = mfxVPPParams.vpp.In; | |
| switch (required_frame_descr.fmt) { | |
| case MediaFormat::NV12: | |
| mfxVPPParams.vpp.Out.FourCC = MFX_FOURCC_NV12; | |
| break; | |
| default: | |
| GAPI_LOG_WARNING(nullptr, "Unsupported MediaFormat in preprocessing: " << | |
| static_cast<int>(required_frame_descr.fmt) << | |
| ". Frame will be rejected"); | |
| throw std::runtime_error("unsupported MediaFormat value in VPP preprocessing"); | |
| } | |
| mfxVPPParams.vpp.Out.ChromaFormat = MFX_CHROMAFORMAT_YUV420; | |
| mfxVPPParams.vpp.Out.Width = static_cast<mfxU16>(required_frame_descr.size.width); | |
| mfxVPPParams.vpp.Out.Height = static_cast<mfxU16>(required_frame_descr.size.height); | |
| mfxVPPParams.vpp.Out.CropW = mfxVPPParams.vpp.Out.Width; | |
| mfxVPPParams.vpp.Out.CropH = mfxVPPParams.vpp.Out.Height; | |
| // check In & Out equally to bypass preproc | |
| if (mfxVPPParams.vpp.Out == mfxVPPParams.vpp.In) { | |
| GAPI_LOG_DEBUG(nullptr, "no preproc required"); | |
| return pp_session::create<vpp_pp_session>(nullptr); | |
| } | |
| // recalculate size param according to VPP alignment | |
| mfxVPPParams.vpp.Out.Width = ALIGN16(mfxVPPParams.vpp.Out.Width); | |
| mfxVPPParams.vpp.Out.Height = ALIGN16(mfxVPPParams.vpp.Out.Height); | |
| mfxVPPParams.vpp.Out.CropW = mfxVPPParams.vpp.Out.Width; | |
| mfxVPPParams.vpp.Out.CropH = mfxVPPParams.vpp.Out.Height; | |
| GAPI_LOG_DEBUG(nullptr, "\nFrom:\n{\n" << mfx_frame_info_to_string(mfxVPPParams.vpp.In) << | |
| "}\nTo:\n{\n" << mfx_frame_info_to_string(mfxVPPParams.vpp.Out) << "}"); | |
| // find existing session | |
| GAPI_LOG_DEBUG(nullptr, "Find existing VPPPreprocSession for requested frame params" | |
| ", total sessions: " << preproc_session_map.size()); | |
| auto it = preproc_session_map.find(mfxVPPParams.vpp.In); | |
| if (it != preproc_session_map.end()) { | |
| GAPI_LOG_DEBUG(nullptr, "[" << it->second->session << "] found"); | |
| return pp_session::create<vpp_pp_session>(std::static_pointer_cast<EngineSession>(it->second)); | |
| } | |
| // NB: make some sanity checks | |
| IDeviceSelector::DeviceScoreTable devices = acceleration_policy->get_device_selector()->select_devices(); | |
| GAPI_Assert(devices.size() == 1 && "Multiple(or zero) acceleration devices case is unsupported"); | |
| AccelType accel_type = devices.begin()->second.get_type(); | |
| // assign acceleration | |
| if (accel_type == AccelType::DX11) { | |
| mfxVPPParams.IOPattern = MFX_IOPATTERN_IN_VIDEO_MEMORY | MFX_IOPATTERN_OUT_VIDEO_MEMORY; | |
| } else { | |
| mfxVPPParams.IOPattern = MFX_IOPATTERN_IN_SYSTEM_MEMORY | MFX_IOPATTERN_OUT_SYSTEM_MEMORY; | |
| } | |
| // clone existing VPL session to inherit VPL loader configuration | |
| // and avoid refer to any global state | |
| // TODO no clone due to clone issue | |
| mfxSession mfx_vpp_session = params.handle; | |
| mfxStatus sts = MFX_ERR_NONE; | |
| // TODO: simply use clone after VPL bug fixing | |
| //sts = MFXCloneSession(params.handle, &mfx_vpp_session); | |
| sts = MFXCreateSession(mfx_handle, impl_number, &mfx_vpp_session); | |
| if (sts != MFX_ERR_NONE) { | |
| GAPI_LOG_WARNING(nullptr, "Cannot clone VPP session, error: " << mfxstatus_to_string(sts)); | |
| GAPI_Error("Cannot continue VPP preprocessing"); | |
| } | |
| sts = MFXJoinSession(params.handle, mfx_vpp_session); | |
| if (sts != MFX_ERR_NONE) { | |
| GAPI_LOG_WARNING(nullptr, "Cannot join VPP sessions, error: " << mfxstatus_to_string(sts)); | |
| GAPI_Error("Cannot continue VPP preprocessing"); | |
| } | |
| GAPI_LOG_INFO(nullptr, "[" << mfx_vpp_session << "] starting pool allocation"); | |
| VPLAccelerationPolicy::pool_key_t vpp_out_pool_key {}; | |
| try { | |
| // assign HW acceleration processor | |
| acceleration_policy->init(mfx_vpp_session); | |
| try { | |
| // ask to allocate external memory pool | |
| mfxFrameAllocRequest vppRequests[2]; | |
| memset(&vppRequests, 0, sizeof(mfxFrameAllocRequest) * 2); | |
| sts = MFXVideoVPP_QueryIOSurf(mfx_vpp_session, &mfxVPPParams, vppRequests); | |
| if (MFX_ERR_NONE != sts) { | |
| GAPI_LOG_WARNING(nullptr, "cannot execute MFXVideoVPP_QueryIOSurf, error: " << | |
| mfxstatus_to_string(sts)); | |
| throw std::runtime_error("Cannot execute MFXVideoVPP_QueryIOSurf"); | |
| } | |
| // NB: Assign ID as upper limit descendant to distinguish specific VPP allocation | |
| // from decode allocations witch started from 0: by local module convention | |
| static uint16_t request_id = 0; | |
| vppRequests[1].AllocId = std::numeric_limits<uint16_t>::max() - request_id++; | |
| GAPI_Assert(request_id != std::numeric_limits<uint16_t>::max() && "Something wrong"); | |
| vppRequests[1].Type |= MFX_MEMTYPE_FROM_VPPIN | MFX_MEMTYPE_SHARED_RESOURCE; | |
| vpp_out_pool_key = acceleration_policy->create_surface_pool(vppRequests[1], | |
| mfxVPPParams.vpp.Out); | |
| sts = MFXVideoVPP_Init(mfx_vpp_session, &mfxVPPParams); | |
| if (MFX_ERR_NONE != sts) { | |
| GAPI_LOG_WARNING(nullptr, "cannot Init VPP, error: " << | |
| mfxstatus_to_string(sts)); | |
| // TODO consider deallocate pool | |
| // but not necessary now cause every fail processed as GAPI_Assert | |
| throw std::runtime_error("Cannot init VPP, error: " + | |
| mfxstatus_to_string(sts)); | |
| } | |
| } catch (const std::exception&) { | |
| GAPI_LOG_WARNING(nullptr, "[" << mfx_vpp_session << "] allocation failed, rollback"); | |
| acceleration_policy->deinit(mfx_vpp_session); | |
| throw; | |
| } | |
| } catch (const std::exception&) { | |
| MFXClose(mfx_vpp_session); | |
| GAPI_Error("Cannot init preproc resources"); | |
| } | |
| // create engine session after all | |
| session_ptr_type sess_ptr = register_session<session_type>(mfx_vpp_session, | |
| mfxVPPParams); | |
| sess_ptr->init_surface_pool(vpp_out_pool_key); | |
| sess_ptr->swap_surface(*this); | |
| bool inserted = preproc_session_map.emplace(mfxVPPParams.vpp.In, sess_ptr).second; | |
| GAPI_Assert(inserted && "preproc session is exist"); | |
| GAPI_LOG_INFO(nullptr, "VPPPreprocSession created, total sessions: " << preproc_session_map.size()); | |
| return pp_session::create<vpp_pp_session>(std::static_pointer_cast<EngineSession>(sess_ptr)); | |
| } | |
| void VPPPreprocEngine::on_frame_ready(session_type& sess, | |
| mfxFrameSurface1* ready_surface) | |
| { | |
| GAPI_LOG_DEBUG(nullptr, "[" << sess.session << "], frame ready"); | |
| // manage memory ownership rely on acceleration policy | |
| ready_surface->Data.Locked--; // TODO -S- workaround | |
| VPLAccelerationPolicy::FrameConstructorArgs args{ready_surface, sess.session}; | |
| auto frame_adapter = acceleration_policy->create_frame_adapter(sess.vpp_pool_id, | |
| args); | |
| ready_frames.emplace(cv::MediaFrame(std::move(frame_adapter)), sess.generate_frame_meta()); | |
| // pop away synced out object | |
| sess.vpp_out_queue.pop(); | |
| } | |
| VPPPreprocEngine::session_ptr | |
| VPPPreprocEngine::initialize_session(mfxSession, | |
| const std::vector<CfgParam>&, | |
| std::shared_ptr<IDataProvider>) { | |
| return {}; | |
| } | |
| cv::MediaFrame VPPPreprocEngine::run_sync(const pp_session& sess, const cv::MediaFrame& in_frame, | |
| const cv::util::optional<cv::Rect> &roi) { | |
| vpp_pp_session pp_sess_impl = sess.get<vpp_pp_session>(); | |
| if (!pp_sess_impl.handle) { | |
| // bypass case | |
| return in_frame; | |
| } | |
| session_ptr_type s = std::static_pointer_cast<session_type>(pp_sess_impl.handle); | |
| GAPI_DbgAssert(s && "Session is nullptr"); | |
| GAPI_DbgAssert(is_applicable(in_frame) && | |
| "VPP preproc is not applicable for the given frame"); | |
| BaseFrameAdapter *vpl_adapter = in_frame.get<BaseFrameAdapter>(); | |
| if (!vpl_adapter) { | |
| GAPI_LOG_WARNING(nullptr, "VPP preproc is inapplicable for a given frame. " | |
| "Make sure the frame is collected using onevpl::GSource"); | |
| throw std::runtime_error("VPP preproc is inapplicable for given frame"); | |
| } | |
| // schedule decoded surface into preproc queue | |
| session_type::incoming_task in_preproc_request {nullptr, | |
| vpl_adapter->get_surface()->get_handle(), | |
| vpl_adapter->get_surface()->get_info(), | |
| in_frame, | |
| roi}; | |
| s->sync_in_queue.emplace(in_preproc_request); | |
| // invoke pipeline to transform decoded surface into preprocessed surface | |
| try | |
| { | |
| ExecutionStatus status = ExecutionStatus::Continue; | |
| while (0 == get_ready_frames_count() && | |
| status == ExecutionStatus::Continue) { | |
| status = process(s->session); | |
| } | |
| if (get_ready_frames_count() == 0) { | |
| GAPI_LOG_WARNING(nullptr, "failed: cannot obtain preprocessed frames, last status: " << | |
| ProcessingEngineBase::status_to_string(status)); | |
| throw std::runtime_error("cannot finalize VPP preprocessing operation"); | |
| } | |
| } catch(const std::exception&) { | |
| throw; | |
| } | |
| // obtain new frame is available | |
| cv::gapi::wip::Data data; | |
| get_frame(data); | |
| preprocessed_frames_count++; | |
| GAPI_LOG_DEBUG(nullptr, "processed frames count: " << preprocessed_frames_count); | |
| return cv::util::get<cv::MediaFrame>(data); | |
| } | |
| ProcessingEngineBase::ExecutionStatus VPPPreprocEngine::process_error(mfxStatus status, session_type& sess) { | |
| GAPI_LOG_DEBUG(nullptr, "status: " << mfxstatus_to_string(status)); | |
| switch (status) { | |
| case MFX_ERR_NONE: | |
| { | |
| // prepare sync object for new surface | |
| try { | |
| sess.swap_surface(*this); | |
| return ExecutionStatus::Continue; | |
| } catch (const std::runtime_error& ex) { | |
| GAPI_LOG_WARNING(nullptr, "[" << sess.session << "] error: " << ex.what()); | |
| return ExecutionStatus::Continue; // read more data | |
| } | |
| } | |
| case MFX_ERR_MORE_DATA: // The function requires more bitstream at input before decoding can proceed | |
| return ExecutionStatus::Processed; | |
| case MFX_ERR_MORE_SURFACE: | |
| { | |
| // The function requires more frame surface at output before decoding can proceed. | |
| // This applies to external memory allocations and should not be expected for | |
| // a simple internal allocation case like this | |
| try { | |
| sess.swap_surface(*this); | |
| return ExecutionStatus::Continue; | |
| } catch (const std::runtime_error& ex) { | |
| GAPI_LOG_WARNING(nullptr, "[" << sess.session << "] error: " << ex.what()); | |
| return ExecutionStatus::Continue; // read more data | |
| } | |
| break; | |
| } | |
| case MFX_ERR_DEVICE_LOST: | |
| // For non-CPU implementations, | |
| // Cleanup if device is lost | |
| GAPI_DbgAssert(false && "VPPPreprocEngine::process_error - " | |
| "MFX_ERR_DEVICE_LOST is not processed"); | |
| break; | |
| case MFX_WRN_DEVICE_BUSY: | |
| // For non-CPU implementations, | |
| // Wait a few milliseconds then try again | |
| GAPI_DbgAssert(false && "VPPPreprocEngine::process_error - " | |
| "MFX_WRN_DEVICE_BUSY is not processed"); | |
| break; | |
| case MFX_WRN_VIDEO_PARAM_CHANGED: | |
| // The decoder detected a new sequence header in the bitstream. | |
| // Video parameters may have changed. | |
| // In external memory allocation case, might need to reallocate the output surface | |
| GAPI_LOG_WARNING(nullptr, "[" << sess.session << "] got MFX_WRN_VIDEO_PARAM_CHANGED"); | |
| return ExecutionStatus::Continue; | |
| break; | |
| case MFX_ERR_INCOMPATIBLE_VIDEO_PARAM: | |
| // The function detected that video parameters provided by the application | |
| // are incompatible with initialization parameters. | |
| // The application should close the component and then reinitialize it | |
| GAPI_DbgAssert(false && "VPPPreprocEngine::process_error - " | |
| "MFX_ERR_INCOMPATIBLE_VIDEO_PARAM is not processed"); | |
| break; | |
| case MFX_ERR_REALLOC_SURFACE: | |
| // Bigger surface_work required. May be returned only if | |
| // mfxInfoMFX::EnableReallocRequest was set to ON during initialization. | |
| // This applies to external memory allocations and should not be expected for | |
| // a simple internal allocation case like this | |
| GAPI_DbgAssert(false && "VPPPreprocEngine::process_error - " | |
| "MFX_ERR_REALLOC_SURFACE is not processed"); | |
| break; | |
| case MFX_WRN_IN_EXECUTION: | |
| GAPI_LOG_DEBUG(nullptr, "[" << sess.session << "] got MFX_WRN_IN_EXECUTION"); | |
| return ExecutionStatus::Continue; | |
| default: | |
| GAPI_LOG_WARNING(nullptr, "Unknown status code: " << mfxstatus_to_string(status) << | |
| ", decoded frames: " << sess.preprocessed_frames_count); | |
| break; | |
| } | |
| return ExecutionStatus::Failed; | |
| } | |
| } // namespace onevpl | |
| } // namespace wip | |
| } // namespace gapi | |
| } // namespace cv | |