From d7bd4dedfe809e88254228d2625b88e9cba56eb7 Mon Sep 17 00:00:00 2001 From: "Huang, Li H" Date: Tue, 19 Nov 2024 19:03:03 +0000 Subject: [PATCH] [Decode]AV1 DPB optimization with flags decreasement and renaming --- .../decode/av1/src/mfx_av1_dec_decode.cpp | 1 + .../codec/av1_dec/include/umc_av1_decoder.h | 13 ++-- .../umc/codec/av1_dec/include/umc_av1_frame.h | 15 ++-- .../umc/codec/av1_dec/src/umc_av1_decoder.cpp | 76 ++++++++----------- .../umc/codec/av1_dec/src/umc_av1_frame.cpp | 14 +--- 5 files changed, 45 insertions(+), 74 deletions(-) diff --git a/_studio/mfx_lib/decode/av1/src/mfx_av1_dec_decode.cpp b/_studio/mfx_lib/decode/av1/src/mfx_av1_dec_decode.cpp index d3446bd04..be0e9fdb5 100755 --- a/_studio/mfx_lib/decode/av1/src/mfx_av1_dec_decode.cpp +++ b/_studio/mfx_lib/decode/av1/src/mfx_av1_dec_decode.cpp @@ -922,6 +922,7 @@ mfxStatus VideoDECODEAV1::QueryFrame(mfxThreadTask task) } mfxStatus sts = DecodeFrame(surface_out, frame); + //repeat frame decrease reference counter here if (info->copyfromframe != UMC::FRAME_MID_INVALID) { m_decoder->FlushRepeatFrame(frame); diff --git a/_studio/shared/umc/codec/av1_dec/include/umc_av1_decoder.h b/_studio/shared/umc/codec/av1_dec/include/umc_av1_decoder.h index 326fe5fbe..34d2c3f97 100755 --- a/_studio/shared/umc/codec/av1_dec/include/umc_av1_decoder.h +++ b/_studio/shared/umc/codec/av1_dec/include/umc_av1_decoder.h @@ -163,7 +163,7 @@ namespace UMC_AV1_DECODER AV1DecoderFrame* DecodeFrameID(UMC::FrameMemID); AV1DecoderFrame* FindFrameInProgress(); AV1DecoderFrame* GetCurrFrame() - { return Curr; } + { return lastest_submitted_frame; } UMC::FrameMemID GetRepeatedFrame(){return repeateFrame;} void SetInFrameRate(mfxF64 rate) { in_framerate = rate; } @@ -209,7 +209,7 @@ namespace UMC_AV1_DECODER AV1DecoderFrame* GetFrameBufferByIdx(FrameHeader const& fh, UMC::FrameMemID id); AV1DecoderFrame* StartAnchorFrame(FrameHeader const& fh, DPBType const& frameDPB, uint32_t idx); - DPBType DPBUpdate(AV1DecoderFrame * prevFrame); + DPBType ReferenceListUpdate(AV1DecoderFrame * prevFrame); protected: @@ -226,18 +226,15 @@ namespace UMC_AV1_DECODER uint32_t counter; AV1DecoderParams params; std::vector outputed_frames; // tore frames need to be output - AV1DecoderFrame* Curr; // store current frame for Poutput - AV1DecoderFrame* Curr_temp; // store current frame insist double updateDPB + AV1DecoderFrame* lastest_submitted_frame; // store current frame for Poutput uint32_t Repeat_show; // show if current frame is repeated frame uint32_t PreFrame_id;//id of previous frame uint32_t OldPreFrame_id;//old id of previous frame. When decode LST clip, need this for parsing twice - DPBType refs_temp; // previous updated frameDPB + DPBType last_updated_refs; // previous updated frameDPB mfxU16 frame_order; mfxF64 in_framerate; UMC::FrameMemID repeateFrame;//frame to be repeated - - FrameHeader last_frame_header; - + FrameHeader last_frame_header; // store last frame's header uint32_t anchor_frames_count; uint32_t tile_list_idx; uint32_t frames_to_skip; diff --git a/_studio/shared/umc/codec/av1_dec/include/umc_av1_frame.h b/_studio/shared/umc/codec/av1_dec/include/umc_av1_frame.h index 3ff74396e..dbcb0b6ac 100644 --- a/_studio/shared/umc/codec/av1_dec/include/umc_av1_frame.h +++ b/_studio/shared/umc/codec/av1_dec/include/umc_av1_frame.h @@ -158,17 +158,16 @@ namespace UMC_AV1_DECODER { return *header; } bool Empty() const; - bool Decoded() const; bool Displayed() const { return displayed; } void Displayed(bool d) { displayed = d; } - bool DpbUpdated() const - { return dpb_updated; } - void DpbUpdated(bool u) - { dpb_updated = u; } + bool RefUpdated() const + { return ref_updated; } + void RefUpdated(bool u) + { ref_updated = u; } bool Outputted() const { return outputted; } @@ -222,7 +221,6 @@ namespace UMC_AV1_DECODER void AddReferenceFrame(AV1DecoderFrame* frm); void FreeReferenceFrames(); void UpdateReferenceList(); - void OnDecodingCompleted(); void SetRefValid(bool valid) { ref_valid = valid; } @@ -265,10 +263,7 @@ namespace UMC_AV1_DECODER bool outputted; // set in [application thread] when frame is mapped to respective output mfxFrameSurface bool displayed; // set in [scheduler thread] when frame decoding is finished and // respective mfxFrameSurface prepared for output to application - bool decoded; // set in [application thread] to signal that frame is completed and respective reference counter decremented - // after it frame still may remain in [AV1Decoder::dpb], but only as reference - - bool dpb_updated; + bool ref_updated; bool decoding_started; // set in [application thread] right after frame submission to the driver started bool decoding_completed; // set in [scheduler thread] after getting driver status report for the frame diff --git a/_studio/shared/umc/codec/av1_dec/src/umc_av1_decoder.cpp b/_studio/shared/umc/codec/av1_dec/src/umc_av1_decoder.cpp index 9cc209e90..caed933a0 100755 --- a/_studio/shared/umc/codec/av1_dec/src/umc_av1_decoder.cpp +++ b/_studio/shared/umc/codec/av1_dec/src/umc_av1_decoder.cpp @@ -43,9 +43,7 @@ namespace UMC_AV1_DECODER , sequence_header(nullptr) , old_seqHdr(nullptr) , counter(0) - , Curr(nullptr) - , Curr_temp(nullptr) - , Repeat_show(0) + , lastest_submitted_frame(nullptr) , PreFrame_id(0) , OldPreFrame_id(0) , frame_order(0) @@ -262,7 +260,7 @@ namespace UMC_AV1_DECODER return UMC::UMC_OK; } - DPBType AV1Decoder::DPBUpdate(AV1DecoderFrame * prevFrame) + DPBType AV1Decoder::ReferenceListUpdate(AV1DecoderFrame * prevFrame) { assert(prevFrame); @@ -277,7 +275,7 @@ namespace UMC_AV1_DECODER const FrameHeader& fh = prevFrame->GetFrameHeader(); - prevFrame->DpbUpdated(true); + prevFrame->RefUpdated(true); if (fh.refresh_frame_flags == 0) return updatedFrameDPB; @@ -396,6 +394,8 @@ namespace UMC_AV1_DECODER if (fh.show_existing_frame) { std::unique_lock l(guard); + + //get repeat frame pFrame = frameDPB[fh.frame_to_show_map_idx]; assert(pFrame); repeateFrame = pFrame->GetMemID(); @@ -403,11 +403,11 @@ namespace UMC_AV1_DECODER //repeat frame reference counter increase here, and will decrease in queryframe() pFrame->IncrementReference(); - FrameHeader const& refFH = pFrame->GetFrameHeader(); - if (!refFH.showable_frame) + FrameHeader const& Repeat_H = pFrame->GetFrameHeader(); + if (!Repeat_H.showable_frame) throw av1_exception(UMC::UMC_ERR_INVALID_STREAM); - FrameHeader const& Repeat_H = pFrame->GetFrameHeader(); + //if repeat key frame, need refresh frame dpb if (Repeat_H.frame_type == KEY_FRAME) { for (uint8_t i = 0; i < NUM_REF_FRAMES; i++) @@ -422,7 +422,8 @@ namespace UMC_AV1_DECODER } } } - refs_temp = frameDPB; + + last_updated_refs = frameDPB; //store updated frame_dpb if (pPrevFrame) { DPBType & prevFrameDPB = pPrevFrame->frame_dpb; @@ -604,22 +605,22 @@ namespace UMC_AV1_DECODER AV1DecoderFrame* pPrevFrame = FindFrameByUID(counter - 1); AV1DecoderFrame* pFrameInProgress = FindFrameInProgress(); - DPBType updated_refs; + DPBType updated_refs; //store latest updated dpb UMC::MediaData tmper = *in; repeateFrame = UMC::FRAME_MID_INVALID; if ((tmper.GetDataSize() >= MINIMAL_DATA_SIZE) && pPrevFrame && !pFrameInProgress) { - if (!Repeat_show) + if (!last_frame_header.show_existing_frame) // if last frame is a repeat frame, don't need to call ReferenceListUpdate() { - if (!pPrevFrame->DpbUpdated()) + if (!pPrevFrame->RefUpdated()) { - updated_refs = DPBUpdate(pPrevFrame); - refs_temp = updated_refs; + updated_refs = ReferenceListUpdate(pPrevFrame); + last_updated_refs = updated_refs; } else { - updated_refs = refs_temp; + updated_refs = last_updated_refs; } } @@ -627,15 +628,11 @@ namespace UMC_AV1_DECODER { DPBType const& prevFrameDPB = pPrevFrame->frame_dpb; updated_refs = prevFrameDPB; - Repeat_show = 0; } - Curr_temp = Curr; } if (!pPrevFrame) { - updated_refs = refs_temp; - Curr_temp = Curr; - Repeat_show = 0; + updated_refs = last_updated_refs; } if ((!updated_refs.empty())&& (updated_refs[0] != nullptr)) @@ -1167,39 +1164,41 @@ namespace UMC_AV1_DECODER assert(size > 0); assert(size <= MAX_EXTERNAL_REFS); - refs_temp.resize(size); + last_updated_refs.resize(size); } void AV1Decoder::CompleteDecodedFrames(FrameHeader const& fh, AV1DecoderFrame* pCurrFrame, AV1DecoderFrame*) { std::unique_lock l(guard); - if ((Curr) && (!last_frame_header.show_existing_frame)) //if last frame is a repeat frame , do not insert it into output frames, its refcounter will decrease in QueryFrame() + + //if last frame is a repeat frame , do not insert it into output frames, its refcounter will decrease in QueryFrame() + if ((lastest_submitted_frame) && (!last_frame_header.show_existing_frame)) { - FrameHeader const& FH_OutTemp = Curr->GetFrameHeader(); - if (FH_OutTemp.show_frame) + FrameHeader const& FH_OutTemp = lastest_submitted_frame->GetFrameHeader(); + if (FH_OutTemp.show_frame) //display frame { bool bAdded = false; for(std::vector::iterator iter=outputed_frames.begin(); iter!=outputed_frames.end(); iter++) { AV1DecoderFrame* temp = *iter; - if (Curr->UID == temp->UID) + if (lastest_submitted_frame->UID == temp->UID) { bAdded = true; break; } } if (!bAdded) - outputed_frames.push_back(Curr); + outputed_frames.push_back(lastest_submitted_frame); } else { - // For no display case, it decrementReference here and frame.completedecoding() in working thread + // For no display frame, it decrementReference here and frame.completedecoding() in working thread if(pCurrFrame) { - if (Curr->UID == -1) - Curr = nullptr; - else if(Curr != pCurrFrame) - Curr->DecrementReference(); + if (lastest_submitted_frame->UID == -1) + lastest_submitted_frame = nullptr; + else if(lastest_submitted_frame != pCurrFrame) + lastest_submitted_frame->DecrementReference(); } } } @@ -1207,7 +1206,7 @@ namespace UMC_AV1_DECODER for(std::vector::iterator iter=outputed_frames.begin(); iter!=outputed_frames.end(); ) { AV1DecoderFrame* temp = *iter; - if(temp->Outputted() && temp->Displayed() && !temp->Decoded() && temp->DpbUpdated()) + if(temp->Outputted() && temp->Displayed() && temp->RefUpdated()) { temp->DecrementReference(); iter = outputed_frames.erase(iter); @@ -1218,24 +1217,15 @@ namespace UMC_AV1_DECODER // When no available buffer, don't update Curr buffer to avoid update DPB duplicated. if(pCurrFrame!= NULL) - Curr = pCurrFrame; + lastest_submitted_frame = pCurrFrame; last_frame_header = fh; //store latest frame header - - if (fh.show_existing_frame) - { - Repeat_show = 1; - } - else - { - Repeat_show = 0; - } } void AV1Decoder::FlushRepeatFrame(AV1DecoderFrame* frame) { std::unique_lock l(guard); - if (frame->Outputted() && frame->Displayed()) //repeat frame only need these 2 flags, as repeat frame will not call DpbUpdate() function + if (frame->Outputted() && frame->Displayed()) //repeat frame only need these 2 flags, as repeat frame will not call ReferenceListUpdate() function { frame->DecrementReference(); //repeat frame decrement here } diff --git a/_studio/shared/umc/codec/av1_dec/src/umc_av1_frame.cpp b/_studio/shared/umc/codec/av1_dec/src/umc_av1_frame.cpp index b93bd8f91..94f26a6fc 100755 --- a/_studio/shared/umc/codec/av1_dec/src/umc_av1_frame.cpp +++ b/_studio/shared/umc/codec/av1_dec/src/umc_av1_frame.cpp @@ -104,9 +104,8 @@ namespace UMC_AV1_DECODER { error = 0; displayed = false; - dpb_updated = false; + ref_updated = false; outputted = false; - decoded = false; skipped = false; decoding_started = false; decoding_completed = false; @@ -212,10 +211,6 @@ namespace UMC_AV1_DECODER return !data[SURFACE_DISPLAY].get(); } - bool AV1DecoderFrame::Decoded() const - { - return decoded; - } UMC::FrameMemID AV1DecoderFrame::GetMemID(int idx) const { @@ -254,13 +249,6 @@ namespace UMC_AV1_DECODER } } - void AV1DecoderFrame::OnDecodingCompleted() - { - DecrementReference(); - FreeReferenceFrames(); - decoded = true; - } - uint32_t AV1DecoderFrame::GetUpscaledWidth() const { return header->UpscaledWidth;