Enable packetization support for multi-slice encoding

call/rtp_video_sender.cc

@@ -324,7 +324,9 @@ EncodedImageCallback::Result RtpVideoSender::OnEncodedImage(
   RTC_DCHECK_LT(stream_index, rtp_modules_.size());
   RTPVideoHeader rtp_video_header = params_[stream_index].GetRtpVideoHeader(
       encoded_image, codec_specific_info, shared_frame_id_);
-
+  if (codec_specific_info->codecSpecific.H264.last_fragment_in_frame)
+    absl::get<RTPVideoHeaderH264>(rtp_video_header.video_type_header)
+        .has_last_fragement = true;
   uint32_t frame_id;
   if (!rtp_modules_[stream_index]->Sending()) {
     // The payload router could be active but this module isn't sending.

modules/rtp_rtcp/source/rtp_format.h

@@ -35,7 +35,7 @@ class RtpPacketizer {
   virtual size_t SetPayloadData(
       const uint8_t* payload_data,
       size_t payload_size,
-      const RTPFragmentationHeader* fragmentation) = 0;
+      const RTPFragmentationHeader* fragmentation, bool end_of_frame) = 0;
 
   // Get the next payload with payload header.
   // Write payload and set marker bit of the |packet|.

modules/rtp_rtcp/source/rtp_format_h264.cc

@@ -85,7 +85,8 @@ RtpPacketizerH264::RtpPacketizerH264(size_t max_payload_len,
     : max_payload_len_(max_payload_len),
       last_packet_reduction_len_(last_packet_reduction_len),
       num_packets_left_(0),
-      packetization_mode_(packetization_mode) {
+      packetization_mode_(packetization_mode),
+      end_of_frame_(true) {
   // Guard against uninitialized memory in packetization_mode.
   RTC_CHECK(packetization_mode == H264PacketizationMode::NonInterleaved ||
             packetization_mode == H264PacketizationMode::SingleNalUnit);
@@ -104,10 +105,16 @@ RtpPacketizerH264::Fragment::Fragment(const Fragment& fragment)
 size_t RtpPacketizerH264::SetPayloadData(
     const uint8_t* payload_data,
     size_t payload_size,
-    const RTPFragmentationHeader* fragmentation) {
+    const RTPFragmentationHeader* fragmentation,
+    bool end_of_frame) {
   RTC_DCHECK(packets_.empty());
   RTC_DCHECK(input_fragments_.empty());
   RTC_DCHECK(fragmentation);
+  end_of_frame_ = end_of_frame;
+
+  // Different fragments allows to be packetized as FU or STAP-A or NAL,
+  // so there's no need to look back into history about how previous slice
+  // was packetized before.
   for (int i = 0; i < fragmentation->fragmentationVectorSize; ++i) {
     const uint8_t* buffer =
         &payload_data[fragmentation->fragmentationOffset[i]];
@@ -348,7 +355,10 @@ bool RtpPacketizerH264::NextPacket(RtpPacketToSend* rtp_packet) {
     RTC_DCHECK_LE(rtp_packet->payload_size(),
                   max_payload_len_ - last_packet_reduction_len_);
   }
-  rtp_packet->SetMarker(packets_.empty());
+  // We will only set marker if this is really last slice
+  // of the AU.
+  rtp_packet->SetMarker(packets_.empty() && end_of_frame_);
+
   --num_packets_left_;
   return true;
 }

modules/rtp_rtcp/source/rtp_format_h264.h

@@ -34,7 +34,8 @@ class RtpPacketizerH264 : public RtpPacketizer {
 
   size_t SetPayloadData(const uint8_t* payload_data,
                         size_t payload_size,
-                        const RTPFragmentationHeader* fragmentation) override;
+                        const RTPFragmentationHeader* fragmentation,
+                        bool end_of_frame) override;
 
   // Get the next payload with H264 payload header.
   // Write payload and set marker bit of the |packet|.
@@ -93,6 +94,7 @@ class RtpPacketizerH264 : public RtpPacketizer {
   const H264PacketizationMode packetization_mode_;
   std::deque<Fragment> input_fragments_;
   std::queue<PacketUnit> packets_;
+  bool end_of_frame_;
 
   RTC_DISALLOW_COPY_AND_ASSIGN(RtpPacketizerH264);
 };

modules/rtp_rtcp/source/rtp_format_h265.cc

@@ -117,7 +117,8 @@ RtpPacketizerH265::Fragment::Fragment(const Fragment& fragment)
 size_t RtpPacketizerH265::SetPayloadData(
     const uint8_t* payload_data,
     size_t payload_size,
-    const RTPFragmentationHeader* fragmentation) {
+    const RTPFragmentationHeader* fragmentation,
+    bool /*end_of_frame*/) {
   RTC_DCHECK(packets_.empty());
   RTC_DCHECK(input_fragments_.empty());
   RTC_DCHECK(fragmentation);

modules/rtp_rtcp/source/rtp_format_h265.h

@@ -25,7 +25,7 @@ class RtpPacketizerH265 : public RtpPacketizer {
 
   size_t SetPayloadData(const uint8_t* payload_data,
                         size_t payload_size,
-                        const RTPFragmentationHeader* fragmentation) override;
+                        const RTPFragmentationHeader* fragmentation, bool end_of_frame) override;
 
   // Get the next payload with H.265 payload header.
   // buffer is a pointer to where the output will be written.

modules/rtp_rtcp/source/rtp_format_video_generic.cc

@@ -43,7 +43,8 @@ RtpPacketizerGeneric::~RtpPacketizerGeneric() {}
 size_t RtpPacketizerGeneric::SetPayloadData(
     const uint8_t* payload_data,
     size_t payload_size,
-    const RTPFragmentationHeader* fragmentation) {
+    const RTPFragmentationHeader* fragmentation,
+    bool /*end_of_frame*/) {
   payload_data_ = payload_data;
   payload_size_ = payload_size;
 

modules/rtp_rtcp/source/rtp_format_video_generic.h

@@ -39,7 +39,8 @@ class RtpPacketizerGeneric : public RtpPacketizer {
   // Returns total number of packets to be generated.
   size_t SetPayloadData(const uint8_t* payload_data,
                         size_t payload_size,
-                        const RTPFragmentationHeader* fragmentation) override;
+                        const RTPFragmentationHeader* fragmentation,
+                        bool end_of_frame) override;
 
   // Get the next payload with generic payload header.
   // Write payload and set marker bit of the |packet|.

modules/rtp_rtcp/source/rtp_format_vp8.cc

@@ -175,7 +175,8 @@ RtpPacketizerVp8::~RtpPacketizerVp8() {}
 size_t RtpPacketizerVp8::SetPayloadData(
     const uint8_t* payload_data,
     size_t payload_size,
-    const RTPFragmentationHeader* /* fragmentation */) {
+    const RTPFragmentationHeader* /* fragmentation */,
+    bool /*end_of_frame*/) {
   payload_data_ = payload_data;
   payload_size_ = payload_size;
   if (GeneratePackets() < 0) {

modules/rtp_rtcp/source/rtp_format_vp8.h

@@ -48,7 +48,8 @@ class RtpPacketizerVp8 : public RtpPacketizer {
 
   size_t SetPayloadData(const uint8_t* payload_data,
                         size_t payload_size,
-                        const RTPFragmentationHeader* fragmentation) override;
+                        const RTPFragmentationHeader* fragmentation,
+                        bool end_of_frame) override;
 
   // Get the next payload with VP8 payload header.
   // Write payload and set marker bit of the |packet|.

modules/rtp_rtcp/source/rtp_format_vp9.cc

@@ -478,7 +478,8 @@ std::string RtpPacketizerVp9::ToString() {
 size_t RtpPacketizerVp9::SetPayloadData(
     const uint8_t* payload,
     size_t payload_size,
-    const RTPFragmentationHeader* fragmentation) {
+    const RTPFragmentationHeader* fragmentation,
+    bool /*end_of_frame*/) {
   payload_ = payload;
   payload_size_ = payload_size;
   GeneratePackets();

modules/rtp_rtcp/source/rtp_format_vp9.h

@@ -43,7 +43,8 @@ class RtpPacketizerVp9 : public RtpPacketizer {
   // The payload data must be one encoded VP9 layer frame.
   size_t SetPayloadData(const uint8_t* payload,
                         size_t payload_size,
-                        const RTPFragmentationHeader* fragmentation) override;
+                        const RTPFragmentationHeader* fragmentation,
+                        bool end_of_frame) override;
 
   // Gets the next payload with VP9 payload header.
   // Write payload and set marker bit of the |packet|.

modules/rtp_rtcp/source/rtp_sender_video.cc

@@ -295,6 +295,7 @@ bool RTPSenderVideo::SendVideo(enum VideoCodecType video_type,
   size_t fec_packet_overhead;
   bool red_enabled;
   int32_t retransmission_settings;
+  bool frame_completed = true;
   {
     rtc::CritScope cs(&crit_);
     // According to
@@ -307,7 +308,12 @@ bool RTPSenderVideo::SendVideo(enum VideoCodecType video_type,
     // packet in each group of packets which make up another type of frame
     // (e.g. a P-Frame) only if the current value is different from the previous
     // value sent.
-    if (video_header) {
+    if (video_header && video_header->codec == kVideoCodecH264 &&
+        !absl::get<RTPVideoHeaderH264>(video_header->video_type_header)
+             .has_last_fragement) {
+      frame_completed = false;
+    }
+    if (video_header && frame_completed) {
       // Set rotation when key frame or when changed (to follow standard).
       // Or when different from 0 (to follow current receiver implementation).
       VideoRotation current_rotation = video_header->rotation;
@@ -363,8 +369,8 @@ bool RTPSenderVideo::SendVideo(enum VideoCodecType video_type,
       video_header ? GetTemporalId(*video_header) : kNoTemporalIdx;
   StorageType storage = GetStorageType(temporal_id, retransmission_settings,
                                        expected_retransmission_time_ms);
-  size_t num_packets =
-      packetizer->SetPayloadData(payload_data, payload_size, fragmentation);
+  size_t num_packets = packetizer->SetPayloadData(
+      payload_data, payload_size, fragmentation, frame_completed);
 
   if (num_packets == 0)
     return false;

modules/video_coding/codecs/h264/include/h264_globals.h

@@ -81,6 +81,13 @@ struct RTPVideoHeaderH264 {
   // depending along with Temporal ID (obtained from RTP header extn).
   // '0' if PictureID does not exist.
   uint16_t picture_id;
+  // For support slice-based transmission, mark end of a frame so that
+  // the H.264 packetizer will not set marker bit for the last fragment of
+  // current outgoing data if it does not contain last fragment of the frame;
+  // and will treat the first fragment of the frame as continuous playload, so
+  // that it will not create FU header or STAP-A header on first fragment if contains
+  // last fragment of the frame.
+  bool has_last_fragement;
 };
 
 }  // namespace webrtc

modules/video_coding/include/video_codec_interface.h

@@ -71,6 +71,7 @@ struct CodecSpecificInfoH264 {
   H264PacketizationMode packetization_mode;
   uint8_t simulcast_idx;
   int16_t picture_id; // Required by temporal scalability
+  bool last_fragment_in_frame;
 };
 
 union CodecSpecificInfoUnion {