diff --git a/trunk/conf/full.conf b/trunk/conf/full.conf index 1d0df76207b..3b61c27c508 100644 --- a/trunk/conf/full.conf +++ b/trunk/conf/full.conf @@ -1530,6 +1530,15 @@ vhost http.remux.srs.com { # Overwrite by env SRS_VHOST_HTTP_REMUX_GUESS_HAS_AV for all vhosts. # Default: on guess_has_av on; + + # Whether to keep the h.264 SEI type NALU packet. + # DJI drone M30T will send many SEI type NALU packet, while iOS hardware decoder (Video Toolbox) + # dislike to feed it so many SEI NALU between NonIDR and IDR NALU packets. + # @see https://github.com/ossrs/srs/issues/4052 + # Overwrite by env SRS_VHOST_HTTP_REMUX_KEEP_AVC_NALU_SEI for all vhosts. + # Default: on + keep_avc_nalu_sei on; + # the stream mount for rtmp to remux to live streaming. # typical mount to [vhost]/[app]/[stream].flv # the variables: diff --git a/trunk/src/app/srs_app_config.cpp b/trunk/src/app/srs_app_config.cpp index 6b0bdb2a4c6..f6e5ba62059 100644 --- a/trunk/src/app/srs_app_config.cpp +++ b/trunk/src/app/srs_app_config.cpp @@ -2646,7 +2646,8 @@ srs_error_t SrsConfig::check_normal_config() for (int j = 0; j < (int)conf->directives.size(); j++) { string m = conf->at(j)->name; if (m != "enabled" && m != "mount" && m != "fast_cache" && m != "drop_if_not_match" - && m != "has_audio" && m != "has_video" && m != "guess_has_av") { + && m != "has_audio" && m != "has_video" && m != "guess_has_av" + && m != "keep_avc_nalu_sei") { return srs_error_new(ERROR_SYSTEM_CONFIG_INVALID, "illegal vhost.http_remux.%s of %s", m.c_str(), vhost->arg0().c_str()); } } @@ -8666,6 +8667,30 @@ bool SrsConfig::get_vhost_http_remux_guess_has_av(string vhost) return SRS_CONF_PERFER_TRUE(conf->arg0()); } +bool SrsConfig::get_vhost_http_remux_keep_avc_nalu_sei(string vhost) +{ + SRS_OVERWRITE_BY_ENV_BOOL2("srs.vhost.http_remux.keep_avc_nalu_sei"); // SRS_VHOST_HTTP_REMUX_KEEP_AVC_NALU_SEI + + static bool DEFAULT = true; + + SrsConfDirective* conf = get_vhost(vhost); + if (!conf) { + return DEFAULT; + } + + conf = conf->get("http_remux"); + if (!conf) { + return DEFAULT; + } + + conf = conf->get("keep_avc_nalu_sei"); + if (!conf || conf->arg0().empty()) { + return DEFAULT; + } + + return SRS_CONF_PERFER_TRUE(conf->arg0()); +} + string SrsConfig::get_vhost_http_remux_mount(string vhost) { SRS_OVERWRITE_BY_ENV_STRING("srs.vhost.http_remux.mount"); // SRS_VHOST_HTTP_REMUX_MOUNT diff --git a/trunk/src/app/srs_app_config.hpp b/trunk/src/app/srs_app_config.hpp index afa700a0c8c..7a7c31f9037 100644 --- a/trunk/src/app/srs_app_config.hpp +++ b/trunk/src/app/srs_app_config.hpp @@ -1098,6 +1098,7 @@ class SrsConfig bool get_vhost_http_remux_has_video(std::string vhost); // Whether guessing stream about audio or video track bool get_vhost_http_remux_guess_has_av(std::string vhost); + bool get_vhost_http_remux_keep_avc_nalu_sei(std::string vhost); // Get the http flv live stream mount point for vhost. // used to generate the flv stream mount path. virtual std::string get_vhost_http_remux_mount(std::string vhost); diff --git a/trunk/src/app/srs_app_http_stream.cpp b/trunk/src/app/srs_app_http_stream.cpp index 914cefd5c6b..e0650e949ee 100755 --- a/trunk/src/app/srs_app_http_stream.cpp +++ b/trunk/src/app/srs_app_http_stream.cpp @@ -19,6 +19,8 @@ using namespace std; #include #include +#include +#include #include #include #include @@ -630,6 +632,7 @@ srs_error_t SrsLiveStream::do_serve_http(ISrsHttpResponseWriter* w, ISrsHttpMess bool has_audio = _srs_config->get_vhost_http_remux_has_audio(req->vhost); bool has_video = _srs_config->get_vhost_http_remux_has_video(req->vhost); bool guess_has_av = _srs_config->get_vhost_http_remux_guess_has_av(req->vhost); + bool keep_avc_nalu_sei = _srs_config->get_vhost_http_remux_keep_avc_nalu_sei(req->vhost); if (srs_string_ends_with(entry->pattern, ".flv")) { w->header()->set_content_type("video/x-flv"); @@ -719,6 +722,9 @@ srs_error_t SrsLiveStream::do_serve_http(ISrsHttpResponseWriter* w, ISrsHttpMess entry->pattern.c_str(), enc_desc.c_str(), srsu2msi(mw_sleep), enc->has_cache(), msgs.max, drop_if_not_match, has_audio, has_video, guess_has_av); + SrsRtmpFormat format; + format.initialize(); + // TODO: free and erase the disabled entry after all related connections is closed. // TODO: FXIME: Support timeout for player, quit infinite-loop. while (entry->enabled) { @@ -748,7 +754,15 @@ srs_error_t SrsLiveStream::do_serve_http(ISrsHttpResponseWriter* w, ISrsHttpMess srs_trace("-> " SRS_CONSTS_LOG_HTTP_STREAM " http: got %d msgs, age=%d, min=%d, mw=%d", count, pprint->age(), SRS_PERF_MW_MIN_MSGS, srsu2msi(mw_sleep)); } - + + // Drop h.264 flv video tags with NALU SEI here to fix http-flv play error in safari mac. + // @see https://github.com/ossrs/srs/issues/4052 + if (!keep_avc_nalu_sei) { + if ((err = filter_avc_nalu_sei(format, msgs.msgs, count)) != srs_success) { + return srs_error_wrap(err, "filter avc/h264 nalu sei"); + } + } + // sendout all messages. if (ffe) { err = ffe->write_tags(msgs.msgs, count); @@ -763,7 +777,7 @@ srs_error_t SrsLiveStream::do_serve_http(ISrsHttpResponseWriter* w, ISrsHttpMess SrsSharedPtrMessage* msg = msgs.msgs[i]; srs_freep(msg); } - + // check send error code. if (err != srs_success) { return srs_error_wrap(err, "send messages"); @@ -873,6 +887,186 @@ srs_error_t SrsLiveStream::streaming_send_messages(ISrsBufferEncoder* enc, SrsSh return err; } +srs_error_t SrsLiveStream::filter_avc_nalu_sei(SrsRtmpFormat& format, SrsSharedPtrMessage** msgs, int& count) { + srs_error_t err = srs_success; + bool has_sei = false; + if ((err = has_avc_nalu_sei(format, msgs, count, has_sei)) != srs_success) { + return srs_error_wrap(err, "check avc nalu sei"); + } + + if (!has_sei) { + return err; + } + + std::vector remuxed_msgs; + for (size_t i = 0; i < count; i++) { + SrsSharedPtrMessage* msg = msgs[i]; + if (msg->is_video()) { + if ((err = format.on_video(msg)) != srs_success) { + return srs_error_wrap(err, "format consume video"); + } + + if (format.vcodec->id == SrsVideoCodecIdAVC && + format.video->avc_packet_type == SrsVideoAvcFrameTraitNALU) { + std::vector sei_idxes; + // found which nalu is SEI, store its index inorder to sei_idxes; + for (size_t j = 0; j < format.video->nb_samples; ++j) { + SrsSample* sample = &format.video->samples[j]; + + SrsAvcNaluType avc_nalu_type = SrsAvcNaluTypeReserved; + if ((err = SrsVideoFrame::parse_avc_nalu_type(sample, avc_nalu_type)) != srs_success) { + return srs_error_wrap(err, "parse avc nalu_type"); + } + if (avc_nalu_type == SrsAvcNaluTypeSEI) { + sei_idxes.push_back(j); + } + } + + if (sei_idxes.size() == 0) { // no sei nalu, keep this msg + remuxed_msgs.push_back(msg->copy()); + continue; + } else if (sei_idxes.size() == format.video->nb_samples) { + // all the nalu is SEI type, remove this msg + continue; + } else { + // part of nalu is SEI, do remux this msg. + int nalus_count = format.video->nb_samples - sei_idxes.size(); + + SrsSample nalus[nalus_count]; + size_t p = 0; + size_t r = 0; + for (size_t j = 0; j < format.video->nb_samples; j++) { + if (sei_idxes[r] == j) { + r++; + } else { + SrsSample* s = &format.video->samples[j]; + nalus[p].size = s->size; + nalus[p].bytes = s->bytes; + p++; + } + } + + SrsSharedPtrMessage* m = NULL; + if ((err = remux_avc_nalus(format, msg, &m, nalus, nalus_count)) + != srs_success) { + return srs_error_wrap(err, "remux avc nalus"); + } + + remuxed_msgs.push_back(m); + continue; + } + } + } + + remuxed_msgs.push_back(msg->copy()); + } + + // free the messages. + for (size_t i = 0; i < count; i++) { + SrsSharedPtrMessage* msg = msgs[i]; + srs_freep(msg); + } + + // copy remuxed_msgs -> msgs + count = remuxed_msgs.size(); + SrsSharedPtrMessage** pmsgs = remuxed_msgs.data(); + memcpy(msgs, pmsgs, count * sizeof(SrsSharedPtrMessage*)); + return err; +} + +srs_error_t SrsLiveStream::has_avc_nalu_sei(SrsRtmpFormat& format, SrsSharedPtrMessage** msgs, int& count, + bool& has_sei) { + srs_error_t err = srs_success; + has_sei = false; + + for (int i = 0; i < count; i++) { + SrsSharedPtrMessage* msg = msgs[i]; + if (msg->is_video()) { + if ((err = format.on_video(msg)) != srs_success) { + srs_error("has_sei: format.on_video error"); + return srs_error_wrap(err, "format consume video"); + } + + if (format.vcodec->id == SrsVideoCodecIdAVC && + format.video->avc_packet_type == SrsVideoAvcFrameTraitNALU) { + + for (size_t i = 0; i < format.video->nb_samples; ++i) { + SrsSample* sample = &format.video->samples[i]; + + SrsAvcNaluType avc_nalu_type = SrsAvcNaluTypeReserved; + if ((err = SrsVideoFrame::parse_avc_nalu_type(sample, avc_nalu_type)) != srs_success) { + return srs_error_wrap(err, "parse avc nalu_type"); + } + + if (avc_nalu_type == SrsAvcNaluTypeSEI) { + has_sei = true; + return err; + } + } + } + } + } + + return err; +} + +srs_error_t SrsLiveStream::remux_avc_nalus(const SrsRtmpFormat& format, + const SrsSharedPtrMessage* msg, + SrsSharedPtrMessage** output_msg, + SrsSample* nalus, + size_t nalu_count) +{ + srs_error_t err = srs_success; + + if (!msg->is_video()) { + return srs_error_new(ERROR_RTMP_MESSAGE_CREATE, "msg is not video"); + } + // normal flv video tag header 5 bytes; + // extened(enhanced) flv video tag header 8 bytes; + // but h264 use normal flv video tag header format; + const int video_tag_header_size = 5; + size_t nalu_header_size = 3; + if (format.vcodec->payload_format == SrsAvcPayloadFormatIbmf) { + nalu_header_size = format.vcodec->NAL_unit_length + 1; + } else if (format.vcodec->payload_format == SrsAvcPayloadFormatAnnexb) { + nalu_header_size = 3; // 00 00 01 + } else { + return srs_error_new(ERROR_RTMP_MESSAGE_CREATE, + "unkown nalu format type %d", + format.vcodec->payload_format); + } + + // calculate new video tag size; + size_t payload_size = video_tag_header_size; + + for (size_t i = 0; i < nalu_count; i++) { + payload_size += nalus[i].size + nalu_header_size; + } + + srs_assert(payload_size <= msg->size); + + SrsSharedPtrMessage* m = new SrsSharedPtrMessage(); + char* payload = (char*)malloc(payload_size); + char* payload_p = payload; + // memcpy flv header + memcpy(payload_p, msg->payload, video_tag_header_size); + payload_p += video_tag_header_size; + // memcpy nalus + for (size_t i = 0; i < nalu_count; i++) { + int n = nalus[i].size + nalu_header_size; + memcpy(payload_p, nalus[i].bytes - nalu_header_size, n); + payload_p += n; + } + + if ((err = m->create(msg, payload, payload_size)) != srs_success) { + return srs_error_wrap(err, "create SrsSharedPtrMessage error."); + } + + *output_msg = m; + + return err; +} + SrsLiveEntry::SrsLiveEntry(std::string m) { mount = m; diff --git a/trunk/src/app/srs_app_http_stream.hpp b/trunk/src/app/srs_app_http_stream.hpp index b3962d072c3..3e1d6d9e7ec 100755 --- a/trunk/src/app/srs_app_http_stream.hpp +++ b/trunk/src/app/srs_app_http_stream.hpp @@ -190,6 +190,13 @@ class SrsLiveStream : public ISrsHttpHandler virtual srs_error_t http_hooks_on_play(ISrsHttpMessage* r); virtual void http_hooks_on_stop(ISrsHttpMessage* r); virtual srs_error_t streaming_send_messages(ISrsBufferEncoder* enc, SrsSharedPtrMessage** msgs, int nb_msgs); + virtual srs_error_t filter_avc_nalu_sei(SrsRtmpFormat& format, SrsSharedPtrMessage** msgs, int& count); + virtual srs_error_t has_avc_nalu_sei(SrsRtmpFormat& format, SrsSharedPtrMessage** msgs, int& count, bool& has_sei); + virtual srs_error_t remux_avc_nalus(const SrsRtmpFormat& format, + const SrsSharedPtrMessage* msg, + SrsSharedPtrMessage** output_msg, + SrsSample* nalus, + size_t nalu_cout); }; // The Live Entry, to handle HTTP Live Streaming. diff --git a/trunk/src/kernel/srs_kernel_codec.hpp b/trunk/src/kernel/srs_kernel_codec.hpp index a1462d33282..165cdf9ae11 100644 --- a/trunk/src/kernel/srs_kernel_codec.hpp +++ b/trunk/src/kernel/srs_kernel_codec.hpp @@ -1367,6 +1367,7 @@ class SrsFormat // The packet is muxed in FLV format, defined in flv specification. // Demux the sps/pps from sequence header. // Demux the samples from NALUs. + // TODO: rename to video_video_demux, because it can demux both h265 and h264. virtual srs_error_t video_avc_demux(SrsBuffer* stream, int64_t timestamp); #ifdef SRS_H265 private: diff --git a/trunk/src/kernel/srs_kernel_flv.cpp b/trunk/src/kernel/srs_kernel_flv.cpp index 20111e8396f..39db771acac 100644 --- a/trunk/src/kernel/srs_kernel_flv.cpp +++ b/trunk/src/kernel/srs_kernel_flv.cpp @@ -258,6 +258,35 @@ srs_error_t SrsSharedPtrMessage::create(SrsMessageHeader* pheader, char* payload return err; } +srs_error_t SrsSharedPtrMessage::create(const SrsSharedPtrMessage* msg, char* payload, int size) +{ + srs_error_t err = srs_success; + + if (size < 0) { + return srs_error_new(ERROR_RTMP_MESSAGE_CREATE, "create message size=%d", size); + } + + srs_assert(!ptr); + ptr = new SrsSharedPtrPayload(); + + // direct attach the data. + if (msg) { + ptr->header.message_type = msg->ptr->header.message_type; + ptr->header.payload_length = size; + ptr->header.perfer_cid = msg->ptr->header.perfer_cid; + this->timestamp = msg->timestamp; + this->stream_id = msg->stream_id; + } + ptr->payload = payload; + ptr->size = size; + + // message can access it. + this->payload = ptr->payload; + this->size = ptr->size; + + return err; +} + void SrsSharedPtrMessage::wrap(char* payload, int size) { srs_assert(!ptr); @@ -298,18 +327,18 @@ bool SrsSharedPtrMessage::check(int stream_id) return false; } -bool SrsSharedPtrMessage::is_av() +bool SrsSharedPtrMessage::is_av() const { return ptr->header.message_type == RTMP_MSG_AudioMessage || ptr->header.message_type == RTMP_MSG_VideoMessage; } -bool SrsSharedPtrMessage::is_audio() +bool SrsSharedPtrMessage::is_audio() const { return ptr->header.message_type == RTMP_MSG_AudioMessage; } -bool SrsSharedPtrMessage::is_video() +bool SrsSharedPtrMessage::is_video() const { return ptr->header.message_type == RTMP_MSG_VideoMessage; } diff --git a/trunk/src/kernel/srs_kernel_flv.hpp b/trunk/src/kernel/srs_kernel_flv.hpp index 349c4c5795f..ed43bb54465 100644 --- a/trunk/src/kernel/srs_kernel_flv.hpp +++ b/trunk/src/kernel/srs_kernel_flv.hpp @@ -304,6 +304,11 @@ class SrsSharedPtrMessage // @remark user should never free the payload. // @param pheader, the header to copy to the message. NULL to ignore. virtual srs_error_t create(SrsMessageHeader* pheader, char* payload, int size); + // Create shared ptr message, + // from another SrsSharedPtrMessage and replace its payload. + // @remark user should never free the payload. + // @param msg, the SrsSharedPtrMessage to copy its ptr->header to the message. NULL to ignore. + virtual srs_error_t create(const SrsSharedPtrMessage* msg, char* payload, int size); // Create shared ptr message from RAW payload. // @remark Note that the header is set to zero. virtual void wrap(char* payload, int size); @@ -317,9 +322,9 @@ class SrsSharedPtrMessage // @return whether stream id already set. virtual bool check(int stream_id); public: - virtual bool is_av(); - virtual bool is_audio(); - virtual bool is_video(); + virtual bool is_av() const; + virtual bool is_audio() const; + virtual bool is_video() const; public: // generate the chunk header to cache. // @return the size of header. diff --git a/trunk/src/utest/srs_utest_config.cpp b/trunk/src/utest/srs_utest_config.cpp index 6bed81306c3..68dfa6eaacc 100644 --- a/trunk/src/utest/srs_utest_config.cpp +++ b/trunk/src/utest/srs_utest_config.cpp @@ -4786,6 +4786,20 @@ VOID TEST(ConfigEnvTest, CheckEnvValuesHttpRemux) SrsSetEnvConfig(guess_has_av2, "SRS_VHOST_HTTP_REMUX_GUESS_HAS_AV", "on"); EXPECT_TRUE(conf.get_vhost_http_remux_guess_has_av("__defaultVhost__")); } + + if (true) { + EXPECT_TRUE(conf.get_vhost_http_remux_keep_avc_nalu_sei("__defaultVhost__")); + + SrsSetEnvConfig(keep_avc_nalu_sei_false, "SRS_VHOST_HTTP_REMUX_KEEP_AVC_NALU_SEI", "off"); + EXPECT_FALSE(conf.get_vhost_http_remux_keep_avc_nalu_sei("__defaultVhost__")); + + SrsSetEnvConfig(keep_avc_nalu_sei_true, "SRS_VHOST_HTTP_REMUX_KEEP_AVC_NALU_SEI", "on"); + EXPECT_TRUE(conf.get_vhost_http_remux_keep_avc_nalu_sei("__defaultVhost__")); + + SrsSetEnvConfig(keep_avc_nalu_sei_default_true, "SRS_VHOST_HTTP_REMUX_KEEP_AVC_NALU_SEI", "offx"); + EXPECT_TRUE(conf.get_vhost_http_remux_keep_avc_nalu_sei("__defaultVhost__")); + + } } VOID TEST(ConfigEnvTest, CheckEnvValuesDash)