Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

VP9 SVC fixes #1849

Merged
merged 1 commit into from
Nov 6, 2019
Merged

VP9 SVC fixes #1849

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion html/vp9svctest.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -646,7 +646,7 @@ function addSvcButtons(feed) {
$('#sl' + index + '-1').removeClass('btn-primary btn-info').addClass('btn-primary');
if(!$('#sl' + index + '-0').hasClass('btn-success'))
$('#sl' + index + '-0').removeClass('btn-primary btn-info').addClass('btn-primary');
feeds[index].send({message: { request: "configure", spatial_layer: 1 }});
feeds[index].send({message: { request: "configure", spatial_layer: 2 }});
});
$('#tl' + index + '-0').removeClass('btn-primary btn-success').addClass('btn-primary')
.unbind('click').click(function() {
Expand Down
167 changes: 99 additions & 68 deletions plugins/janus_streaming.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -977,9 +977,7 @@ typedef struct janus_streaming_rtp_relay_packet {
uint16_t seq_number;
/* The following are only relevant for VP9 SVC*/
gboolean svc;
int spatial_layer;
int temporal_layer;
uint8_t pbit, dbit, ubit, bbit, ebit;
janus_vp9_svc_info svc_info;
} janus_streaming_rtp_relay_packet;
static janus_streaming_rtp_relay_packet exit_packet;
static void janus_streaming_rtp_relay_packet_free(janus_streaming_rtp_relay_packet *pkt) {
Expand Down Expand Up @@ -1163,6 +1161,7 @@ typedef struct janus_streaming_session {
/* The following are only relevant the mountpoint is VP9-SVC, and are not to be confused with VP8
* simulcast, which has similar info (substream/templayer) but in a completely different context */
int spatial_layer, target_spatial_layer;
gint64 last_spatial_layer[2];
int temporal_layer, target_temporal_layer;
gboolean stopping;
volatile gint hangingup;
Expand Down Expand Up @@ -3904,7 +3903,10 @@ static void janus_streaming_hangup_media_internal(janus_plugin_session *handle)
janus_rtp_simulcasting_context_reset(&session->sim_context);
janus_vp8_simulcast_context_reset(&session->vp8_context);
session->spatial_layer = -1;
session->target_spatial_layer = 1; /* FIXME Chrome sends 0 and 1 */
session->target_spatial_layer = 2; /* FIXME Chrome sends 0, 1 and 2 (if using EnabledByFlag_3SL3TL) */
session->last_spatial_layer[0] = 0;
session->last_spatial_layer[1] = 0;
session->last_spatial_layer[2] = 0;
session->temporal_layer = -1;
session->target_temporal_layer = 2; /* FIXME Chrome sends 0, 1 and 2 */
session->stopping = TRUE;
Expand Down Expand Up @@ -4148,7 +4150,7 @@ static void *janus_streaming_handler(void *data) {
}
/* In case this mountpoint is doing VP9-SVC, let's aim high by default */
session->spatial_layer = -1;
session->target_spatial_layer = 1; /* FIXME Chrome sends 0 and 1 */
session->target_spatial_layer = 2; /* FIXME Chrome sends 0, 1 and 2 (if using EnabledByFlag_3SL3TL) */
session->temporal_layer = -1;
session->target_temporal_layer = 2; /* FIXME Chrome sends 0, 1 and 2 */
/* Unless the request contains a target for either layer */
Expand Down Expand Up @@ -6716,19 +6718,10 @@ static void *janus_streaming_relay_thread(void *data) {
int plen = 0;
char *payload = janus_rtp_payload(buffer, bytes, &plen);
if(payload) {
uint8_t pbit = 0, dbit = 0, ubit = 0, bbit = 0, ebit = 0;
int found = 0, spatial_layer = 0, temporal_layer = 0;
if(janus_vp9_parse_svc(payload, plen, &found, &spatial_layer, &temporal_layer, &pbit, &dbit, &ubit, &bbit, &ebit) == 0) {
if(found) {
packet.svc = TRUE;
packet.spatial_layer = spatial_layer;
packet.temporal_layer = temporal_layer;
packet.pbit = pbit;
packet.dbit = dbit;
packet.ubit = ubit;
packet.bbit = bbit;
packet.ebit = ebit;
}
gboolean found = FALSE;
memset(&packet.svc_info, 0, sizeof(packet.svc_info));
if(janus_vp9_parse_svc(payload, plen, &found, &packet.svc_info) == 0) {
packet.svc = found;
}
}
}
Expand Down Expand Up @@ -6943,97 +6936,135 @@ static void janus_streaming_relay_rtp_packet(gpointer data, gpointer user_data)
/* There is: check if this is a layer that can be dropped for this viewer
* Note: Following core inspired by the excellent job done by Sergio Garcia Murillo here:
* https://github.com/medooze/media-server/blob/master/src/vp9/VP9LayerSelector.cpp */
int plen = 0;
char *payload = janus_rtp_payload((char *)packet->data, packet->length, &plen);
gboolean keyframe = janus_vp9_is_keyframe((const char *)payload, plen);
gboolean override_mark_bit = FALSE, has_marker_bit = packet->data->markerbit;
int temporal_layer = session->temporal_layer;
if(session->target_temporal_layer > session->temporal_layer) {
/* We need to upscale */
JANUS_LOG(LOG_HUGE, "We need to upscale temporally:\n");
if(packet->ubit && packet->bbit && packet->temporal_layer <= session->target_temporal_layer) {
JANUS_LOG(LOG_HUGE, " -- Upscaling temporal layer: %u --> %u\n",
packet->temporal_layer, session->target_temporal_layer);
session->temporal_layer = packet->temporal_layer;
temporal_layer = session->temporal_layer;
/* Notify the viewer */
json_t *event = json_object();
json_object_set_new(event, "streaming", json_string("event"));
json_t *result = json_object();
json_object_set_new(result, "temporal_layer", json_integer(session->temporal_layer));
json_object_set_new(event, "result", result);
gateway->push_event(session->handle, &janus_streaming_plugin, NULL, event, NULL);
json_decref(event);
int spatial_layer = session->spatial_layer;
gint64 now = janus_get_monotonic_time();
if(packet->svc_info.spatial_layer >= 0 && packet->svc_info.spatial_layer <= 2)
session->last_spatial_layer[packet->svc_info.spatial_layer] = now;
if(session->target_spatial_layer > session->spatial_layer) {
JANUS_LOG(LOG_HUGE, "We need to upscale spatially: (%d < %d)\n",
session->spatial_layer, session->target_spatial_layer);
/* We need to upscale: wait for a keyframe */
if(keyframe) {
int new_spatial_layer = session->target_spatial_layer;
while(new_spatial_layer > session->spatial_layer && new_spatial_layer > 0) {
if(now - session->last_spatial_layer[new_spatial_layer] >= 250000) {
/* We haven't received packets from this layer for a while, try a lower layer */
JANUS_LOG(LOG_HUGE, "Haven't received packets from layer %d for a while, trying %d instead...\n",
new_spatial_layer, new_spatial_layer-1);
new_spatial_layer--;
} else {
break;
}
}
if(new_spatial_layer > session->spatial_layer) {
JANUS_LOG(LOG_HUGE, " -- Upscaling spatial layer: %d --> %d (need %d)\n",
session->spatial_layer, new_spatial_layer, session->target_spatial_layer);
session->spatial_layer = new_spatial_layer;
spatial_layer = session->spatial_layer;
/* Notify the viewer */
json_t *event = json_object();
json_object_set_new(event, "streaming", json_string("event"));
json_t *result = json_object();
json_object_set_new(result, "spatial_layer", json_integer(session->spatial_layer));
if(session->temporal_layer == -1) {
/* We just started: initialize the temporal layer and notify that too */
session->temporal_layer = 0;
json_object_set_new(result, "temporal_layer", json_integer(session->temporal_layer));
}
json_object_set_new(event, "result", result);
gateway->push_event(session->handle, &janus_streaming_plugin, NULL, event, NULL);
json_decref(event);
}
}
} else if(session->target_temporal_layer < session->temporal_layer) {
} else if(session->target_spatial_layer < session->spatial_layer) {
/* We need to downscale */
JANUS_LOG(LOG_HUGE, "We need to downscale temporally:\n");
if(packet->ebit) {
JANUS_LOG(LOG_HUGE, " -- Downscaling temporal layer: %u --> %u\n",
session->temporal_layer, session->target_temporal_layer);
session->temporal_layer = session->target_temporal_layer;
JANUS_LOG(LOG_HUGE, "We need to downscale spatially: (%d > %d)\n",
session->spatial_layer, session->target_spatial_layer);
gboolean downscaled = FALSE;
if(!packet->svc_info.fbit && keyframe) {
/* Non-flexible mode: wait for a keyframe */
downscaled = TRUE;
} else if(packet->svc_info.fbit && packet->svc_info.ebit) {
/* Flexible mode: check the E bit */
downscaled = TRUE;
}
if(downscaled) {
JANUS_LOG(LOG_HUGE, " -- Downscaling spatial layer: %d --> %d\n",
session->spatial_layer, session->target_spatial_layer);
session->spatial_layer = session->target_spatial_layer;
/* Notify the viewer */
json_t *event = json_object();
json_object_set_new(event, "streaming", json_string("event"));
json_t *result = json_object();
json_object_set_new(result, "temporal_layer", json_integer(session->temporal_layer));
json_object_set_new(result, "spatial_layer", json_integer(session->spatial_layer));
json_object_set_new(event, "result", result);
gateway->push_event(session->handle, &janus_streaming_plugin, NULL, event, NULL);
json_decref(event);
}
}
if(temporal_layer < packet->temporal_layer) {
if(spatial_layer < packet->svc_info.spatial_layer) {
/* Drop the packet: update the context to make sure sequence number is increased normally later */
JANUS_LOG(LOG_HUGE, "Dropping packet (temporal layer %d < %d)\n", temporal_layer, packet->temporal_layer);
JANUS_LOG(LOG_HUGE, "Dropping packet (spatial layer %d < %d)\n", spatial_layer, packet->svc_info.spatial_layer);
session->context.v_base_seq++;
return;
} else if(packet->svc_info.ebit && spatial_layer == packet->svc_info.spatial_layer) {
/* If we stop at layer 0, we need a marker bit now, as the one from layer 1 will not be received */
override_mark_bit = TRUE;
}
int spatial_layer = session->spatial_layer;
if(session->target_spatial_layer > session->spatial_layer) {
JANUS_LOG(LOG_HUGE, "We need to upscale spatially:\n");
int temporal_layer = session->temporal_layer;
if(session->target_temporal_layer > session->temporal_layer) {
/* We need to upscale */
if(packet->pbit == 0 && packet->bbit && packet->spatial_layer == session->spatial_layer+1) {
JANUS_LOG(LOG_HUGE, " -- Upscaling spatial layer: %u --> %u\n",
packet->spatial_layer, session->target_spatial_layer);
session->spatial_layer = packet->spatial_layer;
spatial_layer = session->spatial_layer;
JANUS_LOG(LOG_HUGE, "We need to upscale temporally: (%d < %d)\n",
session->temporal_layer, session->target_temporal_layer);
if(packet->svc_info.ubit && packet->svc_info.bbit &&
packet->svc_info.temporal_layer > session->temporal_layer &&
packet->svc_info.temporal_layer <= session->target_temporal_layer) {
JANUS_LOG(LOG_HUGE, " -- Upscaling temporal layer: %d --> %d (want %d)\n",
session->temporal_layer, packet->svc_info.temporal_layer, session->target_temporal_layer);
session->temporal_layer = packet->svc_info.temporal_layer;
temporal_layer = session->temporal_layer;
/* Notify the viewer */
json_t *event = json_object();
json_object_set_new(event, "streaming", json_string("event"));
json_t *result = json_object();
json_object_set_new(result, "spatial_layer", json_integer(session->spatial_layer));
json_object_set_new(result, "temporal_layer", json_integer(session->temporal_layer));
json_object_set_new(event, "result", result);
gateway->push_event(session->handle, &janus_streaming_plugin, NULL, event, NULL);
json_decref(event);
}
} else if(session->target_spatial_layer < session->spatial_layer) {
} else if(session->target_temporal_layer < session->temporal_layer) {
/* We need to downscale */
JANUS_LOG(LOG_HUGE, "We need to downscale spatially:\n");
if(packet->ebit) {
JANUS_LOG(LOG_HUGE, " -- Downscaling spatial layer: %u --> %u\n",
session->spatial_layer, session->target_spatial_layer);
session->spatial_layer = session->target_spatial_layer;
JANUS_LOG(LOG_HUGE, "We need to downscale temporally: (%d > %d)\n",
session->temporal_layer, session->target_temporal_layer);
if(packet->svc_info.ebit && packet->svc_info.temporal_layer == session->target_temporal_layer) {
JANUS_LOG(LOG_HUGE, " -- Downscaling temporal layer: %d --> %d\n",
session->temporal_layer, session->target_temporal_layer);
session->temporal_layer = session->target_temporal_layer;
/* Notify the viewer */
json_t *event = json_object();
json_object_set_new(event, "streaming", json_string("event"));
json_t *result = json_object();
json_object_set_new(result, "spatial_layer", json_integer(session->spatial_layer));
json_object_set_new(result, "temporal_layer", json_integer(session->temporal_layer));
json_object_set_new(event, "result", result);
gateway->push_event(session->handle, &janus_streaming_plugin, NULL, event, NULL);
json_decref(event);
}
}
if(spatial_layer < packet->spatial_layer) {
if(temporal_layer < packet->svc_info.temporal_layer) {
/* Drop the packet: update the context to make sure sequence number is increased normally later */
JANUS_LOG(LOG_HUGE, "Dropping packet (spatial layer %d < %d)\n", spatial_layer, packet->spatial_layer);
JANUS_LOG(LOG_HUGE, "Dropping packet (temporal layer %d < %d)\n", temporal_layer, packet->svc_info.temporal_layer);
session->context.v_base_seq++;
return;
} else if(packet->ebit && spatial_layer == packet->spatial_layer) {
/* If we stop at layer 0, we need a marker bit now, as the one from layer 1 will not be received */
override_mark_bit = TRUE;
}
/* If we got here, we can send the frame: this doesn't necessarily mean it's
* one of the layers the user wants, as there may be dependencies involved */
JANUS_LOG(LOG_HUGE, "Sending packet (spatial=%d, temporal=%d)\n",
packet->spatial_layer, packet->temporal_layer);
/* Fix sequence number and timestamp (video source switching may be involved) */
packet->svc_info.spatial_layer, packet->svc_info.temporal_layer);
/* Fix sequence number and timestamp (publisher switching may be involved) */
janus_rtp_header_update(packet->data, &session->context, TRUE, 4500);
if(override_mark_bit && !has_marker_bit) {
packet->data->markerbit = 1;
Expand All @@ -7043,7 +7074,7 @@ static void janus_streaming_relay_rtp_packet(gpointer data, gpointer user_data)
if(override_mark_bit && !has_marker_bit) {
packet->data->markerbit = 0;
}
/* Restore the timestamp and sequence number to what the video source set them to */
/* Restore the timestamp and sequence number to what the publisher set them to */
packet->data->timestamp = htonl(packet->timestamp);
packet->data->seq_number = htons(packet->seq_number);
} else if(packet->simulcast) {
Expand Down
Loading