Set is_screencast on NativeVideoSource creation

Makes InternalSource's is_screencast configurable when creating a
NativeVideoSource. This improves the stream's latency when vp9 is
used for sharing screen content.

.nanpa/properly_set_is_screencast.kdl

@@ -0,0 +1,4 @@
+patch type="added" "Set is_screencast on VideoTrackSource creation" package="webrtc-sys"
+patch type="added" "Set is_screencast on NativeVideoSource creation" package="libwebrtc"
+patch type="added" "Extend NewVideoSourceRequest with is_screencast" package="livekit-ffi"
+patch type="changed" "Use new NativeVideoSource constructor in examples" package="examples"

examples/wgpu_room/src/logo_track.rs

@@ -47,10 +47,10 @@ pub struct LogoTrack {
 impl LogoTrack {
     pub fn new(room: Arc<Room>) -> Self {
         Self {
-            rtc_source: NativeVideoSource::new(VideoResolution {
-                width: FB_WIDTH as u32,
-                height: FB_HEIGHT as u32,
-            }),
+            rtc_source: NativeVideoSource::new(
+                VideoResolution { width: FB_WIDTH as u32, height: FB_HEIGHT as u32 },
+                false,
+            ),
             room,
             handle: None,
         }
@@ -83,11 +83,7 @@ impl LogoTrack {
             )
             .await?;
 
-        let handle = TrackHandle {
-            close_tx,
-            task,
-            track,
-        };
+        let handle = TrackHandle { close_tx, task, track };
 
         self.handle = Some(handle);
         Ok(())
@@ -98,10 +94,7 @@ impl LogoTrack {
             let _ = handle.close_tx.send(());
             let _ = handle.task.await;
 
-            self.room
-                .local_participant()
-                .unpublish_track(&handle.track.sid())
-                .await?;
+            self.room.local_participant().unpublish_track(&handle.track.sid()).await?;
         }
         Ok(())
     }

libwebrtc/src/native/video_source.rs

@@ -1,4 +1,4 @@
-// Copyright 2023 LiveKit, Inc.
+// Copyright 2023-2025 LiveKit, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -50,11 +50,12 @@ struct VideoSourceInner {
 }
 
 impl NativeVideoSource {
-    pub fn new(resolution: VideoResolution) -> NativeVideoSource {
+    pub fn new(resolution: VideoResolution, is_screencast: bool) -> NativeVideoSource {
         let source = Self {
-            sys_handle: vt_sys::ffi::new_video_track_source(&vt_sys::ffi::VideoResolution::from(
-                resolution.clone(),
-            )),
+            sys_handle: vt_sys::ffi::new_video_track_source(
+                &vt_sys::ffi::VideoResolution::from(resolution.clone()),
+                is_screencast,
+            ),
             inner: Arc::new(Mutex::new(VideoSourceInner { captured_frames: 0 })),
         };
 

libwebrtc/src/video_source.rs

@@ -1,4 +1,4 @@
-// Copyright 2024 LiveKit, Inc.
+// Copyright 2024-2025 LiveKit, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -58,13 +58,13 @@ pub mod native {
 
     impl Default for NativeVideoSource {
         fn default() -> Self {
-            Self::new(VideoResolution::default())
+            Self::new(VideoResolution::default(), false)
         }
     }
 
     impl NativeVideoSource {
-        pub fn new(resolution: VideoResolution) -> Self {
-            Self { handle: vs_imp::NativeVideoSource::new(resolution) }
+        pub fn new(resolution: VideoResolution, is_screencast: bool) -> Self {
+            Self { handle: vs_imp::NativeVideoSource::new(resolution, is_screencast) }
         }
 
         pub fn capture_frame<T: AsRef<dyn VideoBuffer>>(&self, frame: &VideoFrame<T>) {

livekit-ffi/protocol/video_frame.proto

@@ -1,4 +1,4 @@
-// Copyright 2023 LiveKit, Inc.
+// Copyright 2023-2025 LiveKit, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -44,11 +44,12 @@ message VideoStreamFromParticipantResponse { required OwnedVideoStream stream =
 
 // Create a new VideoSource
 // VideoSource is used to send video frame to a track
-message NewVideoSourceRequest { 
-  required VideoSourceType type = 1; 
+message NewVideoSourceRequest {
+  required VideoSourceType type = 1;
   // Used to determine which encodings to use + simulcast layers
-  // Most of the time it corresponds to the source resolution 
+  // Most of the time it corresponds to the source resolution
   required VideoSourceResolution resolution = 2;
+  optional bool is_screencast = 3;
 }
 message NewVideoSourceResponse { required OwnedVideoSource source = 1; }
 
@@ -70,7 +71,7 @@ message VideoConvertRequest {
 message VideoConvertResponse {
   oneof message {
     string error = 1;
-    OwnedVideoBuffer buffer = 2; 
+    OwnedVideoBuffer buffer = 2;
   }
 }
 
@@ -154,7 +155,7 @@ message OwnedVideoStream {
 
 message VideoStreamEvent {
   required uint64 stream_handle = 1;
-  oneof message { 
+  oneof message {
     VideoFrameReceived frame_received = 2;
     VideoStreamEOS eos = 3;
   }

livekit-ffi/src/livekit.proto.rs

@@ -302,11 +302,11 @@ impl EncryptionState {
 /// # Safety
 /// The foreign language is responsable for disposing handles
 /// Forgetting to dispose the handle may lead to memory leaks
-/// 
+///
 /// Dropping a handle doesn't necessarily mean that the object is destroyed if it is still used
 /// on the FfiServer (Atomic reference counting)
-/// 
-/// When refering to a handle without owning it, we just use a uint32 without this message. 
+///
+/// When refering to a handle without owning it, we just use a uint32 without this message.
 /// (the variable name is suffixed with "_handle")
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
@@ -1762,9 +1762,11 @@ pub struct NewVideoSourceRequest {
     #[prost(enumeration="VideoSourceType", required, tag="1")]
     pub r#type: i32,
     /// Used to determine which encodings to use + simulcast layers
-    /// Most of the time it corresponds to the source resolution 
+    /// Most of the time it corresponds to the source resolution
     #[prost(message, required, tag="2")]
     pub resolution: VideoSourceResolution,
+    #[prost(bool, optional, tag="3")]
+    pub is_screencast: ::core::option::Option<bool>,
 }
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
@@ -2007,6 +2009,8 @@ impl VideoRotation {
         }
     }
 }
+/// Values of this enum must not be changed
+/// It is used to serialize a rtc.VideoFrame on Python
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
 #[repr(i32)]
 pub enum VideoBufferType {
@@ -3451,7 +3455,7 @@ pub struct NewAudioSourceResponse {
     #[prost(message, required, tag="1")]
     pub source: OwnedAudioSource,
 }
-/// Push a frame to an AudioSource 
+/// Push a frame to an AudioSource
 /// The data provided must be available as long as the client receive the callback.
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
@@ -4005,7 +4009,7 @@ pub struct RpcMethodInvocationEvent {
 //    that it receives from the server.
 //
 // Therefore, the ffi client is easier to implement if there is less handles to manage.
-// 
+//
 // - We are mainly using FfiHandle on info messages (e.g: RoomInfo, TrackInfo, etc...)
 //    For this reason, info are only sent once, at creation (We're not using them for updates, we can infer them from
 //    events on the client implementation).

livekit-ffi/src/server/video_source.rs

@@ -1,4 +1,4 @@
-// Copyright 2023 LiveKit, Inc.
+// Copyright 2023-2025 LiveKit, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -35,8 +35,10 @@ impl FfiVideoSource {
             #[cfg(not(target_arch = "wasm32"))]
             proto::VideoSourceType::VideoSourceNative => {
                 use livekit::webrtc::video_source::native::NativeVideoSource;
-
-                let video_source = NativeVideoSource::new(new_source.resolution.into());
+                let video_source = NativeVideoSource::new(
+                    new_source.resolution.into(),
+                    new_source.is_screencast(),
+                );
                 RtcVideoSource::Native(video_source)
             }
             _ => return Err(FfiError::InvalidRequest("unsupported video source type".into())),

webrtc-sys/include/livekit/video_track.h

@@ -1,5 +1,5 @@
 /*
- * Copyright 2023 LiveKit
+ * Copyright 2023-2025 LiveKit
  *
  * Licensed under the Apache License, Version 2.0 (the “License”);
  * you may not use this file except in compliance with the License.
@@ -87,10 +87,11 @@ std::shared_ptr<NativeVideoSink> new_native_video_sink(
 class VideoTrackSource {
   class InternalSource : public rtc::AdaptedVideoTrackSource {
    public:
-    InternalSource(const VideoResolution&
-                       resolution);  // (0, 0) means no resolution/optional, the
-                                     // source will guess the resolution at the
-                                     // first captured frame
+    /*
+     * resolution set to (0, 0) means no resolution/optional, the source will
+     * guess the resolution at the first captured frame.
+     */
+    InternalSource(const VideoResolution& resolution, bool is_screencast);
     ~InternalSource() override;
 
     bool is_screencast() const override;
@@ -104,10 +105,11 @@ class VideoTrackSource {
     mutable webrtc::Mutex mutex_;
     rtc::TimestampAligner timestamp_aligner_;
     VideoResolution resolution_;
+    bool is_screencast_;
   };
 
  public:
-  VideoTrackSource(const VideoResolution& resolution);
+  VideoTrackSource(const VideoResolution& resolution, bool is_screencast);
 
   VideoResolution video_resolution() const;
 
@@ -121,7 +123,8 @@ class VideoTrackSource {
 };
 
 std::shared_ptr<VideoTrackSource> new_video_track_source(
-    const VideoResolution& resolution);
+    const VideoResolution& resolution,
+    bool is_screencast);
 
 static std::shared_ptr<MediaStreamTrack> video_to_media(
     std::shared_ptr<VideoTrack> track) {

webrtc-sys/src/video_track.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2023 LiveKit
+ * Copyright 2023-2025 LiveKit
  *
  * Licensed under the Apache License, Version 2.0 (the “License”);
  * you may not use this file except in compliance with the License.
@@ -105,13 +105,16 @@ std::shared_ptr<NativeVideoSink> new_native_video_sink(
 }
 
 VideoTrackSource::InternalSource::InternalSource(
-    const VideoResolution& resolution)
-    : rtc::AdaptedVideoTrackSource(4), resolution_(resolution) {}
+    const VideoResolution& resolution,
+    bool is_screencast)
+    : rtc::AdaptedVideoTrackSource(4),
+      resolution_(resolution),
+      is_screencast_(is_screencast) {}
 
 VideoTrackSource::InternalSource::~InternalSource() {}
 
 bool VideoTrackSource::InternalSource::is_screencast() const {
-  return false;
+  return is_screencast_;
 }
 
 absl::optional<bool> VideoTrackSource::InternalSource::needs_denoising() const {
@@ -175,8 +178,9 @@ bool VideoTrackSource::InternalSource::on_captured_frame(
   return true;
 }
 
-VideoTrackSource::VideoTrackSource(const VideoResolution& resolution) {
-  source_ = rtc::make_ref_counted<InternalSource>(resolution);
+VideoTrackSource::VideoTrackSource(const VideoResolution& resolution,
+                                   bool is_screencast) {
+  source_ = rtc::make_ref_counted<InternalSource>(resolution, is_screencast);
 }
 
 VideoResolution VideoTrackSource::video_resolution() const {
@@ -195,8 +199,9 @@ rtc::scoped_refptr<VideoTrackSource::InternalSource> VideoTrackSource::get()
 }
 
 std::shared_ptr<VideoTrackSource> new_video_track_source(
-    const VideoResolution& resolution) {
-  return std::make_shared<VideoTrackSource>(resolution);
+    const VideoResolution& resolution,
+    bool is_screencast) {
+  return std::make_shared<VideoTrackSource>(resolution, is_screencast);
 }
 
 }  // namespace livekit

webrtc-sys/src/video_track.rs

@@ -1,4 +1,4 @@
-// Copyright 2023 LiveKit, Inc.
+// Copyright 2023-2025 LiveKit, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -67,7 +67,10 @@ pub mod ffi {
 
         fn video_resolution(self: &VideoTrackSource) -> VideoResolution;
         fn on_captured_frame(self: &VideoTrackSource, frame: &UniquePtr<VideoFrame>) -> bool;
-        fn new_video_track_source(resolution: &VideoResolution) -> SharedPtr<VideoTrackSource>;
+        fn new_video_track_source(
+            resolution: &VideoResolution,
+            is_screencast: bool,
+        ) -> SharedPtr<VideoTrackSource>;
         fn video_to_media(track: SharedPtr<VideoTrack>) -> SharedPtr<MediaStreamTrack>;
         unsafe fn media_to_video(track: SharedPtr<MediaStreamTrack>) -> SharedPtr<VideoTrack>;
         fn _shared_video_track() -> SharedPtr<VideoTrack>;