Merge pull request #1272 from AudricV/yt_clients_changes_and_potokens…

…_support

[YouTube] Refactor player clients, add support for poTokens, extract visitor data from the service and more

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ClientsConstants.java

@@ -0,0 +1,118 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+final class ClientsConstants {
+    private ClientsConstants() {
+    }
+
+    // Common client fields
+
+    static final String DESKTOP_CLIENT_PLATFORM = "DESKTOP";
+    static final String MOBILE_CLIENT_PLATFORM = "MOBILE";
+    static final String WATCH_CLIENT_SCREEN = "WATCH";
+    static final String EMBED_CLIENT_SCREEN = "EMBED";
+
+    // WEB (YouTube desktop) client fields
+
+    static final String WEB_CLIENT_ID = "1";
+    static final String WEB_CLIENT_NAME = "WEB";
+    /**
+     * The client version for InnerTube requests with the {@code WEB} client, used as the last
+     * fallback if the extraction of the real one failed.
+     */
+    static final String WEB_HARDCODED_CLIENT_VERSION = "2.20250122.04.00";
+
+    // WEB_REMIX (YouTube Music) client fields
+
+    static final String WEB_REMIX_CLIENT_ID = "67";
+    static final String WEB_REMIX_CLIENT_NAME = "WEB_REMIX";
+    static final String WEB_REMIX_HARDCODED_CLIENT_VERSION = "1.20250122.01.00";
+
+    // TVHTML5 (YouTube on TVs and consoles using HTML5) client fields
+    static final String TVHTML5_CLIENT_ID = "7";
+    static final String TVHTML5_CLIENT_NAME = "TVHTML5";
+    static final String TVHTML5_CLIENT_VERSION = "7.20250122.15.00";
+    static final String TVHTML5_CLIENT_PLATFORM = "GAME_CONSOLE";
+    static final String TVHTML5_DEVICE_MAKE = "Sony";
+    static final String TVHTML5_DEVICE_MODEL_AND_OS_NAME = "PlayStation 4";
+    // CHECKSTYLE:OFF
+    static final String TVHTML5_USER_AGENT =
+            "Mozilla/5.0 (PlayStation; PlayStation 4/12.00) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.4 Safari/605.1.15";
+    // CHECKSTYLE:ON
+
+    // WEB_EMBEDDED_PLAYER (YouTube embeds)
+
+    static final String WEB_EMBEDDED_CLIENT_ID = "56";
+    static final String WEB_EMBEDDED_CLIENT_NAME = "WEB_EMBEDDED_PLAYER";
+    static final String WEB_EMBEDDED_CLIENT_VERSION = "1.20250121.00.00";
+
+    // IOS (iOS YouTube app) client fields
+
+    static final String IOS_CLIENT_ID = "5";
+    static final String IOS_CLIENT_NAME = "IOS";
+
+    /**
+     * The hardcoded client version of the iOS app used for InnerTube requests with this client.
+     *
+     * <p>
+     * It can be extracted by getting the latest release version of the app on
+     * <a href="https://apps.apple.com/us/app/youtube-watch-listen-stream/id544007664/">the App
+     * Store page of the YouTube app</a>, in the {@code What’s New} section.
+     * </p>
+     */
+    static final String IOS_CLIENT_VERSION = "20.03.02";
+
+    /**
+     * The device machine id for the iPhone 15 Pro Max, used to get 60fps with the {@code iOS}
+     * client.
+     *
+     * <p>
+     * See <a href="https://gist.github.com/adamawolf/3048717">this GitHub Gist</a> for more
+     * information.
+     * </p>
+     */
+    static final String IOS_DEVICE_MODEL = "iPhone16,2";
+
+    /**
+     * The iOS version to be used in JSON POST requests, the one of an iPhone 15 Pro Max running
+     * iOS 18.2.1 with the hardcoded version of the iOS app (for the {@code "osVersion"} field).
+     *
+     * <p>
+     * The value of this field seems to use the following structure:
+     * "iOS major version.minor version.patch version.build version", where
+     * "patch version" is equal to 0 if it isn't set
+     * The build version corresponding to the iOS version used can be found on
+     * <a href="https://theapplewiki.com/wiki/Firmware/iPhone/18.x#iPhone_15_Pro_Max">
+     *     https://theapplewiki.com/wiki/Firmware/iPhone/18.x#iPhone_15_Pro_Max</a>
+     * </p>
+     *
+     * @see #IOS_USER_AGENT_VERSION
+     */
+    static final String IOS_OS_VERSION = "18.2.1.22C161";
+
+    /**
+     * The iOS version to be used in the HTTP user agent for requests.
+     *
+     * <p>
+     * This should be the same of as {@link #IOS_OS_VERSION}.
+     * </p>
+     *
+     * @see #IOS_OS_VERSION
+     */
+    static final String IOS_USER_AGENT_VERSION = "18_2_1";
+
+    // ANDROID (Android YouTube app) client fields
+
+    static final String ANDROID_CLIENT_ID = "3";
+    static final String ANDROID_CLIENT_NAME = "ANDROID";
+
+    /**
+     * The hardcoded client version of the Android app used for InnerTube requests with this
+     * client.
+     *
+     * <p>
+     * It can be extracted by getting the latest release version of the app in an APK repository
+     * such as <a href="https://www.apkmirror.com/apk/google-inc/youtube/">APKMirror</a>.
+     * </p>
+     */
+    static final String ANDROID_CLIENT_VERSION = "19.28.35";
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/InnertubeClientRequestInfo.java

@@ -0,0 +1,148 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_ID;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_NAME;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_VERSION;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.DESKTOP_CLIENT_PLATFORM;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.EMBED_CLIENT_SCREEN;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_ID;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_NAME;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_VERSION;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_DEVICE_MODEL;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_OS_VERSION;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.MOBILE_CLIENT_PLATFORM;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_ID;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_NAME;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_PLATFORM;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_VERSION;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_DEVICE_MAKE;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_DEVICE_MODEL_AND_OS_NAME;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WATCH_CLIENT_SCREEN;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_ID;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_NAME;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_EMBEDDED_CLIENT_ID;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_EMBEDDED_CLIENT_NAME;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_HARDCODED_CLIENT_VERSION;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_HARDCODED_CLIENT_VERSION;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+// TODO: add docs
+
+public final class InnertubeClientRequestInfo {
+
+    @Nonnull
+    public ClientInfo clientInfo;
+    @Nonnull
+    public DeviceInfo deviceInfo;
+
+    public static final class ClientInfo {
+
+        @Nonnull
+        public String clientName;
+        @Nonnull
+        public String clientVersion;
+        @Nonnull
+        public String clientScreen;
+        @Nullable
+        public String clientId;
+        @Nullable
+        public String visitorData;
+
+        private ClientInfo(@Nonnull final String clientName,
+                           @Nonnull final String clientVersion,
+                           @Nonnull final String clientScreen,
+                           @Nullable final String clientId,
+                           @Nullable final String visitorData) {
+            this.clientName = clientName;
+            this.clientVersion = clientVersion;
+            this.clientScreen = clientScreen;
+            this.clientId = clientId;
+            this.visitorData = visitorData;
+        }
+    }
+
+    public static final class DeviceInfo {
+
+        @Nonnull
+        public String platform;
+        @Nullable
+        public String deviceMake;
+        @Nullable
+        public String deviceModel;
+        @Nullable
+        public String osName;
+        @Nullable
+        public String osVersion;
+        public int androidSdkVersion;
+
+        private DeviceInfo(@Nonnull final String platform,
+                           @Nullable final String deviceMake,
+                           @Nullable final String deviceModel,
+                           @Nullable final String osName,
+                           @Nullable final String osVersion,
+                           final int androidSdkVersion) {
+            this.platform = platform;
+            this.deviceMake = deviceMake;
+            this.deviceModel = deviceModel;
+            this.osName = osName;
+            this.osVersion = osVersion;
+            this.androidSdkVersion = androidSdkVersion;
+        }
+    }
+
+    private InnertubeClientRequestInfo(@Nonnull final ClientInfo clientInfo,
+                                       @Nonnull final DeviceInfo deviceInfo) {
+        this.clientInfo = clientInfo;
+        this.deviceInfo = deviceInfo;
+    }
+
+    @Nonnull
+    public static InnertubeClientRequestInfo ofWebClient() {
+        return new InnertubeClientRequestInfo(
+                new InnertubeClientRequestInfo.ClientInfo(
+                        WEB_CLIENT_NAME, WEB_HARDCODED_CLIENT_VERSION, WATCH_CLIENT_SCREEN,
+                        WEB_CLIENT_ID, null),
+                new InnertubeClientRequestInfo.DeviceInfo(DESKTOP_CLIENT_PLATFORM, null, null,
+                        null, null, -1));
+    }
+
+    @Nonnull
+    public static InnertubeClientRequestInfo ofWebEmbeddedPlayerClient() {
+        return new InnertubeClientRequestInfo(
+                new InnertubeClientRequestInfo.ClientInfo(WEB_EMBEDDED_CLIENT_NAME,
+                        WEB_REMIX_HARDCODED_CLIENT_VERSION, EMBED_CLIENT_SCREEN,
+                        WEB_EMBEDDED_CLIENT_ID, null),
+                new InnertubeClientRequestInfo.DeviceInfo(DESKTOP_CLIENT_PLATFORM, null, null,
+                        null, null, -1));
+    }
+
+    @Nonnull
+    public static InnertubeClientRequestInfo ofTvHtml5Client() {
+        return new InnertubeClientRequestInfo(
+                new InnertubeClientRequestInfo.ClientInfo(TVHTML5_CLIENT_NAME,
+                        TVHTML5_CLIENT_VERSION, WATCH_CLIENT_SCREEN, TVHTML5_CLIENT_ID, null),
+                new InnertubeClientRequestInfo.DeviceInfo(TVHTML5_CLIENT_PLATFORM,
+                        TVHTML5_DEVICE_MAKE, TVHTML5_DEVICE_MODEL_AND_OS_NAME,
+                        TVHTML5_DEVICE_MODEL_AND_OS_NAME, "", -1));
+    }
+
+    @Nonnull
+    public static InnertubeClientRequestInfo ofAndroidClient() {
+        return new InnertubeClientRequestInfo(
+                new InnertubeClientRequestInfo.ClientInfo(ANDROID_CLIENT_NAME,
+                        ANDROID_CLIENT_VERSION, WATCH_CLIENT_SCREEN, ANDROID_CLIENT_ID, null),
+                new InnertubeClientRequestInfo.DeviceInfo(MOBILE_CLIENT_PLATFORM, null, null,
+                        "Android", "15", 35));
+    }
+
+    @Nonnull
+    public static InnertubeClientRequestInfo ofIosClient() {
+        return new InnertubeClientRequestInfo(
+                new InnertubeClientRequestInfo.ClientInfo(IOS_CLIENT_NAME, IOS_CLIENT_VERSION,
+                        WATCH_CLIENT_SCREEN, IOS_CLIENT_ID, null),
+                new InnertubeClientRequestInfo.DeviceInfo(MOBILE_CLIENT_PLATFORM, "Apple",
+                        IOS_DEVICE_MODEL, "iOS", IOS_OS_VERSION, -1));
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenProvider.java

@@ -0,0 +1,120 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import javax.annotation.Nullable;
+
+/**
+ * Interface to provide {@code poToken}s to YouTube player requests.
+ *
+ * <p>
+ * On some major clients, YouTube requires that the integrity of the device passes some checks to
+ * allow playback.
+ * </p>
+ *
+ * <p>
+ * These checks involve running codes to verify the integrity and using their result to generate
+ * one or multiple {@code poToken}(s) (which stands for proof of origin token(s)).
+ * </p>
+ *
+ * <p>
+ * These tokens may have a role in triggering the sign in requirement.
+ * </p>
+ *
+ * <p>
+ * If an implementation does not want to return a {@code poToken} for a specific client, it <b>must
+ * return {@code null}</b>.
+ * </p>
+ *
+ * <p>
+ * <b>Implementations of this interface are expected to be thread-safe, as they may be accessed by
+ * multiple threads.</b>
+ * </p>
+ */
+public interface PoTokenProvider {
+
+    /**
+     * Get a {@link PoTokenResult} specific to the desktop website, a.k.a. the WEB InnerTube client.
+     *
+     * <p>
+     * To be generated and valid, {@code poToken}s from this client must be generated using Google's
+     * BotGuard machine, which requires a JavaScript engine with a good DOM implementation. They
+     * must be added to adaptive/DASH streaming URLs with the {@code pot} parameter.
+     * </p>
+     *
+     * <p>
+     * Note that YouTube desktop website generates two {@code poToken}s:
+     * - one for the player requests {@code poToken}s, using the videoId as the minter value;
+     * - one for the streaming URLs, using a visitor data for logged-out users as the minter value.
+     * </p>
+     *
+     * @return a {@link PoTokenResult} specific to the WEB InnerTube client
+     */
+    @Nullable
+    PoTokenResult getWebClientPoToken(String videoId);
+
+    /**
+     * Get a {@link PoTokenResult} specific to the web embeds, a.k.a. the WEB_EMBEDDED_PLAYER
+     * InnerTube client.
+     *
+     * <p>
+     * To be generated and valid, {@code poToken}s from this client must be generated using Google's
+     * BotGuard machine, which requires a JavaScript engine with a good DOM implementation. They
+     * should be added to adaptive/DASH streaming URLs with the {@code pot} parameter.
+     * </p>
+     *
+     * <p>
+     * As of writing, like the YouTube desktop website previously did, it generates only one
+     * {@code poToken}, sent in player requests and streaming URLs, using a visitor data for
+     * logged-out users. {@code poToken}s do not seem to be mandatory for now on this client.
+     * </p>
+     *
+     * @return a {@link PoTokenResult} specific to the WEB_EMBEDDED_PLAYER InnerTube client
+     */
+    @Nullable
+    PoTokenResult getWebEmbedClientPoToken(String videoId);
+
+    /**
+     * Get a {@link PoTokenResult} specific to the Android app, a.k.a. the ANDROID InnerTube client.
+     *
+     * <p>
+     * Implementation details are not known, the app uses DroidGuard, a downloaded native virtual
+     * machine ran by Google Play Services for which its code is updated pretty frequently.
+     * </p>
+     *
+     * <p>
+     * As of writing, DroidGuard seem to check for the Android app signature and package ID, as
+     * non-rooted YouTube patched with reVanced doesn't work without spoofing another InnerTube
+     * client while the rooted version works without any client spoofing.
+     * </p>
+     *
+     * <p>
+     * There should be only one {@code poToken} needed for the player requests, it shouldn't be
+     * required for regular adaptive URLs (i.e. not server adaptive bitrate (SABR) URLs). HLS
+     * formats returned (only for premieres and running and post-live livestreams) in the client's
+     * HLS manifest URL should work without {@code poToken}s.
+     * </p>
+     *
+     * @return a {@link PoTokenResult} specific to the ANDROID InnerTube client
+     */
+    @Nullable
+    PoTokenResult getAndroidClientPoToken(String videoId);
+
+    /**
+     * Get a {@link PoTokenResult} specific to the iOS app, a.k.a. the IOS InnerTube client.
+     *
+     * <p>
+     * Implementation details are not known, the app seem to use something called iosGuard which
+     * should be similar to Android's DroidGuard. It may rely on Apple's attestation APIs.
+     * </p>
+     *
+     * <p>
+     * As of writing, there should be only one {@code poToken} needed for the player requests, it
+     * shouldn't be required for regular adaptive URLs (i.e. not server adaptive bitrate (SABR)
+     * URLs). HLS formats returned in the client's HLS manifest URL should also work without a
+     * {@code poToken}.
+     * </p>
+     *
+     * @return a {@link PoTokenResult} specific to the IOS InnerTube client
+     */
+    @Nullable
+    PoTokenResult getIosClientPoToken(String videoId);
+}