Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Change consent to pending to avoid youtube tracking #2207

Closed
wants to merge 1 commit into from
Closed

Change consent to pending to avoid youtube tracking #2207

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions src/invidious.cr
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -449,7 +449,7 @@ get "/modify_notifications" do |env|
headers = HTTP::Headers.new
headers["Cookie"] = env.request.headers["Cookie"]

html = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers)
html = YT_POOL.client &.get("/subscription_manager?disable_polymer=1&ucbcb=1", headers)

cookies = HTTP::Cookies.from_client_headers(headers)
html.cookies.each do |cookie|
Expand Down Expand Up @@ -794,7 +794,7 @@ post "/data_control" do |env|
if match = channel["url"].as_s.match(/\/channel\/(?<channel>UC[a-zA-Z0-9_-]{22})/)
next match["channel"]
elsif match = channel["url"].as_s.match(/\/user\/(?<user>.+)/)
response = YT_POOL.client &.get("/user/#{match["user"]}?disable_polymer=1&hl=en&gl=US")
response = YT_POOL.client &.get("/user/#{match["user"]}?disable_polymer=1&hl=en&gl=US&ucbcb=1")
html = XML.parse_html(response.body)
ucid = html.xpath_node(%q(//link[@rel="canonical"])).try &.["href"].split("/")[-1]
next ucid if ucid
Expand Down Expand Up @@ -1591,7 +1591,7 @@ end
value = env.request.resource.split("/")[2]
body = ""
{"channel", "user", "c"}.each do |type|
response = YT_POOL.client &.get("/#{type}/#{value}/live?disable_polymer=1")
response = YT_POOL.client &.get("/#{type}/#{value}/live?disable_polymer=1&ucbcb=1")
if response.status_code == 200
body = response.body
end
Expand Down Expand Up @@ -1624,7 +1624,7 @@ get "/c/:user" do |env|

user = env.params.url["user"]

response = YT_POOL.client &.get("/c/#{user}")
response = YT_POOL.client &.get("/c/#{user}?ucbcb=1")
html = XML.parse_html(response.body)

ucid = html.xpath_node(%q(//link[@rel="canonical"])).try &.["href"].split("/")[-1]
Expand Down Expand Up @@ -3885,7 +3885,7 @@ error 404 do |env|
item = md["id"]

# Check if item is branding URL e.g. https://youtube.com/gaming
response = YT_POOL.client &.get("/#{item}")
response = YT_POOL.client &.get("/#{item}?ucbcb=1")

if response.status_code == 301
response = YT_POOL.client &.get(URI.parse(response.headers["Location"]).request_target)
Expand Down Expand Up @@ -3916,7 +3916,7 @@ error 404 do |env|
end

# Check if item is video ID
if item.match(/^[a-zA-Z0-9_-]{11}$/) && YT_POOL.client &.head("/watch?v=#{item}").status_code != 404
if item.match(/^[a-zA-Z0-9_-]{11}$/) && YT_POOL.client &.head("/watch?v=#{item}?ucbcb=1").status_code != 404
env.response.headers["Location"] = url
halt env, status_code: 302
end
Expand Down
10 changes: 5 additions & 5 deletions src/invidious/channels.cr
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -355,7 +355,7 @@ def fetch_channel_playlists(ucid, author, continuation, sort_by)
continuation = continuationItems.as_a.last["continuationItemRenderer"]?
.try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s
else
url = "/channel/#{ucid}/playlists?flow=list&view=1"
url = "/channel/#{ucid}/playlists?flow=list&view=1&ucbcb=1"

case sort_by
when "last", "last_added"
Expand Down Expand Up @@ -500,9 +500,9 @@ end

# TODO: Add "sort_by"
def fetch_channel_community(ucid, continuation, locale, format, thin_mode)
response = YT_POOL.client &.get("/channel/#{ucid}/community?gl=US&hl=en")
response = YT_POOL.client &.get("/channel/#{ucid}/community?gl=US&hl=en&ucbcb=1")
if response.status_code != 200
response = YT_POOL.client &.get("/user/#{ucid}/community?gl=US&hl=en")
response = YT_POOL.client &.get("/user/#{ucid}/community?gl=US&hl=en&ucbcb=1")
end

if response.status_code != 200
Expand Down Expand Up @@ -775,9 +775,9 @@ def extract_channel_community_cursor(continuation)
end

def get_about_info(ucid, locale)
result = YT_POOL.client &.get("/channel/#{ucid}/about?gl=US&hl=en")
result = YT_POOL.client &.get("/channel/#{ucid}/about?gl=US&hl=en&ucbcb=1")
if result.status_code != 200
result = YT_POOL.client &.get("/user/#{ucid}/about?gl=US&hl=en")
result = YT_POOL.client &.get("/user/#{ucid}/about?gl=US&hl=en&ucbcb=1")
end

if md = result.headers["location"]?.try &.match(/\/channel\/(?<ucid>UC[a-zA-Z0-9_-]{22})/)
Expand Down
2 changes: 1 addition & 1 deletion src/invidious/helpers/signatures.cr
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ struct DecryptFunction
end

private def fetch_decrypt_function(id = "CvFH_6DNRCY")
document = YT_POOL.client &.get("/watch?v=#{id}&gl=US&hl=en").body
document = YT_POOL.client &.get("/watch?v=#{id}&gl=US&hl=en&ucbcb=1").body
url = document.match(/src="(?<url>\/s\/player\/[^\/]+\/player_ias[^\/]+\/en_US\/base.js)"/).not_nil!["url"]
player = YT_POOL.client &.get(url).body

Expand Down
2 changes: 1 addition & 1 deletion src/invidious/helpers/utils.cr
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ def add_yt_headers(request)
request.headers["x-youtube-client-name"] ||= "1"
request.headers["x-youtube-client-version"] ||= "2.20200609"
# Preserve original cookies and add new YT consent cookie for EU servers
request.headers["cookie"] = "#{request.headers["cookie"]?}; CONSENT=YES+"
request.headers["cookie"] = "#{request.headers["cookie"]?}; CONSENT=PENDING+#{Random.new.rand(900)}"
Copy link
Member

@SamantazFox SamantazFox Jun 25, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Watch out, the random number should be >=100 and <= 999!
See what the peeps at NewPipe did

Copy link
Contributor

@B0pol B0pol Jun 26, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't understand how that happened but it can be over 1000, it is sometimes over 100 000

https://github.com/TeamNewPipe/NewPipeExtractor/blob/3966178979972da43f6dcbc90170058605be052b/extractor/src/test/resources/org/schabi/newpipe/extractor/kiosk/generated_mock_1.json#L10

https://github.com/TeamNewPipe/NewPipeExtractor/blob/3966178979972da43f6dcbc90170058605be052b/extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/gronkh/generated_mock_1.json#L10

and so on… (you can search in NewPipeExtractor mocks)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't understand how that happened but it can be over 1000, it is sometimes over 100 000

Oh, okay. I've never seen it being over 3 digits on youtube

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should go with Random.new.rand(100..999) for now, so we can merge this PR and change that later, if needed...

if !CONFIG.cookies.empty?
request.headers["cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}"
end
Expand Down
2 changes: 1 addition & 1 deletion src/invidious/jobs/bypass_captcha_job.cr
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob
if !random_video
random_video = {id: "zj82_v2R6ts", ucid: "UCK87Lox575O_HCHBWaBSyGA"}
end
{"/watch?v=#{random_video["id"]}&gl=US&hl=en&has_verified=1&bpctr=9999999999", produce_channel_videos_url(ucid: random_video["ucid"])}.each do |path|
{"/watch?v=#{random_video["id"]}&gl=US&hl=en&has_verified=1&bpctr=9999999999&ucbcb=1", produce_channel_videos_url(ucid: random_video["ucid"])}.each do |path|
response = YT_POOL.client &.get(path)
if response.body.includes?("To continue with your YouTube experience, please fill out the form below.")
html = XML.parse_html(response.body)
Expand Down
2 changes: 1 addition & 1 deletion src/invidious/mixes.cr
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil)
end

video_id = "CvFH_6DNRCY" if rdid.starts_with? "OLAK5uy_"
response = YT_POOL.client &.get("/watch?v=#{video_id}&list=#{rdid}&gl=US&hl=en", headers)
response = YT_POOL.client &.get("/watch?v=#{video_id}&list=#{rdid}&gl=US&hl=en&ucbcb=1", headers)
initial_data = extract_initial_data(response.body)

if !initial_data["contents"]["twoColumnWatchNextResults"]["playlist"]?
Expand Down
2 changes: 1 addition & 1 deletion src/invidious/routes/embed.cr
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ class Invidious::Routes::Embed < Invidious::Routes::BaseRoute

return env.redirect url
when "live_stream"
response = YT_POOL.client &.get("/embed/live_stream?channel=#{env.params.query["channel"]? || ""}")
response = YT_POOL.client &.get("/embed/live_stream?channel=#{env.params.query["channel"]? || ""}&ucbcb=1")
video_id = response.body.match(/"video_id":"(?<video_id>[a-zA-Z0-9_-]{11})"/).try &.["video_id"]

env.params.query.delete_all("channel")
Expand Down
6 changes: 3 additions & 3 deletions src/invidious/search.cr
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,11 +231,11 @@ end
alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist

def channel_search(query, page, channel)
response = YT_POOL.client &.get("/channel/#{channel}")
response = YT_POOL.client &.get("/channel/#{channel}?ucbcb=1")

if response.status_code == 404
response = YT_POOL.client &.get("/user/#{channel}")
response = YT_POOL.client &.get("/c/#{channel}") if response.status_code == 404
response = YT_POOL.client &.get("/user/#{channel}?ucbcb=1")
response = YT_POOL.client &.get("/c/#{channel}?ucbcb=1") if response.status_code == 404
initial_data = extract_initial_data(response.body)
ucid = initial_data["header"]["c4TabbedHeaderRenderer"]?.try &.["channelId"].as_s?
raise InfoException.new("Impossible to extract channel ID from page") if !ucid
Expand Down
4 changes: 2 additions & 2 deletions src/invidious/users.cr
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -332,7 +332,7 @@ def get_user(sid, headers, db, refresh = true)
end

def fetch_user(sid, headers, db)
feed = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers)
feed = YT_POOL.client &.get("/subscription_manager?disable_polymer=1&ucbcb=1", headers)
feed = XML.parse_html(feed.body)

channels = [] of String
Expand Down Expand Up @@ -462,7 +462,7 @@ def subscribe_ajax(channel_id, action, env_headers)
headers = HTTP::Headers.new
headers["Cookie"] = env_headers["Cookie"]

html = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers)
html = YT_POOL.client &.get("/subscription_manager?disable_polymer=1&ucbcb=1", headers)

cookies = HTTP::Cookies.from_client_headers(headers)
html.cookies.each do |cookie|
Expand Down
8 changes: 4 additions & 4 deletions src/invidious/videos.cr
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -963,7 +963,7 @@ def get_video(id, db, refresh = true, region = nil, force_refresh = false)
end

def fetch_video(id, region)
response = YT_POOL.client(region, &.get("/watch?v=#{id}&gl=US&hl=en&has_verified=1&bpctr=9999999999"))
response = YT_POOL.client(region, &.get("/watch?v=#{id}&gl=US&hl=en&has_verified=1&bpctr=9999999999&ucbcb=1"))

if md = response.headers["location"]?.try &.match(/v=(?<id>[a-zA-Z0-9_-]{11})/)
raise VideoRedirect.new(video_id: md["id"])
Expand All @@ -978,7 +978,7 @@ def fetch_video(id, region)
bypass_regions = PROXY_LIST.keys & allowed_regions
if !bypass_regions.empty?
region = bypass_regions[rand(bypass_regions.size)]
response = YT_POOL.client(region, &.get("/watch?v=#{id}&gl=US&hl=en&has_verified=1&bpctr=9999999999"))
response = YT_POOL.client(region, &.get("/watch?v=#{id}&gl=US&hl=en&has_verified=1&bpctr=9999999999&ucbcb=1"))

region_info = extract_polymer_config(response.body)
region_info["region"] = JSON::Any.new(region) if region
Expand All @@ -989,9 +989,9 @@ def fetch_video(id, region)

# Try to pull streams from embed URL
if info["reason"]?
embed_page = YT_POOL.client &.get("/embed/#{id}").body
embed_page = YT_POOL.client &.get("/embed/#{id}?ucbcb=1").body
sts = embed_page.match(/"sts"\s*:\s*(?<sts>\d+)/).try &.["sts"]? || ""
embed_info = HTTP::Params.parse(YT_POOL.client &.get("/get_video_info?html5=1&video_id=#{id}&eurl=https://youtube.googleapis.com/v/#{id}&gl=US&hl=en&sts=#{sts}").body)
embed_info = HTTP::Params.parse(YT_POOL.client &.get("/get_video_info?html5=1&video_id=#{id}&eurl=https://youtube.googleapis.com/v/#{id}&gl=US&hl=en&sts=#{sts}&ucbcb=1").body)

if embed_info["player_response"]?
player_response = JSON.parse(embed_info["player_response"])
Expand Down