From 7b23bc64e5432aa2f754b7620ce204956a661ded Mon Sep 17 00:00:00 2001 From: Sam Ford <1584702+samford@users.noreply.github.com> Date: Wed, 25 May 2022 13:31:54 -0400 Subject: [PATCH 1/2] Curl: Rename :status to :status_code The return hash from `#curl_http_content_headers_and_checksum` contains a `:status`, which is the status code of the last response. This string value comes from `#parse_curl_response`, where the key is `:status_code` instead. Aligning these keys technically allows us to pass either of these hashes to the `#url_protected_by_*` methods, as both contain `:status_code` and `:headers` in the expected format. --- Library/Homebrew/test/utils/curl_spec.rb | 6 +++--- Library/Homebrew/utils/curl.rb | 18 +++++++++--------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Library/Homebrew/test/utils/curl_spec.rb b/Library/Homebrew/test/utils/curl_spec.rb index 06a2d7e8bd6d2..2f38fa54f9f56 100644 --- a/Library/Homebrew/test/utils/curl_spec.rb +++ b/Library/Homebrew/test/utils/curl_spec.rb @@ -14,7 +14,7 @@ details[:normal][:no_cookie] = { url: "https://www.example.com/", final_url: nil, - status: "403", + status_code: "403", headers: { "age" => "123456", "cache-control" => "max-age=604800", @@ -35,7 +35,7 @@ } details[:normal][:ok] = Marshal.load(Marshal.dump(details[:normal][:no_cookie])) - details[:normal][:ok][:status] = "200" + details[:normal][:ok][:status_code] = "200" details[:normal][:single_cookie] = Marshal.load(Marshal.dump(details[:normal][:no_cookie])) details[:normal][:single_cookie][:headers]["set-cookie"] = "a_cookie=for_testing" @@ -52,7 +52,7 @@ details[:cloudflare][:single_cookie] = { url: "https://www.example.com/", final_url: nil, - status: "403", + status_code: "403", headers: { "date" => "Wed, 1 Jan 2020 01:23:45 GMT", "content-type" => "text/plain; charset=UTF-8", diff --git a/Library/Homebrew/utils/curl.rb b/Library/Homebrew/utils/curl.rb index ded6c287201cb..6e1d26e412872 100644 --- a/Library/Homebrew/utils/curl.rb +++ b/Library/Homebrew/utils/curl.rb @@ -205,7 +205,7 @@ def curl_output(*args, **options) sig { params(details: T::Hash[Symbol, T.untyped]).returns(T::Boolean) } def url_protected_by_cloudflare?(details) return false if details[:headers].blank? - return false unless [403, 503].include?(details[:status].to_i) + return false unless [403, 503].include?(details[:status_code].to_i) set_cookie_header = Array(details[:headers]["set-cookie"]) has_cloudflare_cookie_header = set_cookie_header.compact.any? do |cookie| @@ -228,7 +228,7 @@ def url_protected_by_cloudflare?(details) sig { params(details: T::Hash[Symbol, T.untyped]).returns(T::Boolean) } def url_protected_by_incapsula?(details) return false if details[:headers].blank? - return false if details[:status].to_i != 403 + return false if details[:status_code].to_i != 403 set_cookie_header = Array(details[:headers]["set-cookie"]) set_cookie_header.compact.any? { |cookie| cookie.match?(/^(visid_incap|incap_ses)_/i) } @@ -255,7 +255,7 @@ def curl_check_http_content(url, url_type, specs: {}, user_agents: [:default], next end - next unless http_status_ok?(secure_details[:status]) + next unless http_status_ok?(secure_details[:status_code]) hash_needed = true user_agents = [user_agent] @@ -273,20 +273,20 @@ def curl_check_http_content(url, url_type, specs: {}, user_agents: [:default], use_homebrew_curl: use_homebrew_curl, user_agent: user_agent, ) - break if http_status_ok?(details[:status]) + break if http_status_ok?(details[:status_code]) end - unless details[:status] + unless details[:status_code] # Hack around https://github.com/Homebrew/brew/issues/3199 return if MacOS.version == :el_capitan return "The #{url_type} #{url} is not reachable" end - unless http_status_ok?(details[:status]) + unless http_status_ok?(details[:status_code]) return if url_protected_by_cloudflare?(details) || url_protected_by_incapsula?(details) - return "The #{url_type} #{url} is not reachable (HTTP status code #{details[:status]})" + return "The #{url_type} #{url} is not reachable (HTTP status code #{details[:status_code]})" end if url.start_with?("https://") && Homebrew::EnvConfig.no_insecure_redirect? && @@ -296,7 +296,7 @@ def curl_check_http_content(url, url_type, specs: {}, user_agents: [:default], return unless secure_details - return if !http_status_ok?(details[:status]) || !http_status_ok?(secure_details[:status]) + return if !http_status_ok?(details[:status_code]) || !http_status_ok?(secure_details[:status_code]) etag_match = details[:etag] && details[:etag] == secure_details[:etag] @@ -397,7 +397,7 @@ def curl_http_content_headers_and_checksum( { url: url, final_url: final_url, - status: status_code, + status_code: status_code, headers: headers, etag: etag, content_length: content_length, From 403a4d4a494643d0ecfd91f92812a6f5bfa9a9ff Mon Sep 17 00:00:00 2001 From: Sam Ford <1584702+samford@users.noreply.github.com> Date: Wed, 25 May 2022 13:45:31 -0400 Subject: [PATCH 2/2] Curl: Check all responses for protected cookies The response from a URL protected by Cloudflare may only provide a relevant cookie on the first response but `#curl_http_content_headers_and_checksum` only returns the headers of the final response. In this scenario, `#curl_check_http_content` isn't able to properly detect the protected URL and this is surfaced as an error instead of skipping the URL. This resolves the issue by including the array of response hashes in the return value from `#curl_http_content_headers_and_checksum`, so we can check all the responses in `#curl_check_http_content`. --- Library/Homebrew/utils/curl.rb | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/Library/Homebrew/utils/curl.rb b/Library/Homebrew/utils/curl.rb index 6e1d26e412872..5b094f7f07a63 100644 --- a/Library/Homebrew/utils/curl.rb +++ b/Library/Homebrew/utils/curl.rb @@ -198,21 +198,20 @@ def curl_output(*args, **options) end # Check if a URL is protected by CloudFlare (e.g. badlion.net and jaxx.io). - # @param details [Hash] Response information from - # `#curl_http_content_headers_and_checksum`. + # @param response [Hash] A response hash from `#parse_curl_response`. # @return [true, false] Whether a response contains headers indicating that # the URL is protected by Cloudflare. - sig { params(details: T::Hash[Symbol, T.untyped]).returns(T::Boolean) } - def url_protected_by_cloudflare?(details) - return false if details[:headers].blank? - return false unless [403, 503].include?(details[:status_code].to_i) + sig { params(response: T::Hash[Symbol, T.untyped]).returns(T::Boolean) } + def url_protected_by_cloudflare?(response) + return false if response[:headers].blank? + return false unless [403, 503].include?(response[:status_code].to_i) - set_cookie_header = Array(details[:headers]["set-cookie"]) + set_cookie_header = Array(response[:headers]["set-cookie"]) has_cloudflare_cookie_header = set_cookie_header.compact.any? do |cookie| cookie.match?(/^(__cfduid|__cf_bm)=/i) end - server_header = Array(details[:headers]["server"]) + server_header = Array(response[:headers]["server"]) has_cloudflare_server = server_header.compact.any? do |server| server.match?(/^cloudflare/i) end @@ -221,16 +220,15 @@ def url_protected_by_cloudflare?(details) end # Check if a URL is protected by Incapsula (e.g. corsair.com). - # @param details [Hash] Response information from - # `#curl_http_content_headers_and_checksum`. + # @param response [Hash] A response hash from `#parse_curl_response`. # @return [true, false] Whether a response contains headers indicating that # the URL is protected by Incapsula. - sig { params(details: T::Hash[Symbol, T.untyped]).returns(T::Boolean) } - def url_protected_by_incapsula?(details) - return false if details[:headers].blank? - return false if details[:status_code].to_i != 403 + sig { params(response: T::Hash[Symbol, T.untyped]).returns(T::Boolean) } + def url_protected_by_incapsula?(response) + return false if response[:headers].blank? + return false if response[:status_code].to_i != 403 - set_cookie_header = Array(details[:headers]["set-cookie"]) + set_cookie_header = Array(response[:headers]["set-cookie"]) set_cookie_header.compact.any? { |cookie| cookie.match?(/^(visid_incap|incap_ses)_/i) } end @@ -284,7 +282,9 @@ def curl_check_http_content(url, url_type, specs: {}, user_agents: [:default], end unless http_status_ok?(details[:status_code]) - return if url_protected_by_cloudflare?(details) || url_protected_by_incapsula?(details) + return if details[:responses].any? do |response| + url_protected_by_cloudflare?(response) || url_protected_by_incapsula?(response) + end return "The #{url_type} #{url} is not reachable (HTTP status code #{details[:status_code]})" end @@ -403,6 +403,7 @@ def curl_http_content_headers_and_checksum( content_length: content_length, file: file_contents, file_hash: file_hash, + responses: responses, } ensure file.unlink