pr-upload: Upload bottles to the Internet Archive #10677
+278
−14
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
| @@ -0,0 +1,177 @@ | ||||||
| # typed: false | ||||||
| # frozen_string_literal: true | ||||||
|
|
||||||
| require "digest/md5" | ||||||
| require "utils/curl" | ||||||
|
|
||||||
| # The Internet Archive API client. | ||||||
| # | ||||||
| # @api private | ||||||
| class Archive | ||||||
| extend T::Sig | ||||||
|
|
||||||
| include Context | ||||||
| include Utils::Curl | ||||||
|
|
||||||
| class Error < RuntimeError | ||||||
| end | ||||||
|
|
||||||
| sig { returns(String) } | ||||||
| def inspect | ||||||
| "#<Archive: item=#{@archive_item}>" | ||||||
| end | ||||||
|
|
||||||
| sig { params(item: T.nilable(String)).void } | ||||||
| def initialize(item: "homebrew") | ||||||
| raise UsageError, "Must set the Archive item!" unless item | ||||||
|
|
||||||
| @archive_item = item | ||||||
| end | ||||||
|
|
||||||
| def open_api(url, *args, auth: true) | ||||||
| if auth | ||||||
| key = Homebrew::EnvConfig.internet_archive_key | ||||||
| raise UsageError, "HOMEBREW_INTERNET_ARCHIVE_KEY is unset." if key.blank? | ||||||
|
|
||||||
| if key.exclude?(":") | ||||||
| raise UsageError, "Use HOMEBREW_INTERNET_ARCHIVE_KEY=access:secret. See https://archive.org/account/s3.php" | ||||||
| end | ||||||
|
|
||||||
| args += ["--header", "Authorization: AWS #{key}"] | ||||||
| end | ||||||
|
|
||||||
| curl(*args, url, print_stdout: false, secrets: key) | ||||||
| end | ||||||
|
|
||||||
| sig { | ||||||
| params(local_file: String, | ||||||
| directory: String, | ||||||
| remote_file: String, | ||||||
| warn_on_error: T.nilable(T::Boolean)).void | ||||||
| } | ||||||
| def upload(local_file, directory:, remote_file:, warn_on_error: false) | ||||||
| local_file = Pathname.new(local_file) | ||||||
| unless local_file.exist? | ||||||
| msg = "#{local_file} for upload doesn't exist!" | ||||||
| raise Error, msg unless warn_on_error | ||||||
|
|
||||||
| # Warn and return early here since we know this upload is going to fail. | ||||||
| opoo msg | ||||||
| return | ||||||
| end | ||||||
|
|
||||||
| md5_base64 = Digest::MD5.base64digest(local_file.read) | ||||||
| url = "https://#{@archive_item}.s3.us.archive.org/#{directory}/#{remote_file}" | ||||||
|
There was a problem hiding this comment. Would be good to make |
||||||
| args = ["--upload-file", local_file, "--header", "Content-MD5: #{md5_base64}"] | ||||||
| args << "--fail" unless warn_on_error | ||||||
| result = T.unsafe(self).open_api(url, *args) | ||||||
| return if result.success? && result.stdout.exclude?("Error") | ||||||
|
|
||||||
| msg = "Bottle upload failed: #{result.stdout}" | ||||||
| raise msg unless warn_on_error | ||||||
|
|
||||||
| opoo msg | ||||||
| end | ||||||
|
|
||||||
| sig { | ||||||
| params(formula: Formula, | ||||||
| directory: String, | ||||||
| warn_on_error: T::Boolean).returns(String) | ||||||
| } | ||||||
| def mirror_formula(formula, directory: "mirror", warn_on_error: false) | ||||||
| formula.downloader.fetch | ||||||
|
|
||||||
| filename = ERB::Util.url_encode(formula.downloader.basename) | ||||||
| destination_url = "https://archive.org/download/#{@archive_item}/#{directory}/#{filename}" | ||||||
|
|
||||||
| odebug "Uploading to #{destination_url}" | ||||||
|
|
||||||
| upload( | ||||||
| formula.downloader.cached_location, | ||||||
| directory: directory, | ||||||
| remote_file: filename, | ||||||
| warn_on_error: warn_on_error, | ||||||
| ) | ||||||
|
|
||||||
| destination_url | ||||||
| end | ||||||
|
|
||||||
| # Gets the MD5 hash of the specified remote file. | ||||||
| # | ||||||
| # @return the hash, the empty string (if the file doesn't have a hash), nil (if the file doesn't exist) | ||||||
| sig { params(directory: String, remote_file: String).returns(T.nilable(String)) } | ||||||
| def remote_md5(directory:, remote_file:) | ||||||
| url = "https://#{@archive_item}.s3.us.archive.org/#{directory}/#{remote_file}" | ||||||
| result = curl_output "--fail", "--silent", "--head", "--location", url | ||||||
| if result.success? | ||||||
| result.stdout.match(/^ETag: "(\h{32})"/)&.values_at(1)&.first || "" | ||||||
| else | ||||||
| raise Error if result.status.exitstatus != 22 && result.stderr.exclude?("404 Not Found") | ||||||
|
|
||||||
| nil | ||||||
| end | ||||||
| end | ||||||
|
|
||||||
| sig { params(directory: String, filename: String).returns(String) } | ||||||
| def file_delete_instructions(directory, filename) | ||||||
| <<~EOS | ||||||
| Run: | ||||||
| curl -X DELETE -H "Authorization: AWS $HOMEBREW_INTERNET_ARCHIVE_KEY" https://#{@archive_item}.s3.us.archive.org/#{directory}/#{filename} | ||||||
| Or run: | ||||||
| ia delete #{@archive_item} #{directory}/#{filename} | ||||||
| EOS | ||||||
| end | ||||||
|
|
||||||
| sig { | ||||||
| params(bottles_hash: T::Hash[String, T.untyped], | ||||||
| warn_on_error: T.nilable(T::Boolean)).void | ||||||
| } | ||||||
| def upload_bottles(bottles_hash, warn_on_error: false) | ||||||
| bottles_hash.each do |_formula_name, bottle_hash| | ||||||
|
There was a problem hiding this comment.
Suggested change
|
||||||
| directory = bottle_hash["bintray"]["repository"] | ||||||
| bottle_count = bottle_hash["bottle"]["tags"].length | ||||||
|
|
||||||
| bottle_hash["bottle"]["tags"].each_value do |tag_hash| | ||||||
| filename = tag_hash["filename"] # URL encoded in Bottle::Filename#archive | ||||||
| delete_instructions = file_delete_instructions(directory, filename) | ||||||
|
|
||||||
| local_filename = tag_hash["local_filename"] | ||||||
| md5 = Digest::MD5.hexdigest(File.read(local_filename)) | ||||||
|
|
||||||
| odebug "Checking remote file #{@archive_item}/#{directory}/#{filename}" | ||||||
| result = remote_md5(directory: directory, remote_file: filename) | ||||||
| case result | ||||||
| when nil | ||||||
| # File doesn't exist. | ||||||
| odebug "Uploading #{@archive_item}/#{directory}/#{filename}" | ||||||
| upload(local_filename, | ||||||
| directory: directory, | ||||||
| remote_file: filename, | ||||||
| warn_on_error: warn_on_error) | ||||||
| when md5 | ||||||
| # File exists, hash matches. | ||||||
| odebug "#{filename} is already published with matching hash." | ||||||
| bottle_count -= 1 | ||||||
| when "" | ||||||
| # File exists, but can't find hash | ||||||
| failed_message = "#{filename} is already published!" | ||||||
| raise Error, "#{failed_message}\n#{delete_instructions}" unless warn_on_error | ||||||
|
|
||||||
| opoo failed_message | ||||||
| else | ||||||
| # File exists, but hash either doesn't exist or is mismatched. | ||||||
| failed_message = <<~EOS | ||||||
| #{filename} is already published with a mismatched hash! | ||||||
| Expected: #{md5} | ||||||
| Actual: #{result} | ||||||
| EOS | ||||||
| raise Error, "#{failed_message}#{delete_instructions}" unless warn_on_error | ||||||
|
|
||||||
| opoo failed_message | ||||||
| end | ||||||
| end | ||||||
|
|
||||||
| odebug "Uploaded #{bottle_count} bottles" | ||||||
| end | ||||||
| end | ||||||
| end | ||||||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| # typed: false | ||
| # frozen_string_literal: true | ||
|
|
||
| require "archive" | ||
|
|
||
| describe Archive, :needs_network do | ||
| subject(:archive) { described_class.new(item: "homebrew") } | ||
|
|
||
| describe "::remote_checksum" do | ||
| it "detects a published file" do | ||
| hash = archive.remote_md5(directory: ".", remote_file: "cmake-3.1.2.yosemite.bottle.tar.gz") | ||
| expect(hash).to eq("c6e525d472124670b0b635800488f438") | ||
| end | ||
|
|
||
| it "fails on a non-existent file" do | ||
| hash = archive.remote_md5(directory: "bottles", remote_file: "my-fake-bottle-1.0.snow_hyena.tar.gz") | ||
| expect(hash).to be nil | ||
| end | ||
| end | ||
| end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Would be good to get some tests for this class.