Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Load casks from the JSON API with HOMEBREW_INSTALL_FROM_API #14304

Merged
merged 8 commits into from
Jan 6, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions Library/Homebrew/api.rb
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ module API

API_DOMAIN = "https://formulae.brew.sh/api"
HOMEBREW_CACHE_API = (HOMEBREW_CACHE/"api").freeze
MAX_RETRIES = 3

sig { params(endpoint: String, json: T::Boolean).returns(T.any(String, Hash)) }
def fetch(endpoint, json: true)
Expand All @@ -38,5 +39,24 @@ def fetch(endpoint, json: true)
rescue JSON::ParserError
raise ArgumentError, "Invalid JSON file: #{Tty.underline}#{api_url}#{Tty.reset}"
end

def fetch_json_api_file(endpoint, target:)
retry_count = 0

url = "#{API_DOMAIN}/#{endpoint}"
begin
curl_args = %W[--compressed --silent #{url}]
curl_args.prepend("--time-cond", target) if target.exist? && !target.empty?
Utils::Curl.curl_download(*curl_args, to: target, max_time: 5)

JSON.parse(target.read)
rescue JSON::ParserError
target.unlink
retry_count += 1
odie "Cannot download non-corrupt #{url}!" if retry_count > MAX_RETRIES

retry
end
end
end
end
12 changes: 12 additions & 0 deletions Library/Homebrew/api/cask.rb
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,18 @@ class << self
def fetch(name)
Homebrew::API.fetch "cask/#{name}.json"
end

sig { returns(Hash) }
def all_casks
@all_casks ||= begin
json_casks = Homebrew::API.fetch_json_api_file "cask.json",
target: HOMEBREW_CACHE_API/"cask.json"

json_casks.to_h do |json_cask|
[json_cask["token"], json_cask.except("token")]
end
end
end
end
end
end
Expand Down
35 changes: 3 additions & 32 deletions Library/Homebrew/api/formula.rb
Original file line number Diff line number Diff line change
Expand Up @@ -10,45 +10,16 @@ module Formula
class << self
extend T::Sig

MAX_RETRIES = 3

sig { returns(String) }
def formula_api_path
"formula"
end
alias generic_formula_api_path formula_api_path

sig { returns(String) }
def cached_formula_json_file
HOMEBREW_CACHE_API/"#{formula_api_path}.json"
end

sig { params(name: String).returns(Hash) }
def fetch(name)
Homebrew::API.fetch "#{formula_api_path}/#{name}.json"
Homebrew::API.fetch "formula/#{name}.json"
end

sig { returns(Hash) }
def all_formulae
@all_formulae ||= begin
retry_count = 0

url = "https://formulae.brew.sh/api/formula.json"
json_formulae = begin
curl_args = %W[--compressed --silent #{url}]
if cached_formula_json_file.exist? && !cached_formula_json_file.empty?
curl_args.prepend("--time-cond", cached_formula_json_file)
end
curl_download(*curl_args, to: cached_formula_json_file, max_time: 5)

JSON.parse(cached_formula_json_file.read)
rescue JSON::ParserError
cached_formula_json_file.unlink
retry_count += 1
odie "Cannot download non-corrupt #{url}!" if retry_count > MAX_RETRIES

retry
end
json_formulae = Homebrew::API.fetch_json_api_file "formula.json",
target: HOMEBREW_CACHE_API/"formula.json"

@all_aliases = {}
json_formulae.to_h do |json_formula|
Expand Down
100 changes: 97 additions & 3 deletions Library/Homebrew/cask/cask_loader.rb
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,100 @@ def load(config:)
end
end

# Loads a cask from the JSON API.
class FromAPILoader
attr_reader :token, :path

FLIGHT_STANZAS = [:preflight, :postflight, :uninstall_preflight, :uninstall_postflight].freeze

def self.can_load?(ref)
Homebrew::API::Cask.all_casks.key? ref
end

def initialize(token)
@token = token
@path = CaskLoader.default_path(token)
end

def load(config:)
json_cask = Homebrew::API::Cask.all_casks[token]

if (bottle_tag = ::Utils::Bottles.tag.to_s.presence) &&
(variations = json_cask["variations"].presence) &&
(variation = variations[bottle_tag].presence)
json_cask = json_cask.merge(variation)
end

json_cask.deep_symbolize_keys!

# Use the cask-source API if there are any `*flight` blocks
if json_cask[:artifacts].any? { |artifact| FLIGHT_STANZAS.include?(artifact.keys.first) }
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not for this PR: I wonder if it might be nice to push this to the API instead/as well so we have something like needs_cask_source: true in the JSON?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you think we want to remove the *flight block source that was added in #14324 from the API if we're not actually going to use it?

cask_source = Homebrew::API::CaskSource.fetch(token)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
cask_source = Homebrew::API::CaskSource.fetch(token)
cask_source = Homebrew::API::CaskSource.fetch(token, sha)

Not for this PR: I think it would be good for this API to require fetching the cask source at a particular Git SHA/revision that matches that from the JSON, for consistency and to be able to use the e.g. https://raw.githubusercontent.com/Homebrew/homebrew-cask/25e4143c41a98abb50cde6a5042e957fe5988236/Casks/0-ad.rb endpoints.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we originally opted to host our own source files on formulae.brew.sh because we felt that it would be faster than the files GitHub was hosting. Now that we're only using this method for a small number of casks, should we switch to using the GitHub-provided source URLs with hashes?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@Rylan12 Yeh, I think so. Some advantages:

  • don't need to duplicate these files
  • can ensure that we store e.g. the homebrew/cask revision in the "all casks" .json file and that we are always querying the correct/consistent version of the cask based on this revision

return FromContentLoader.new(cask_source).load(config: config)
end

Cask.new(token, source: cask_source, config: config) do
version json_cask[:version]

if json_cask[:sha256] == "no_check"
sha256 :no_check
else
sha256 json_cask[:sha256]
end

url json_cask[:url]
appcast json_cask[:appcast] if json_cask[:appcast].present?
json_cask[:name].each do |cask_name|
name cask_name
end
desc json_cask[:desc]
homepage json_cask[:homepage]

auto_updates json_cask[:auto_updates] if json_cask[:auto_updates].present?
conflicts_with(**json_cask[:conflicts_with]) if json_cask[:conflicts_with].present?

if json_cask[:depends_on].present?
dep_hash = json_cask[:depends_on].to_h do |dep_key, dep_value|
next [dep_key, dep_value] unless dep_key == :macos

dep_type = dep_value.keys.first
if dep_type == :==
version_symbols = dep_value[dep_type].map do |version|
MacOSVersions::SYMBOLS.key(version) || version
end
next [dep_key, version_symbols]
end

version_symbol = dep_value[dep_type].first
version_symbol = MacOSVersions::SYMBOLS.key(version_symbol) || version_symbol
[dep_key, "#{dep_type} :#{version_symbol}"]
end.compact
depends_on(**dep_hash)
end

if json_cask[:container].present?
container_hash = json_cask[:container].to_h do |container_key, container_value|
next [container_key, container_value] unless container_key == :type

[container_key, container_value.to_sym]
end
container(**container_hash)
end

json_cask[:artifacts].each do |artifact|
key = artifact.keys.first
if FLIGHT_STANZAS.include?(key)
instance_eval(artifact[key])
else
send(key, *artifact[key])
end
end

caveats json_cask[:caveats] if json_cask[:caveats].present?
end
end
end

# Pseudo-loader which raises an error when trying to load the corresponding cask.
class NullLoader < FromPathLoader
extend T::Sig
Expand Down Expand Up @@ -225,15 +319,15 @@ def self.for(ref, need_path: false)
next unless loader_class.can_load?(ref)

if loader_class == FromTapLoader && Homebrew::EnvConfig.install_from_api? &&
ref.start_with?("homebrew/cask/") && Homebrew::API::CaskSource.available?(ref)
return FromContentLoader.new(Homebrew::API::CaskSource.fetch(ref))
ref.start_with?("homebrew/cask/") && FromAPILoader.can_load?(ref)
return FromAPILoader.new(ref)
end

return loader_class.new(ref)
end

if Homebrew::EnvConfig.install_from_api? && !need_path && Homebrew::API::CaskSource.available?(ref)
return FromContentLoader.new(Homebrew::API::CaskSource.fetch(ref))
return FromAPILoader.new(ref)
end

return FromTapPathLoader.new(default_path(ref)) if FromTapPathLoader.can_load?(default_path(ref))
Expand Down
44 changes: 24 additions & 20 deletions Library/Homebrew/cmd/update.sh
Original file line number Diff line number Diff line change
Expand Up @@ -754,28 +754,32 @@ EOS
if [[ -n "${HOMEBREW_INSTALL_FROM_API}" ]]
then
mkdir -p "${HOMEBREW_CACHE}/api"
if [[ -f "${HOMEBREW_CACHE}/api/formula.json" ]]
then
INITIAL_JSON_BYTESIZE="$(wc -c "${HOMEBREW_CACHE}"/api/formula.json)"
fi
curl \
"${CURL_DISABLE_CURLRC_ARGS[@]}" \
--fail --compressed --silent --max-time 5 \
--location --remote-time --output "${HOMEBREW_CACHE}/api/formula.json" \
--time-cond "${HOMEBREW_CACHE}/api/formula.json" \
--user-agent "${HOMEBREW_USER_AGENT_CURL}" \
"https://formulae.brew.sh/api/formula.json"
curl_exit_code=$?
if [[ ${curl_exit_code} -eq 0 ]]
then
CURRENT_JSON_BYTESIZE="$(wc -c "${HOMEBREW_CACHE}"/api/formula.json)"
if [[ "${INITIAL_JSON_BYTESIZE}" != "${CURRENT_JSON_BYTESIZE}" ]]

for formula_or_cask in formula cask
do
if [[ -f "${HOMEBREW_CACHE}/api/${formula_or_cask}.json" ]]
then
HOMEBREW_UPDATED="1"
INITIAL_JSON_BYTESIZE="$(wc -c "${HOMEBREW_CACHE}"/api/"${formula_or_cask}".json)"
fi
else
echo "Failed to download formula.json!" >>"${update_failed_file}"
fi
curl \
"${CURL_DISABLE_CURLRC_ARGS[@]}" \
--fail --compressed --silent --max-time 5 \
--location --remote-time --output "${HOMEBREW_CACHE}/api/${formula_or_cask}.json" \
--time-cond "${HOMEBREW_CACHE}/api/${formula_or_cask}.json" \
--user-agent "${HOMEBREW_USER_AGENT_CURL}" \
"https://formulae.brew.sh/api/${formula_or_cask}.json"
curl_exit_code=$?
if [[ ${curl_exit_code} -eq 0 ]]
then
CURRENT_JSON_BYTESIZE="$(wc -c "${HOMEBREW_CACHE}"/api/"${formula_or_cask}".json)"
if [[ "${INITIAL_JSON_BYTESIZE}" != "${CURRENT_JSON_BYTESIZE}" ]]
then
HOMEBREW_UPDATED="1"
fi
else
echo "Failed to download ${formula_or_cask}.json!" >>"${update_failed_file}"
fi
done
fi

safe_cd "${HOMEBREW_REPOSITORY}"
Expand Down
9 changes: 4 additions & 5 deletions Library/Homebrew/dev-cmd/bottle.rb
Original file line number Diff line number Diff line change
Expand Up @@ -557,11 +557,10 @@ def bottle_formula(f, args:)
"date" => Pathname(filename.to_s).mtime.strftime("%F"),
"tags" => {
bottle_tag.to_s => {
"filename" => filename.url_encode,
"local_filename" => filename.to_s,
"sha256" => sha256,
"formulae_brew_sh_path" => Homebrew::API::Formula.formula_api_path,
"tab" => tab.to_bottle_hash,
"filename" => filename.url_encode,
"local_filename" => filename.to_s,
"sha256" => sha256,
"tab" => tab.to_bottle_hash,
},
},
},
Expand Down
3 changes: 1 addition & 2 deletions Library/Homebrew/github_packages.rb
Original file line number Diff line number Diff line change
Expand Up @@ -354,8 +354,7 @@ def upload_bottle(user, token, skopeo, formula_full_name, bottle_hash, keep_old:

config_json_sha256, config_json_size = write_image_config(platform_hash, tar_sha256.hexdigest, blobs)

formulae_dir = tag_hash["formulae_brew_sh_path"]
documentation = "https://formulae.brew.sh/#{formulae_dir}/#{formula_name}" if formula_core_tap
documentation = "https://formulae.brew.sh/formula/#{formula_name}" if formula_core_tap

descriptor_annotations_hash = {
"org.opencontainers.image.ref.name" => tag,
Expand Down
44 changes: 44 additions & 0 deletions Library/Homebrew/test/api/cask_spec.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
# typed: false
# frozen_string_literal: true

require "api"

describe Homebrew::API::Cask do
let(:cache_dir) { mktmpdir }

before do
stub_const("Homebrew::API::HOMEBREW_CACHE_API", cache_dir)
end

def mock_curl_download(stdout:)
allow(Utils::Curl).to receive(:curl_download) do |*_args, **kwargs|
kwargs[:to].write stdout
end
end

describe "::all_casks" do
let(:casks_json) {
<<~EOS
[{
"token": "foo",
"url": "https://brew.sh/foo"
}, {
"token": "bar",
"url": "https://brew.sh/bar"
}]
EOS
}
let(:casks_hash) {
{
"foo" => { "url" => "https://brew.sh/foo" },
"bar" => { "url" => "https://brew.sh/bar" },
}
}

it "returns the expected cask JSON list" do
mock_curl_download stdout: casks_json
casks_output = described_class.all_casks
expect(casks_output).to eq casks_hash
end
end
end
Loading