From ed37c86a39456e268d9f91ad98a4563beb4688a7 Mon Sep 17 00:00:00 2001 From: Maxim Samsonov Date: Tue, 9 Jan 2024 19:28:41 +0300 Subject: [PATCH] Implemented rubygems update to ensure correct linux-gnu/linux-musl differentiation by bundler #120 --- .github/workflows/gem-test-and-release.yml | 1 - CMakeLists.txt | 18 +-- common.env | 1 - exe/tebako-packager | 15 ++- lib/tebako/cli_helpers.rb | 3 +- lib/tebako/packager.rb | 121 ++++++++++----------- lib/tebako/packager/patch_helpers.rb | 50 ++++++++- lib/tebako/version.rb | 2 +- tests-2/tebako-test.rb | 2 +- tests/scripts/functional-tests.sh | 2 +- tests/test-01/tebako-test-run.rb | 1 + 11 files changed, 126 insertions(+), 90 deletions(-) diff --git a/.github/workflows/gem-test-and-release.yml b/.github/workflows/gem-test-and-release.yml index afc8d33f..140b086d 100644 --- a/.github/workflows/gem-test-and-release.yml +++ b/.github/workflows/gem-test-and-release.yml @@ -74,7 +74,6 @@ jobs: uses: ruby/setup-ruby@v1 with: ruby-version: ${{ env.RUBY_VER }} - bundler: ${{ env.BUNDLER_VER }} bundler-cache: true - name: Build gem diff --git a/CMakeLists.txt b/CMakeLists.txt index dea799c7..e47a890d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2023 [Ribose Inc](https://www.ribose.com). +# Copyright (c) 2021-2024 [Ribose Inc](https://www.ribose.com). # All rights reserved. # This file is a part of tebako # @@ -245,14 +245,6 @@ endif(IS_MSYS) def_ext_prj_g(INCBIN "348e36b") def_ext_prj_g(DWARFS_WR "v0.4.0") -if (DEFINED ENV{BUNDLER_VER}) - set(BUNDLER_VER $ENV{BUNDLER_VER}) - set(BUNDLER_ANNOTATION "environment") -else() - set(BUNDLER_VER "2.3.22") - set(BUNDLER_ANNOTATION "default") -endif() - find_library(_LIBNCURSES "libncurses.a") if(${_LIBNCURSES} STREQUAL "_LIBNCURSES-NOTFOUND") set(WITH_NCURSES_BUILD ON) @@ -268,7 +260,6 @@ endif(${RUBY_VER} VERSION_LESS "3.2.0") message("Configuration summary:") message(STATUS "ruby: v${RUBY_VER} at ${RUBY_SOURCE_DIR}") -message(STATUS "bundler version: ${BUNDLER_VER} (${BUNDLER_ANNOTATION})") if(WITH_OPENSSL_BUILD) message(STATUS "openssl: building @${OPENSSL_TAG} at ${OPENSSL_SOURCE_DIR}") endif(WITH_OPENSSL_BUILD) @@ -511,7 +502,7 @@ else (${SETUP_MODE}) list(LENGTH GEMS GLENGTH) add_custom_target(clean_filesystem - COMMAND ruby ${EXE}/tebako-packager deploy ${RUBY_STASH_DIR} ${DATA_SRC_DIR} ${DATA_PRE_DIR} ${DATA_BIN_DIR} ${TBD} ${TGD} + COMMAND ruby ${EXE}/tebako-packager deploy ${RUBY_STASH_DIR} ${DATA_SRC_DIR} ${DATA_PRE_DIR} ${DATA_BIN_DIR} ${GFLENGTH} DEPENDS ${RUBY_PRJ} ) if(GSLENGTH GREATER 0) @@ -527,8 +518,6 @@ else (${SETUP_MODE}) add_custom_target(source_filesystem COMMAND ${CMAKE_COMMAND} -DSOURCE_DIR=${FS_ROOT} -DTARGET_DIR=${DATA_PRE_DIR} -P ${CMAKE_SOURCE_DIR}/cmake/copy_dir.cmake COMMAND ${CMAKE_COMMAND} -E make_directory ${TGD} - COMMAND ${CMAKE_COMMAND} -E chdir ${DATA_PRE_DIR} ${CMAKE_COMMAND} -E env --unset=GEM_HOME --unset=GEM_PATH TEBAKO_PASS_THROUGH=1 - ${TBD}/gem${CMD_SUFFIX} install bundler -v '${BUNDLER_VER}' --source 'https://rubygems.org/' --no-document --install-dir ${TGD} COMMAND ${CMAKE_COMMAND} -E chdir ${DATA_PRE_DIR} ${CMAKE_COMMAND} -E env --unset=GEM_HOME --unset=GEM_PATH TEBAKO_PASS_THROUGH=1 ${TBD}/bundle${CMD_SUFFIX} config set --local force_ruby_platform ${FORCE_RUBY_PLATFORM} COMMAND ${CMAKE_COMMAND} -E chdir ${DATA_PRE_DIR} ${CMAKE_COMMAND} -E env --unset=GEM_HOME --unset=GEM_PATH TEBAKO_PASS_THROUGH=1 @@ -572,9 +561,6 @@ else (${SETUP_MODE}) ${CMAKE_COMMAND} -E false ) COMMAND ${CMAKE_COMMAND} -E make_directory ${TLD} COMMAND ${CMAKE_COMMAND} -DSOURCE_DIR=${FS_ROOT} -DTARGET_DIR=${TLD} -P ${CMAKE_SOURCE_DIR}/cmake/copy_dir.cmake - COMMAND ${CMAKE_COMMAND} -E chdir ${TLD} ${CMAKE_COMMAND} -E env --unset=GEM_HOME --unset=GEM_PATH TEBAKO_PASS_THROUGH=1 - ${TBD}/gem${CMD_SUFFIX} install bundler -v '${BUNDLER_VER}' - --source 'https://rubygems.org/' --no-document --install-dir ${TGD} COMMAND ${CMAKE_COMMAND} -E chdir ${TLD} ${CMAKE_COMMAND} -E env --unset=GEM_HOME --unset=GEM_PATH TEBAKO_PASS_THROUGH=1 ${TBD}/bundle${CMD_SUFFIX} config build.ffi --disable-system-libffi COMMAND ${CMAKE_COMMAND} -E chdir ${TLD} ${CMAKE_COMMAND} -E env --unset=GEM_HOME --unset=GEM_PATH TEBAKO_PASS_THROUGH=1 diff --git a/common.env b/common.env index e21471b1..906fe35a 100644 --- a/common.env +++ b/common.env @@ -2,5 +2,4 @@ BUILD_TYPE=Release DEPS=deps INCBIN_TAG=348e36b DWARFS_WR_TAG=v0.4.0 -BUNDLER_VER=2.3.22 RUBY_VER=3.1.4 diff --git a/exe/tebako-packager b/exe/tebako-packager index ba6a1932..3c7bc092 100755 --- a/exe/tebako-packager +++ b/exe/tebako-packager @@ -1,7 +1,7 @@ #!/usr/bin/env ruby # frozen_string_literal: true -# Copyright (c) 2023 [Ribose Inc](https://www.ribose.com). +# Copyright (c) 2023-2024 [Ribose Inc](https://www.ribose.com). # All rights reserved. # This file is a part of tebako # @@ -78,13 +78,16 @@ begin # ARGV[2] -- DATA_SRC_DIR # ARGV[3] -- DATA_PRE_DIR # ARGV[4] -- DATA_BIN_DIR - # ARGV[5] -- TARGET_BIN_DIR (TBD) - # ARGV[6] -- TARGET_GEM_DIR (TGD) - unless ARGV.length == 7 + # ARGV[5] -- GFLENGTH + unless ARGV.length == 6 raise Tebako::Error, - "tebako-packager deploy command expects 7 arguments, #{ARGV.length} has been provided." + "tebako-packager deploy command expects 6 arguments, #{ARGV.length} has been provided." end - Tebako::Packager.deploy(ARGV[1], ARGV[2], ARGV[3], ARGV[4], ARGV[5]) + Tebako::Packager.init(ARGV[1], ARGV[2], ARGV[3], ARGV[4]) + # Assume that "" is /bin" + # That shall match CMakeLists.txt settings + # Tebako::Packager.update("#{ARGV[1]}/bin", "2.7.7") + Tebako::Packager.deploy(ARGV[2], "#{ARGV[2]}/bin", ARGV[5]) else raise Tebako::Error, "tebako-packager cannot process #{ARGV[0]} command" end diff --git a/lib/tebako/cli_helpers.rb b/lib/tebako/cli_helpers.rb index 6c0cb84a..3e746ae2 100644 --- a/lib/tebako/cli_helpers.rb +++ b/lib/tebako/cli_helpers.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -# Copyright (c) 2023 [Ribose Inc](https://www.ribose.com). +# Copyright (c) 2023-2024 [Ribose Inc](https://www.ribose.com). # All rights reserved. # This file is a part of tebako # @@ -130,6 +130,7 @@ def packaging_error(code) def prefix @prefix ||= if options["prefix"].nil? + puts "No prefix specified, using ~/.tebako" File.expand_path("~/.tebako") elsif options["prefix"] == "PWD" Dir.pwd diff --git a/lib/tebako/packager.rb b/lib/tebako/packager.rb index 7a9efa3f..e6e99c40 100644 --- a/lib/tebako/packager.rb +++ b/lib/tebako/packager.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -# Copyright (c) 2021-2023 [Ribose Inc](https://www.ribose.com). +# Copyright (c) 2021-2024 [Ribose Inc](https://www.ribose.com). # All rights reserved. # This file is a part of tebako # @@ -30,6 +30,7 @@ require_relative "error" require_relative "packager/pass1" require_relative "packager/pass2" +require_relative "packager/patch_helpers" # Tebako - an executable packager module Tebako @@ -62,20 +63,45 @@ module Packager "TEBAKO_PASS_THROUGH" => "1" }.freeze + # Magic version numbers used to ensure compatibility for Ruby 2.7.x, 3.0.x + # These are the minimal versions required to provide linux-gnu / linux-musl differentiantion by bundler + # Ruby 3.1+ default bubdler/rubygems versions work correctly out of the box + BUNDLER_VERSION = "2.4.22" + RUBYGEMS_VERSION = "3.4.22" + class << self + # Deploy + def deploy(src_dir, tbd, gflength) + puts "-- Running deploy script" + + ruby_ver = ruby_version(tbd) + update_rubygems(tbd, "#{src_dir}/lib", ruby_ver, RUBYGEMS_VERSION) + install_gem tbd, "tebako-runtime" + install_gem tbd, "bundler", (PatchHelpers.ruby31?(ruby_ver) ? nil : BUNDLER_VERSION) if gflength.to_i != 0 + end + + # Deploy + def init(stash_dir, src_dir, pre_dir, bin_dir) + puts "-- Running init script" + + puts " ... creating packaging environment at #{src_dir}" + PatchHelpers.recreate([src_dir, pre_dir, bin_dir]) + FileUtils.cp_r "#{stash_dir}/.", src_dir + end + # Pass1 # Executed before Ruby build, patching ensures that Ruby itself is linked statically def pass1(ostype, ruby_source_dir, mount_point, src_dir, ruby_ver) puts "-- Running pass1 script" - recreate(src_dir) + PatchHelpers.recreate(src_dir) do_patch(Pass1.get_patch_map(ostype, mount_point, ruby_ver), ruby_source_dir) # Roll back pass2 patches # Just in case we are recovering after some error - restore_and_save_files(FILES_TO_RESTORE, ruby_source_dir) - restore_and_save_files(FILES_TO_RESTORE_MUSL, ruby_source_dir) if ostype =~ /linux-musl/ - restore_and_save_files(FILES_TO_RESTORE_MSYS, ruby_source_dir) if ostype =~ /msys/ + PatchHelpers.restore_and_save_files(FILES_TO_RESTORE, ruby_source_dir) + PatchHelpers.restore_and_save_files(FILES_TO_RESTORE_MUSL, ruby_source_dir) if ostype =~ /linux-musl/ + PatchHelpers.restore_and_save_files(FILES_TO_RESTORE_MSYS, ruby_source_dir) if ostype =~ /msys/ end # Pass2 @@ -100,81 +126,54 @@ def stash(src_dir, stash_dir) # end puts " ... saving pristine ruby environment to #{stash_dir}" - recreate(stash_dir) + PatchHelpers.recreate(stash_dir) FileUtils.cp_r "#{src_dir}/.", stash_dir end - # Deploy - def deploy(stash_dir, src_dir, pre_dir, bin_dir, tbd) - puts "-- Running deploy script" - - puts " ... creating packaging environment at #{src_dir}" - recreate([src_dir, pre_dir, bin_dir]) - FileUtils.cp_r "#{stash_dir}/.", src_dir - - install_gem tbd, "tebako-runtime" - end - private - def install_gem(tbd, name) - puts " ... installing #{name} gem" - with_env(DEPLOY_ENV) do - out, st = Open3.capture2e("#{tbd}/gem", "install", name.to_s, "--no-doc") + def install_gem(tbd, name, ver = nil) + puts " ... installing #{name} gem#{" version #{ver}" if ver}" + PatchHelpers.with_env(DEPLOY_ENV) do + params = ["#{tbd}/gem", "install", name.to_s] + params.push("-v", ver.to_s) if ver + + out, st = Open3.capture2e(*params) raise Tebako::Error, "Failed to install #{name} (#{st}):\n #{out}" unless st.exitstatus.zero? end end def do_patch(patch_map, root) - patch_map.each { |fname, mapping| patch_file("#{root}/#{fname}", mapping) } + patch_map.each { |fname, mapping| PatchHelpers.patch_file("#{root}/#{fname}", mapping) } end - def patch_file(fname, mapping) - raise Tebako::Error, "Could not patch #{fname} because it does not exist." unless File.exist?(fname) + def ruby_version(tbd) + ruby_version = nil + PatchHelpers.with_env(DEPLOY_ENV) do + out, st = Open3.capture2e("#{tbd}/ruby", "--version") + raise Tebako::Error, "Failed to run ruby --version" unless st.exitstatus.zero? - puts " ... patching #{fname}" - restore_and_save(fname) - contents = File.read(fname) - mapping.each { |pattern, subst| contents.sub!(pattern, subst) } - File.open(fname, "w") { |file| file << contents } - end - - def recreate(dirname) - FileUtils.rm_rf(dirname, noop: nil, verbose: nil, secure: true) - FileUtils.mkdir(dirname) - end - - def restore_and_save(fname) - raise Tebako::Error, "Could not save #{fname} because it does not exist." unless File.exist?(fname) + match = out.match(/ruby (\d+\.\d+\.\d+)/) + raise Tebako::Error, "Failed to parse Ruby version from #{out}" unless match - old_fname = "#{fname}.old" - if File.exist?(old_fname) - FileUtils.rm_f(fname) - File.rename(old_fname, fname) + ruby_version = match[1] end - FileUtils.cp(fname, old_fname) + ruby_version end - def restore_and_save_files(files, ruby_source_dir) - files.each do |fname| - restore_and_save "#{ruby_source_dir}/#{fname}" - end - end + def update_rubygems(tbd, tld, ruby_ver, gem_ver) + return if PatchHelpers.ruby31?(ruby_ver) - # Sets up temporary environment variables and yields to the - # block. When the block exits, the environment variables are set - # back to their original values. - def with_env(hash) - old = {} - hash.each do |k, v| - old[k] = ENV.fetch(k, nil) - ENV[k] = v - end - begin - yield - ensure - hash.each { |k, _v| ENV[k] = old[k] } + puts " ... updating rubygems to #{gem_ver}" + PatchHelpers.with_env(DEPLOY_ENV) do + out, st = Open3.capture2e("#{tbd}/gem", "update", "--no-doc", "--system", gem_ver.to_s) + raise Tebako::Error, "Failed to update rubugems to #{gem_ver} (#{st}):\n #{out}" unless st.exitstatus.zero? end + ruby_api_ver = ruby_ver.split(".")[0..1].join(".") + # Autoload cannot handle statically linked openssl extension + # Changing it to require seems to be the simplest solution + PatchHelpers.patch_file("#{tld}/ruby/site_ruby/#{ruby_api_ver}.0/rubygems/openssl.rb", + { "autoload :OpenSSL, \"openssl\"" => "require \"openssl\"" }) end end end diff --git a/lib/tebako/packager/patch_helpers.rb b/lib/tebako/packager/patch_helpers.rb index 5c063e4a..3b6b4d3b 100755 --- a/lib/tebako/packager/patch_helpers.rb +++ b/lib/tebako/packager/patch_helpers.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -# Copyright (c) 2023 [Ribose Inc](https://www.ribose.com). +# Copyright (c) 2023-2024 [Ribose Inc](https://www.ribose.com). # All rights reserved. # This file is a part of tebako # @@ -34,6 +34,16 @@ module Packager # Ruby patching helpers (pass2) module PatchHelpers class << self + def patch_file(fname, mapping) + raise Tebako::Error, "Could not patch #{fname} because it does not exist." unless File.exist?(fname) + + puts " ... patching #{fname}" + restore_and_save(fname) + contents = File.read(fname) + mapping.each { |pattern, subst| contents.sub!(pattern, subst) } + File.open(fname, "w") { |file| file << contents } + end + def get_prefix_macos(package) out, st = Open3.capture2("brew --prefix #{package}") raise Tebako::Error, "brew --prefix #{package} failed with code #{st.exitstatus}" unless st.exitstatus.zero? @@ -51,6 +61,28 @@ def get_prefix_linux(package) out end + def recreate(dirname) + FileUtils.rm_rf(dirname, noop: nil, verbose: nil, secure: true) + FileUtils.mkdir(dirname) + end + + def restore_and_save(fname) + raise Tebako::Error, "Could not save #{fname} because it does not exist." unless File.exist?(fname) + + old_fname = "#{fname}.old" + if File.exist?(old_fname) + FileUtils.rm_f(fname) + File.rename(old_fname, fname) + end + FileUtils.cp(fname, old_fname) + end + + def restore_and_save_files(files, ruby_source_dir) + files.each do |fname| + restore_and_save "#{ruby_source_dir}/#{fname}" + end + end + def ruby3x?(ruby_ver) ruby_ver[0] == "3" end @@ -63,6 +95,22 @@ def ruby32?(ruby_ver) ruby3x?(ruby_ver) && ruby_ver[2].to_i >= 2 end + # Sets up temporary environment variables and yields to the + # block. When the block exits, the environment variables are set + # back to their original values. + def with_env(hash) + old = {} + hash.each do |k, v| + old[k] = ENV.fetch(k, nil) + ENV[k] = v + end + begin + yield + ensure + hash.each_key { |k| ENV[k] = old[k] } + end + end + def yaml_reference(ruby_ver) ruby32?(ruby_ver) ? "-l:libyaml.a" : "" end diff --git a/lib/tebako/version.rb b/lib/tebako/version.rb index 073acdbc..420f5544 100644 --- a/lib/tebako/version.rb +++ b/lib/tebako/version.rb @@ -26,5 +26,5 @@ # POSSIBILITY OF SUCH DAMAGE. module Tebako - VERSION = "0.5.5" + VERSION = "0.5.6" end diff --git a/tests-2/tebako-test.rb b/tests-2/tebako-test.rb index d089693f..4ea145fd 100755 --- a/tests-2/tebako-test.rb +++ b/tests-2/tebako-test.rb @@ -60,7 +60,7 @@ def with_env(hash) begin yield ensure - hash.each { |k, _v| ENV[k] = old[k] } + hash.each_key { |k| ENV[k] = old[k] } end end diff --git a/tests/scripts/functional-tests.sh b/tests/scripts/functional-tests.sh index d44e8ebb..7f1a01c0 100755 --- a/tests/scripts/functional-tests.sh +++ b/tests/scripts/functional-tests.sh @@ -330,6 +330,6 @@ DIR_BIN=$( cd "$DIR_ROOT"/exe && pwd ) DIR_TESTS=$( cd "$DIR_ROOT"/tests && pwd ) RUBY_VER=${RUBY_VER:-3.1.4} -echo "Running tebako tests" +echo "Running tebako tests for Ruby $RUBY_VER" # shellcheck source=/dev/null . "$DIR_TESTS/shunit2/shunit2" diff --git a/tests/test-01/tebako-test-run.rb b/tests/test-01/tebako-test-run.rb index 449b3d97..8daf14af 100755 --- a/tests/test-01/tebako-test-run.rb +++ b/tests/test-01/tebako-test-run.rb @@ -3,6 +3,7 @@ puts "Hello! This is test-01 talking from inside DwarFS" puts "Gem path: #{Gem.path}" +puts "Rubygems version: #{Gem::rubygems_version}" if defined?(TebakoRuntime::VERSION) puts "Using tebako-runtime v#{TebakoRuntime::VERSION}" else