Implemented rubygems update to ensure correct linux-gnu/linux-musl di…

…fferentiation by bundler #120

.github/workflows/gem-test-and-release.yml

@@ -74,7 +74,6 @@ jobs:
       uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ env.RUBY_VER }}
-        bundler: ${{ env.BUNDLER_VER }}
         bundler-cache: true
 
     - name: Build gem

CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2021-2023 [Ribose Inc](https://www.ribose.com).
+# Copyright (c) 2021-2024 [Ribose Inc](https://www.ribose.com).
 # All rights reserved.
 # This file is a part of tebako
 #
@@ -245,14 +245,6 @@ endif(IS_MSYS)
 def_ext_prj_g(INCBIN "348e36b")
 def_ext_prj_g(DWARFS_WR "v0.4.0")
 
-if (DEFINED ENV{BUNDLER_VER})
- set(BUNDLER_VER $ENV{BUNDLER_VER})
- set(BUNDLER_ANNOTATION "environment")
-else()
-  set(BUNDLER_VER "2.3.22")
-  set(BUNDLER_ANNOTATION "default")
-endif()
-
 find_library(_LIBNCURSES "libncurses.a")
 if(${_LIBNCURSES} STREQUAL "_LIBNCURSES-NOTFOUND")
   set(WITH_NCURSES_BUILD ON)
@@ -268,7 +260,6 @@ endif(${RUBY_VER} VERSION_LESS "3.2.0")
 
 message("Configuration summary:")
 message(STATUS "ruby: v${RUBY_VER} at ${RUBY_SOURCE_DIR}")
-message(STATUS "bundler version: ${BUNDLER_VER} (${BUNDLER_ANNOTATION})")
 if(WITH_OPENSSL_BUILD)
   message(STATUS "openssl: building @${OPENSSL_TAG} at ${OPENSSL_SOURCE_DIR}")
 endif(WITH_OPENSSL_BUILD)
@@ -511,7 +502,7 @@ else (${SETUP_MODE})
   list(LENGTH GEMS     GLENGTH)
 
   add_custom_target(clean_filesystem
-      COMMAND ruby ${EXE}/tebako-packager deploy ${RUBY_STASH_DIR} ${DATA_SRC_DIR} ${DATA_PRE_DIR} ${DATA_BIN_DIR} ${TBD} ${TGD}
+      COMMAND ruby ${EXE}/tebako-packager deploy ${RUBY_STASH_DIR} ${DATA_SRC_DIR} ${DATA_PRE_DIR} ${DATA_BIN_DIR} ${GFLENGTH}
       DEPENDS ${RUBY_PRJ}
   )
   if(GSLENGTH GREATER 0)
@@ -527,8 +518,6 @@ else (${SETUP_MODE})
         add_custom_target(source_filesystem
            COMMAND ${CMAKE_COMMAND} -DSOURCE_DIR=${FS_ROOT} -DTARGET_DIR=${DATA_PRE_DIR} -P ${CMAKE_SOURCE_DIR}/cmake/copy_dir.cmake
            COMMAND ${CMAKE_COMMAND} -E make_directory ${TGD}
-           COMMAND ${CMAKE_COMMAND} -E chdir ${DATA_PRE_DIR} ${CMAKE_COMMAND} -E env --unset=GEM_HOME --unset=GEM_PATH TEBAKO_PASS_THROUGH=1
-                                       ${TBD}/gem${CMD_SUFFIX} install bundler -v '${BUNDLER_VER}' --source 'https://rubygems.org/' --no-document --install-dir ${TGD}
            COMMAND ${CMAKE_COMMAND} -E chdir ${DATA_PRE_DIR} ${CMAKE_COMMAND} -E env --unset=GEM_HOME --unset=GEM_PATH TEBAKO_PASS_THROUGH=1
                                        ${TBD}/bundle${CMD_SUFFIX} config set --local force_ruby_platform ${FORCE_RUBY_PLATFORM}
            COMMAND ${CMAKE_COMMAND} -E chdir ${DATA_PRE_DIR} ${CMAKE_COMMAND} -E  env --unset=GEM_HOME --unset=GEM_PATH TEBAKO_PASS_THROUGH=1
@@ -572,9 +561,6 @@ else (${SETUP_MODE})
                    ${CMAKE_COMMAND} -E false )
        COMMAND ${CMAKE_COMMAND} -E make_directory ${TLD}
        COMMAND ${CMAKE_COMMAND} -DSOURCE_DIR=${FS_ROOT} -DTARGET_DIR=${TLD} -P ${CMAKE_SOURCE_DIR}/cmake/copy_dir.cmake
-       COMMAND ${CMAKE_COMMAND} -E chdir ${TLD} ${CMAKE_COMMAND} -E env --unset=GEM_HOME --unset=GEM_PATH TEBAKO_PASS_THROUGH=1
-                                           ${TBD}/gem${CMD_SUFFIX} install bundler -v '${BUNDLER_VER}'
-                                              --source 'https://rubygems.org/' --no-document --install-dir ${TGD}
        COMMAND ${CMAKE_COMMAND} -E chdir ${TLD} ${CMAKE_COMMAND} -E env --unset=GEM_HOME --unset=GEM_PATH TEBAKO_PASS_THROUGH=1
                                            ${TBD}/bundle${CMD_SUFFIX} config build.ffi --disable-system-libffi
        COMMAND ${CMAKE_COMMAND} -E chdir ${TLD} ${CMAKE_COMMAND} -E env --unset=GEM_HOME --unset=GEM_PATH TEBAKO_PASS_THROUGH=1

common.env

@@ -2,5 +2,4 @@ BUILD_TYPE=Release
 DEPS=deps
 INCBIN_TAG=348e36b
 DWARFS_WR_TAG=v0.4.0
-BUNDLER_VER=2.3.22
 RUBY_VER=3.1.4

exe/tebako-packager

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-# Copyright (c) 2023 [Ribose Inc](https://www.ribose.com).
+# Copyright (c) 2023-2024 [Ribose Inc](https://www.ribose.com).
 # All rights reserved.
 # This file is a part of tebako
 #
@@ -78,13 +78,16 @@ begin
     #       ARGV[2] -- DATA_SRC_DIR
     #       ARGV[3] -- DATA_PRE_DIR
     #       ARGV[4] -- DATA_BIN_DIR
-    #       ARGV[5] -- TARGET_BIN_DIR (TBD)
-    #       ARGV[6] -- TARGET_GEM_DIR (TGD)
-    unless ARGV.length == 7
+    #       ARGV[5] -- GFLENGTH
+    unless ARGV.length == 6
       raise Tebako::Error,
-            "tebako-packager deploy command expects 7 arguments, #{ARGV.length} has been provided."
+            "tebako-packager deploy command expects 6 arguments, #{ARGV.length} has been provided."
     end
-    Tebako::Packager.deploy(ARGV[1], ARGV[2], ARGV[3], ARGV[4], ARGV[5])
+    Tebako::Packager.init(ARGV[1], ARGV[2], ARGV[3], ARGV[4])
+    # Assume that "<TARGET_BIN_DIR (TBD)>" is <DATA_SRC_DIR>/bin"
+    # That shall match CMakeLists.txt settings
+    # Tebako::Packager.update("#{ARGV[1]}/bin", "2.7.7")
+    Tebako::Packager.deploy(ARGV[2], "#{ARGV[2]}/bin", ARGV[5])
   else
     raise Tebako::Error, "tebako-packager cannot process #{ARGV[0]} command"
   end

lib/tebako/cli_helpers.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-# Copyright (c) 2023 [Ribose Inc](https://www.ribose.com).
+# Copyright (c) 2023-2024 [Ribose Inc](https://www.ribose.com).
 # All rights reserved.
 # This file is a part of tebako
 #
@@ -130,6 +130,7 @@ def packaging_error(code)
 
     def prefix
       @prefix ||= if options["prefix"].nil?
+                    puts "No prefix specified, using ~/.tebako"
                     File.expand_path("~/.tebako")
                   elsif options["prefix"] == "PWD"
                     Dir.pwd

lib/tebako/packager.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-# Copyright (c) 2021-2023 [Ribose Inc](https://www.ribose.com).
+# Copyright (c) 2021-2024 [Ribose Inc](https://www.ribose.com).
 # All rights reserved.
 # This file is a part of tebako
 #
@@ -30,6 +30,7 @@
 require_relative "error"
 require_relative "packager/pass1"
 require_relative "packager/pass2"
+require_relative "packager/patch_helpers"
 
 # Tebako - an executable packager
 module Tebako
@@ -62,20 +63,45 @@ module Packager
       "TEBAKO_PASS_THROUGH" => "1"
     }.freeze
 
+    # Magic version numbers used to ensure compatibility for Ruby 2.7.x, 3.0.x
+    # These are the minimal versions required to provide linux-gnu / linux-musl differentiantion by bundler
+    # Ruby 3.1+ default bubdler/rubygems versions work correctly out of the box
+    BUNDLER_VERSION = "2.4.22"
+    RUBYGEMS_VERSION = "3.4.22"
+
     class << self
+      # Deploy
+      def deploy(src_dir, tbd, gflength)
+        puts "-- Running deploy script"
+
+        ruby_ver = ruby_version(tbd)
+        update_rubygems(tbd, "#{src_dir}/lib", ruby_ver, RUBYGEMS_VERSION)
+        install_gem tbd, "tebako-runtime"
+        install_gem tbd, "bundler", (PatchHelpers.ruby31?(ruby_ver) ? nil : BUNDLER_VERSION) if gflength.to_i != 0
+      end
+
+      # Deploy
+      def init(stash_dir, src_dir, pre_dir, bin_dir)
+        puts "-- Running init script"
+
+        puts "   ... creating packaging environment at #{src_dir}"
+        PatchHelpers.recreate([src_dir, pre_dir, bin_dir])
+        FileUtils.cp_r "#{stash_dir}/.", src_dir
+      end
+
       # Pass1
       # Executed before Ruby build, patching ensures that Ruby itself is linked statically
       def pass1(ostype, ruby_source_dir, mount_point, src_dir, ruby_ver)
         puts "-- Running pass1 script"
 
-        recreate(src_dir)
+        PatchHelpers.recreate(src_dir)
         do_patch(Pass1.get_patch_map(ostype, mount_point, ruby_ver), ruby_source_dir)
 
         # Roll back pass2 patches
         # Just in case we are recovering after some error
-        restore_and_save_files(FILES_TO_RESTORE, ruby_source_dir)
-        restore_and_save_files(FILES_TO_RESTORE_MUSL, ruby_source_dir) if ostype =~ /linux-musl/
-        restore_and_save_files(FILES_TO_RESTORE_MSYS, ruby_source_dir) if ostype =~ /msys/
+        PatchHelpers.restore_and_save_files(FILES_TO_RESTORE, ruby_source_dir)
+        PatchHelpers.restore_and_save_files(FILES_TO_RESTORE_MUSL, ruby_source_dir) if ostype =~ /linux-musl/
+        PatchHelpers.restore_and_save_files(FILES_TO_RESTORE_MSYS, ruby_source_dir) if ostype =~ /msys/
       end
 
       # Pass2
@@ -100,81 +126,54 @@ def stash(src_dir, stash_dir)
         #    end
 
         puts "   ... saving pristine ruby environment to #{stash_dir}"
-        recreate(stash_dir)
+        PatchHelpers.recreate(stash_dir)
         FileUtils.cp_r "#{src_dir}/.", stash_dir
       end
 
-      # Deploy
-      def deploy(stash_dir, src_dir, pre_dir, bin_dir, tbd)
-        puts "-- Running deploy script"
-
-        puts "   ... creating packaging environment at #{src_dir}"
-        recreate([src_dir, pre_dir, bin_dir])
-        FileUtils.cp_r "#{stash_dir}/.", src_dir
-
-        install_gem tbd, "tebako-runtime"
-      end
-
       private
 
-      def install_gem(tbd, name)
-        puts "   ... installing #{name} gem"
-        with_env(DEPLOY_ENV) do
-          out, st = Open3.capture2e("#{tbd}/gem", "install", name.to_s, "--no-doc")
+      def install_gem(tbd, name, ver = nil)
+        puts "   ... installing #{name} gem#{" version #{ver}" if ver}"
+        PatchHelpers.with_env(DEPLOY_ENV) do
+          params = ["#{tbd}/gem", "install", name.to_s]
+          params.push("-v", ver.to_s) if ver
+
+          out, st = Open3.capture2e(*params)
           raise Tebako::Error, "Failed to install #{name} (#{st}):\n #{out}" unless st.exitstatus.zero?
         end
       end
 
       def do_patch(patch_map, root)
-        patch_map.each { |fname, mapping| patch_file("#{root}/#{fname}", mapping) }
+        patch_map.each { |fname, mapping| PatchHelpers.patch_file("#{root}/#{fname}", mapping) }
       end
 
-      def patch_file(fname, mapping)
-        raise Tebako::Error, "Could not patch #{fname} because it does not exist." unless File.exist?(fname)
+      def ruby_version(tbd)
+        ruby_version = nil
+        PatchHelpers.with_env(DEPLOY_ENV) do
+          out, st = Open3.capture2e("#{tbd}/ruby", "--version")
+          raise Tebako::Error, "Failed to run ruby --version" unless st.exitstatus.zero?
 
-        puts "   ... patching #{fname}"
-        restore_and_save(fname)
-        contents = File.read(fname)
-        mapping.each { |pattern, subst| contents.sub!(pattern, subst) }
-        File.open(fname, "w") { |file| file << contents }
-      end
-
-      def recreate(dirname)
-        FileUtils.rm_rf(dirname, noop: nil, verbose: nil, secure: true)
-        FileUtils.mkdir(dirname)
-      end
-
-      def restore_and_save(fname)
-        raise Tebako::Error, "Could not save #{fname} because it does not exist." unless File.exist?(fname)
+          match = out.match(/ruby (\d+\.\d+\.\d+)/)
+          raise Tebako::Error, "Failed to parse Ruby version from #{out}" unless match
 
-        old_fname = "#{fname}.old"
-        if File.exist?(old_fname)
-          FileUtils.rm_f(fname)
-          File.rename(old_fname, fname)
+          ruby_version = match[1]
         end
-        FileUtils.cp(fname, old_fname)
+        ruby_version
       end
 
-      def restore_and_save_files(files, ruby_source_dir)
-        files.each do |fname|
-          restore_and_save "#{ruby_source_dir}/#{fname}"
-        end
-      end
+      def update_rubygems(tbd, tld, ruby_ver, gem_ver)
+        return if PatchHelpers.ruby31?(ruby_ver)
 
-      # Sets up temporary environment variables and yields to the
-      # block. When the block exits, the environment variables are set
-      # back to their original values.
-      def with_env(hash)
-        old = {}
-        hash.each do |k, v|
-          old[k] = ENV.fetch(k, nil)
-          ENV[k] = v
-        end
-        begin
-          yield
-        ensure
-          hash.each { |k, _v| ENV[k] = old[k] }
+        puts "   ... updating rubygems to #{gem_ver}"
+        PatchHelpers.with_env(DEPLOY_ENV) do
+          out, st = Open3.capture2e("#{tbd}/gem", "update", "--no-doc", "--system", gem_ver.to_s)
+          raise Tebako::Error, "Failed to update rubugems to #{gem_ver} (#{st}):\n #{out}" unless st.exitstatus.zero?
         end
+        ruby_api_ver = ruby_ver.split(".")[0..1].join(".")
+        # Autoload cannot handle statically linked openssl extension
+        # Changing it to require seems to be the simplest solution
+        PatchHelpers.patch_file("#{tld}/ruby/site_ruby/#{ruby_api_ver}.0/rubygems/openssl.rb",
+                                { "autoload :OpenSSL, \"openssl\"" => "require \"openssl\"" })
       end
     end
   end

lib/tebako/packager/patch_helpers.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-# Copyright (c) 2023 [Ribose Inc](https://www.ribose.com).
+# Copyright (c) 2023-2024 [Ribose Inc](https://www.ribose.com).
 # All rights reserved.
 # This file is a part of tebako
 #
@@ -34,6 +34,16 @@ module Packager
     # Ruby patching helpers (pass2)
     module PatchHelpers
       class << self
+        def patch_file(fname, mapping)
+          raise Tebako::Error, "Could not patch #{fname} because it does not exist." unless File.exist?(fname)
+
+          puts "   ... patching #{fname}"
+          restore_and_save(fname)
+          contents = File.read(fname)
+          mapping.each { |pattern, subst| contents.sub!(pattern, subst) }
+          File.open(fname, "w") { |file| file << contents }
+        end
+
         def get_prefix_macos(package)
           out, st = Open3.capture2("brew --prefix #{package}")
           raise Tebako::Error, "brew --prefix #{package} failed with code #{st.exitstatus}" unless st.exitstatus.zero?
@@ -51,6 +61,28 @@ def get_prefix_linux(package)
           out
         end
 
+        def recreate(dirname)
+          FileUtils.rm_rf(dirname, noop: nil, verbose: nil, secure: true)
+          FileUtils.mkdir(dirname)
+        end
+
+        def restore_and_save(fname)
+          raise Tebako::Error, "Could not save #{fname} because it does not exist." unless File.exist?(fname)
+
+          old_fname = "#{fname}.old"
+          if File.exist?(old_fname)
+            FileUtils.rm_f(fname)
+            File.rename(old_fname, fname)
+          end
+          FileUtils.cp(fname, old_fname)
+        end
+
+        def restore_and_save_files(files, ruby_source_dir)
+          files.each do |fname|
+            restore_and_save "#{ruby_source_dir}/#{fname}"
+          end
+        end
+
         def ruby3x?(ruby_ver)
           ruby_ver[0] == "3"
         end
@@ -63,6 +95,22 @@ def ruby32?(ruby_ver)
           ruby3x?(ruby_ver) && ruby_ver[2].to_i >= 2
         end
 
+        # Sets up temporary environment variables and yields to the
+        # block. When the block exits, the environment variables are set
+        # back to their original values.
+        def with_env(hash)
+          old = {}
+          hash.each do |k, v|
+            old[k] = ENV.fetch(k, nil)
+            ENV[k] = v
+          end
+          begin
+            yield
+          ensure
+            hash.each_key { |k| ENV[k] = old[k] }
+          end
+        end
+
         def yaml_reference(ruby_ver)
           ruby32?(ruby_ver) ? "-l:libyaml.a" : ""
         end

lib/tebako/version.rb

@@ -26,5 +26,5 @@
 # POSSIBILITY OF SUCH DAMAGE.
 
 module Tebako
-  VERSION = "0.5.5"
+  VERSION = "0.5.6"
 end

tests-2/tebako-test.rb

@@ -60,7 +60,7 @@ def with_env(hash)
     begin
       yield
     ensure
-      hash.each { |k, _v| ENV[k] = old[k] }
+      hash.each_key { |k| ENV[k] = old[k] }
     end
   end
 

tests/scripts/functional-tests.sh

@@ -330,6 +330,6 @@ DIR_BIN=$( cd "$DIR_ROOT"/exe && pwd )
 DIR_TESTS=$( cd "$DIR_ROOT"/tests && pwd )
 RUBY_VER=${RUBY_VER:-3.1.4}
 
-echo "Running tebako tests"
+echo "Running tebako tests for Ruby $RUBY_VER"
 # shellcheck source=/dev/null
 . "$DIR_TESTS/shunit2/shunit2"