fix: stripping debug information from tebako package [except Windows]

.github/workflows/alpine.yml

@@ -48,7 +48,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  CACHE_VER: 55
+  CACHE_VER: 02
   TZ: "Etc/UTC"
   VERBOSE: no
 

.github/workflows/gem-test-and-release.yml

@@ -49,7 +49,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  CACHE_VER: 55
+  CACHE_VER: 02
   DEBIAN_FRONTEND: "noninteractive"
   TZ: "Etc/UTC"
 # show cmake output

.github/workflows/macos.yml

@@ -48,7 +48,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  CACHE_VER: 55
+  CACHE_VER: 02
   VERBOSE: no
 
 jobs:

.github/workflows/ubuntu.yml

@@ -48,7 +48,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  CACHE_VER: 55
+  CACHE_VER: 02
   DEBIAN_FRONTEND: "noninteractive"
   TZ: "Etc/UTC"
 # show cmake output (yes/no)

.github/workflows/windows-msys.yml

@@ -58,7 +58,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  CACHE_VER: 55
+  CACHE_VER: 02
   VERBOSE: no
 
 jobs:

CMakeLists.txt

@@ -116,7 +116,7 @@ set(IS_MUSL OFF)
 set(IS_MSYS OFF)
 set(IS_DARWIN OFF)
 set(RB_W32 OFF)
-set(RUBY_WITHOUT_EXT "dbm,win32,win32ole,'-test-'")
+set(RUBY_WITHOUT_EXT "dbm,win32,win32ole,-test-\*")
 set(RUBY_NAME "ruby")
 set(RUBY_SUFFIX "")
 set(EXE_SUFFIX "")
@@ -131,7 +131,7 @@ elseif("${OSTYPE_TXT}" MATCHES "^msys*")
   set(IS_MSYS ON)
   # set(DWARFS_PRELOAD ON)
   set(RB_W32 ON)
-  set(RUBY_WITHOUT_EXT "dbm,syslog,pty,gdbm,readline,'-test-'")
+  set(RUBY_WITHOUT_EXT "dbm,syslog,pty,gdbm,readline,-test-\*")
   #set(RUBY_SUFFIX "w")
   set(EXE_SUFFIX ".exe")
   set(CMD_SUFFIX ".cmd")
@@ -462,7 +462,6 @@ else (${SETUP_MODE})
   )
 
   add_custom_target(packaged_filesystem
-    COMMAND ruby ${EXE}/tebako-packager strip ${OSTYPE_TXT} ${DATA_SRC_DIR}
     COMMAND ${GNU_BASH} -c "chmod +x ${DEPS_BIN_DIR}/mkdwarfs${EXE_SUFFIX}"
     # No progress below is critical since in reporting mode mkdwarfs tries to work directly with terminal
     # and it creates issues for stderr redirects (&2 > 1) used by Ninja and our test srcipts
@@ -487,22 +486,18 @@ else (${SETUP_MODE})
   add_dependencies(tebako-fs packaged_filesystem)
 
   if (${RUBY_VER} VERSION_LESS "3.0.0")
-    add_custom_target(patched_ruby
+    add_custom_target(tebako
       COMMAND ${CMAKE_COMMAND} -E chdir ${RUBY_SOURCE_DIR} make ${RUBY_MAKEFILE} -j${NCORES}
-    )
+      COMMAND ruby ${EXE}/tebako-packager finalize ${OSTYPE_TXT} ${RUBY_SOURCE_DIR} ${APP_NAME}
+      )
   else()
-    add_custom_target(patched_ruby
+    add_custom_target(tebako
       COMMAND ${CMAKE_COMMAND} -E chdir ${RUBY_SOURCE_DIR} make ruby ${RUBY_MAKEFILE} -j${NCORES}
       COMMAND ${CMAKE_COMMAND} -E chdir ${RUBY_SOURCE_DIR} make ${RUBY_MAKEFILE} -j${NCORES}
-    )
+      COMMAND ruby ${EXE}/tebako-packager finalize ${OSTYPE_TXT} ${RUBY_SOURCE_DIR} ${APP_NAME}
+      )
   endif()
 
-  add_dependencies(patched_ruby setup tebako-fs)
-
-  add_custom_target(tebako ALL
-    COMMAND ${CMAKE_COMMAND} -E copy ${RUBY_SOURCE_DIR}/ruby${RUBY_SUFFIX}${EXE_SUFFIX} ${APP_NAME}${EXE_SUFFIX}
-    COMMAND ${CMAKE_COMMAND} -E echo "Tebako packaging has completed"
-    DEPENDS patched_ruby
-  )
+  add_dependencies(tebako setup tebako-fs)
 
 endif(${SETUP_MODE})

README.adoc

@@ -38,7 +38,15 @@ The Tebako packager is tested on the following platforms:
 * MacOS: macOS 12 (Monterey), 13 (Ventura), 14 (Sonoma)
 * Windows: 2019, 2022 (using MinGW ucrt64 toolchain)
 
-Please note that Windows support is in pre-release and is a subject of further testing and optimization
+There are two known Windows issues:
+
+* tebako may face errors related to CMake path length limitations (https://gitlab.kitware.com/cmake/cmake/-/issues/25936).
+This error may affect not tebako itself but the gems that need to be package and use CMake to build native extensions.
+There is no workaround for this issue as it looks like is a limitation of the manifest used to build CMake executable.
+* MSys strip utility creates broken executable when tebako image is processed. Linking with '-s' flag produces unusable
+executables as well.
+Until this issue (https://github.com/tamatebako/tebako/issues/172) is resolved we plan to produce an Windows executable
+with debug information unstripped. You can opt to run 'strip -S' manually, it most cases it works.
 
 == Supported Ruby versions
 
@@ -135,7 +143,7 @@ To ensure the correct version of bison is picked, run this command before the bu
 
 [source,sh]
 ----
-export PATH="$(brew --prefix bison)/bin:$PATH" 
+export PATH="$(brew --prefix bison)/bin:$PATH"
 ----
 
 === Windows (workstation 10, 11; Server 2019, 2022)
@@ -151,11 +159,6 @@ Once it is installed use the following commands:
             fmt glog dlfcn gtest autotools ncurses libyaml
 ----
 
-Please note that on Windows tebako may face issues related by CMake path length limitations.
-(https://gitlab.kitware.com/cmake/cmake/-/issues/25936)
-This error may affect not tebako itself but the gems that need to be package and use CMake to build native extensions.
-There is no workaround for this issue as it lloks like is a limitation of the manifest used to build CMake executable.
-
 == Installation
 
 === General

exe/tebako-packager

@@ -48,26 +48,7 @@ begin
     end
 
     Tebako::Packager.pass1(ARGV[1], ARGV[2], ARGV[3], ARGV[4], ARGV[5])
-  when "stash"
-    #       ARGV[0] -- command
-    #       ARGV[1] -- DATA_SRC_DIR
-    #       ARGV[2] -- RUBY_STASH_DIR
-    unless ARGV.length == 3
-      raise Tebako::Error,
-            "tebako-packager stash command expects 3 arguments, #{ARGV.length} has been provided."
-    end
-
-    Tebako::Packager.stash(ARGV[1], ARGV[2])
-  when "strip"
-    #       ARGV[0] -- command
-    #       ARGV[1] -- OSTYPE
-    #       ARGV[2] -- DATA_SRC_DIR
-    unless ARGV.length == 3
-      raise Tebako::Error,
-            "tebako-packager strip command expects 3 arguments, #{ARGV.length} has been provided."
-    end
 
-    Tebako::Stripper.strip(ARGV[1], ARGV[2])
   when "pass1a"
     #       ARGV[0] -- command
     #       ARGV[1] -- RUBY_SOURCE_DIR
@@ -114,12 +95,21 @@ begin
     # That shall match CMakeLists.txt settings
     Tebako::Packager.deploy(ARGV[1], ARGV[4], ARGV[5], ARGV[11], ARGV[7], ARGV[8], ARGV[9])
 
+  when "finalize"
+    #       ARGV[0] -- command
+    #       ARGV[1] -- OSTYPE
+    #       ARGV[2] -- RUBY_SOURCE_DIR
+    #       ARGV[3] -- APP_NAME
+    unless ARGV.length == 4
+      raise Tebako::Error,
+            "tebako-packager finalize command expects 4 arguments, #{ARGV.length} has been provided."
+    end
+    Tebako::Packager.finalize(ARGV[1], ARGV[2], ARGV[3])
   else
     raise Tebako::Error, "tebako-packager cannot process #{ARGV[0]} command"
   end
 rescue Tebako::Error => e
   puts "tebako-packager failed: #{e.message} [#{e.error_code}]"
   exit(e.error_code)
 end
-
 exit(0)

lib/tebako/cli.rb

@@ -61,8 +61,8 @@ def clean
     def clean_ruby
       puts "Cleaning Ruby sources from tebako packaging environment"
       suffix = options["Ruby"].nil? ? "" : "_#{options["Ruby"]}"
-      nmr = "src/_ruby_#{suffix}*"
-      nms = "stash_#{suffix}"
+      nmr = "src/_ruby#{suffix}*"
+      nms = "stash#{suffix}*"
       FileUtils.rm_rf(Dir.glob(File.join(deps, nmr)), secure: true)
       FileUtils.rm_rf(Dir.glob(File.join(deps, nms)), secure: true)
     end

lib/tebako/deploy_helper.rb

@@ -29,6 +29,7 @@
 require "find"
 
 require_relative "error"
+require_relative "packager/patch_helpers"
 
 # Tebako - an executable packager
 module Tebako
@@ -65,11 +66,16 @@ def config(os_type, ruby_ver)
     end
 
     def deploy
-      system("#{gem_command} env")
-      install_gem("tebako-runtime")
-      install_gem("bundler", Tebako::BUNDLER_VERSION) if needs_bundler?
-
-      deploy_solution
+      Packager::PatchHelpers.with_env(deploy_env) do
+        unless Packager::PatchHelpers.ruby31?(@ruby_ver)
+          update_rubygems
+          patch_after_rubygems_update(@target_dir, @ruby_api_version)
+        end
+        system("#{gem_command} env")
+        install_gem("tebako-runtime")
+        install_gem("bundler", BUNDLER_VERSION) if needs_bundler?
+        deploy_solution
+      end
     end
 
     def deploy_env
@@ -264,6 +270,13 @@ def ncores
                   end
     end
 
+    def patch_after_rubygems_update(target_dir, ruby_api_ver)
+      # Autoload cannot handle statically linked openssl extension
+      # Changing it to require seems to be the simplest solution
+      Packager::PatchHelpers.patch_file("#{target_dir}/lib/ruby/site_ruby/#{ruby_api_ver}/rubygems/openssl.rb",
+                                        { "autoload :OpenSSL, \"openssl\"" => "require \"openssl\"" })
+    end
+
     def run_with_capture(args)
       puts "   ... @ #{args.join(" ")}"
       out, st = Open3.capture2e(*args)

lib/tebako/packager.rb

@@ -31,6 +31,7 @@
 
 require_relative "error"
 require_relative "deploy_helper"
+require_relative "stripper"
 require_relative "packager/pass1"
 require_relative "packager/pass1a"
 require_relative "packager/pass2"
@@ -79,14 +80,22 @@ def deploy(os_type, target_dir, pre_dir, ruby_ver, fs_root, fs_entrance, fs_moun
 
         deploy_helper = Tebako::DeployHelper.new(fs_root, fs_entrance, fs_mount_point, target_dir, pre_dir)
         deploy_helper.config(os_type, ruby_ver)
+        deploy_helper.deploy
+        Tebako::Stripper.strip(os_type, target_dir)
+      end
 
-        PatchHelpers.with_env(deploy_helper.deploy_env) do
-          unless PatchHelpers.ruby31?(ruby_ver)
-            deploy_helper.update_rubygems
-            patch_after_rubygems_update(target_dir, deploy_helper.ruby_api_version)
-          end
-          deploy_helper.deploy
+      def finalize(os_type, src_dir, app_name)
+        exe_suffix = Packager::PatchHelpers.exe_suffix(os_type)
+        src_name = File.join(src_dir, "ruby#{exe_suffix}")
+        package_name = "#{app_name}#{exe_suffix}"
+        # [TODO] On MSys strip sometimes creates a broken executable
+        # https://github.com/tamatebako/tebako/issues/172
+        if Packager::PatchHelpers.msys?(os_type)
+          FileUtils.cp(src_name, package_name)
+        else
+          Tebako::Stripper.strip_file(src_name, package_name)
         end
+        puts "Created tebako package at \"#{package_name}\""
       end
 
       # Init
@@ -188,13 +197,6 @@ def ruby_version(tbd)
         end
         ruby_version
       end
-
-      def patch_after_rubygems_update(target_dir, ruby_api_ver)
-        # Autoload cannot handle statically linked openssl extension
-        # Changing it to require seems to be the simplest solution
-        PatchHelpers.patch_file("#{target_dir}/lib/ruby/site_ruby/#{ruby_api_ver}/rubygems/openssl.rb",
-                                { "autoload :OpenSSL, \"openssl\"" => "require \"openssl\"" })
-      end
     end
   end
 end

lib/tebako/packager/patch_helpers.rb

@@ -62,10 +62,18 @@ def get_prefix_linux(package)
           out
         end
 
+        def exe_suffix(ostype)
+          msys?(ostype) ? ".exe" : ""
+        end
+
         def msys?(ostype)
           ostype =~ /msys|cygwin|mingw/
         end
 
+        def macos?(ostype)
+          ostype =~ /darwin/
+        end
+
         def patch_c_file_pre(pattern)
           {
             pattern => "#{PatchLiterals::C_FILE_SUBST}\n#{pattern}"

lib/tebako/packager/patch_libraries.rb

@@ -40,13 +40,12 @@ class << self
         DARWIN_BREW_LIBS = [
           ["zlib", "z"],            ["gdbm", "gdbm"],           ["readline", "readline"], ["libffi", "ffi"],
           ["ncurses", "ncurses"],   ["fmt", "fmt"],             ["lz4", "lz4"],           ["xz", "lzma"],
-          ["libyaml", "yaml"],      ["boost", "boost_chrono"],
-          ["double-conversion", "double-conversion"]
+          ["libyaml", "yaml"],      ["boost", "boost_chrono"],  ["double-conversion", "double-conversion"]
         ].freeze
 
         DARWIN_BREW_LIBS_PRE_31 = [["[email protected]", "ssl"], ["[email protected]", "crypto"]].freeze
 
-        DARWIN_BREW_LIBS_31 = [["libyaml", "yaml"], ["openssl@3", "ssl"], ["openssl@3", "crypto"]].freeze
+        DARWIN_BREW_LIBS_31 = [["openssl@3", "ssl"], ["openssl@3", "crypto"]].freeze
 
         DARWIN_DEP_LIBS = ["glog", "gflags", "brotlienc", "brotlidec", "brotlicommon"].freeze
         # rubocop:enable Style/WordArray