Module: Tebako::Packager

Defined in:
lib/tebako/packager.rb,
lib/tebako/packager/patch.rb,
lib/tebako/packager/patch_main.rb,
lib/tebako/packager/pass1_patch.rb,
lib/tebako/packager/pass2_patch.rb,
lib/tebako/packager/pass1a_patch.rb,
lib/tebako/packager/patch_helpers.rb,
lib/tebako/packager/patch_literals.rb,
lib/tebako/packager/rubygems_patch.rb,
lib/tebako/packager/pass2_patch_crt.rb,
lib/tebako/packager/pass2msys_patch.rb,
lib/tebako/packager/patch_libraries.rb,
lib/tebako/packager/patch_buildsystem.rb

Overview

Packager module

Defined Under Namespace

Modules: PatchBuildsystem, PatchHelpers, PatchLibraries, PatchLiterals, PatchMain Classes: Pass1APatch, Pass1DarwinPatch, Pass1MSysPatch, Pass1Patch, Pass2MSysPatch, Pass2NonMSysPatch, Pass2Patch, Patch, RubygemsPatch, RubygemsUpdatePatch

Constant Summary collapse

FILES_TO_RESTORE = 
%w[
  common.mk
  configure
  config.status
  dir.c
  dln.c
  file.c
  gem_prelude.rb
  io.c
  main.c
  Makefile
  ruby.c
  thread_pthread.c
  util.c
  ext/bigdecimal/bigdecimal.h
  ext/Setup
  cygwin/GNUmakefile.in
  include/ruby/onigmo.h
  lib/rubygems/openssl.rb
  lib/rubygems/path_support.rb
  template/Makefile.in
  tool/mkconfig.rb
  win32/winmain.c
  win32/file.c
].freeze
GNUMAKEFILE_IN_DLLTOOL_SUBST =

Substitutions for cygwin/Gnumakefile.in shared across Pass1 and Pass2

<<~SUBST
  $(Q) dlltool --output-lib=$(LIBRUBY) --output-def=tebako.def --export-all $(LIBRUBY_A) --output-exp=$(RUBY_EXP)   # tebako patched
SUBST
GNUMAKEFILE_IN_WINMAIN_SUBST =

This MSYS specific thing ensure compilation of winmain.c Did try to understand why it did not work out of the box

<<~SUBST
  RUBYDEF = $(DLL_BASE_NAME).def

  # Start of tebako patch
  WINMAINOBJ    = win32/winmain.$(OBJEXT)
  $(WINMAINOBJ): win32/winmain.c
  # End of tebako patch
SUBST
{
  "$(MAINOBJ) $(DLDOBJS) $(LIBRUBY_A) $(MAINLIBS) $(EXTLIBS) $(LIBS) $(OUTFLAG)$@" =>
    "$(MAINOBJ) $(DLDOBJS) $(LIBRUBY_A) -Wl,--start-group $(MAINLIBS) $(EXTLIBS) $(LIBS) " \
    "-Wl,--end-group $(OUTFLAG)$@ # tebako patched"
}.freeze
{
  "$(LIBRUBYARG) $(MAINLIBS) $(EXTLIBS) $(OUTFLAG)$@" =>
    "$(LIBRUBYARG) -Wl,--start-group $(MAINLIBS) $(EXTLIBS) -Wl,--end-group $(OUTFLAG)$@ " \
    "# tebako patched",
  "$(LIBRUBYARG) $(MAINLIBS) $(LIBS) $(EXTLIBS) $(OUTFLAG)$@" =>
    "$(LIBRUBYARG) -Wl,--start-group $(MAINLIBS) $(LIBS) $(EXTLIBS) -Wl,--end-group " \
    "$(OUTFLAG)$@ # tebako patched"
}.freeze
IO_C_MSYS_BASE_PATCH = 
{
  "#define open	rb_w32_uopen" => "#define open(p, f, m) tebako_open(3, (p), (f), (m))"
}.freeze
IO_C_MSYS_PATCH_PRE_32 = 
{
  "(rb_w32_io_cancelable_p((fptr)->fd) ? Qnil : rb_io_wait(fptr->self, RB_INT2NUM(RUBY_IO_READABLE), Qnil))" =>
      "((is_tebako_file_descriptor((fptr)->fd) || rb_w32_io_cancelable_p((fptr)->fd)) ? \\\n" \
      "Qnil : rb_io_wait(fptr->self, RB_INT2NUM(RUBY_IO_READABLE), Qnil))"
}.freeze
IO_C_MSYS_PATCH = 
{
  "(rb_w32_io_cancelable_p((fptr)->fd) ? Qnil : rb_io_wait(fptr->self, " \
  "RB_INT2NUM(RUBY_IO_READABLE), RUBY_IO_TIMEOUT_DEFAULT))" =>
      "((is_tebako_file_descriptor((fptr)->fd) || rb_w32_io_cancelable_p((fptr)->fd)) ? \\\n" \
      "Qnil : rb_io_wait(fptr->self, RB_INT2NUM(RUBY_IO_READABLE), RUBY_IO_TIMEOUT_DEFAULT))"
}.freeze
RUBY_C_MSYS_PATH_SUBST = 
<<~SUBST
  /* -- Start of tebako patch -- */
          VALUE path = within_tebako_memfs(paths) ?
                          rb_str_new_cstr(paths) :
                          RUBY_RELATIVE(paths, len);
          /* -- End of tebako patch -- */
SUBST
RUBY_C_MSYS_PATCHES = 
{
  "#ifndef MAXPATHLEN" => "#{PatchLiterals::C_FILE_SUBST_LESS}\n#ifndef MAXPATHLEN",
  "VALUE path = RUBY_RELATIVE(paths, len);" => RUBY_C_MSYS_PATH_SUBST
}.freeze
WIN32_FILE_C_MSYS_SUBST = 
<<~SUBST
  /* -- Start of tebako patch -- */
    if (tebako_file_load_ok(path)) return ret;
      /* -- End of tebako patch -- */
  wpath = mbstr_to_wstr(CP_UTF8, path, -1, &len);
SUBST
WIN32_FILE_C_MSYS_PATCHES = 
{
  "#ifndef INVALID_FILE_ATTRIBUTES" => "#{PatchLiterals::C_FILE_SUBST_LESS}\n#ifndef INVALID_FILE_ATTRIBUTES",
  "wpath = mbstr_to_wstr(CP_UTF8, path, -1, &len);" => WIN32_FILE_C_MSYS_SUBST
}.freeze
WIN32_WIN32_C_MSYS_SUBST = 
<<~SUBST
  /* -- Start of tebako patch -- */
    if (is_tebako_cwd()) {
      char* tebako_cwd = tebako_getcwd(NULL,0);
      if (tebako_cwd == NULL) {
        errno = ENOMEM;
        return NULL;
      }
      len = strlen(tebako_cwd) + 1;

      if (buffer) {
        if (size < len) {
          free(tebako_cwd);
          errno = ERANGE;
          return NULL;
        }
      }
      else {
        buffer = (*alloc)(len, arg);
        if (!buffer) {
          free(tebako_cwd);
          errno = ENOMEM;
          return NULL;
        }
      }
      translate_char(tebako_cwd, (char)0x5c, '/', CP_UTF8);
      strcpy(buffer, tebako_cwd);
      free(tebako_cwd);
      return buffer;
    }
    /* -- End of tebako patch -- */

    len = GetCurrentDirectoryW(0, NULL);
SUBST
WIN32_WIN32_C_MSYS_PATCHES = 
{
  "#if defined _MSC_VER && _MSC_VER <= 1200" =>
    "#{PatchLiterals::C_FILE_SUBST_LESS}\n#if defined _MSC_VER && _MSC_VER <= 1200",
  "len = GetCurrentDirectoryW(0, NULL);" => WIN32_WIN32_C_MSYS_SUBST,
  # Keep the clock renames in sync with Pass1MSysPatch::WIN32_WIN32_C_CLOCK_PATCH:
  # pass1 applies them for the toolchain build, this pass re-applies them
  # from pristine sources for the final build
  "clock_gettime(clockid_t clock_id, struct timespec *sp)" =>
    "_dummy_clock_gettime(clockid_t clock_id, struct timespec *sp)",
  "clock_getres(clockid_t clock_id, struct timespec *sp)" =>
    "_dummy_clock_getres(clockid_t clock_id, struct timespec *sp)"
}.freeze

Class Method Summary collapse

Class Method Details

.align_gem_ext_arch!(data_src_dir, layout_dir, api_ver) ⇒ Object

Native-gem extensions dir uses the dashed flavor of the arch string (arm64-darwin-23); align it to the runtime's as well



173
174
175
176
177
178
179
180
181
182
183
184
 
# File 'lib/tebako/packager.rb', line 173

def align_gem_ext_arch!(data_src_dir, layout_dir, api_ver)
  img_ext = File.join(data_src_dir, "lib", "ruby", "gems", api_ver, "extensions")
  rt_ext = File.join(layout_dir, "lib", "ruby", "gems", api_ver, "extensions")
  runtime_ext = first_dir(rt_ext)
  return if runtime_ext.nil? || !Dir.exist?(img_ext)

  Dir.children(img_ext).each do |d|
    next if d == runtime_ext || !File.directory?(File.join(img_ext, d))

    FileUtils.mv(File.join(img_ext, d), File.join(img_ext, runtime_ext))
  end
end

.align_layout_to_runtime!(data_src_dir, layout_dir, ruby_ver) ⇒ Object

Rename the image's arch directories to the runtime's names and drop in the runtime's own rbconfig.rb. No-op when the conventions already match (always the case off macOS, where the arch string carries no OS version).



154
155
156
157
 
# File 'lib/tebako/packager.rb', line 154

def align_layout_to_runtime!(data_src_dir, layout_dir, ruby_ver)
  align_stdlib_arch!(data_src_dir, layout_dir, ruby_ver.api_version)
  align_gem_ext_arch!(data_src_dir, layout_dir, ruby_ver.api_version)
end

.align_stdlib_arch!(data_src_dir, layout_dir, api_ver) ⇒ Object 



159
160
161
162
163
164
165
166
167
168
169
 
# File 'lib/tebako/packager.rb', line 159

def align_stdlib_arch!(data_src_dir, layout_dir, api_ver)
  runtime_arch = arch_dir_of(File.join(layout_dir, "lib", "ruby", api_ver), "rbconfig.rb")
  image_arch = arch_dir_of(File.join(data_src_dir, "lib", "ruby", api_ver), "rbconfig.rb")
  return if runtime_arch.nil? || image_arch.nil? || runtime_arch == image_arch

  puts "   ... aligning app image layout to the runtime (#{image_arch} -> #{runtime_arch})"
  FileUtils.mv(File.join(data_src_dir, "lib", "ruby", api_ver, image_arch),
               File.join(data_src_dir, "lib", "ruby", api_ver, runtime_arch))
  FileUtils.cp(File.join(layout_dir, "lib", "ruby", api_ver, runtime_arch, "rbconfig.rb"),
               File.join(data_src_dir, "lib", "ruby", api_ver, runtime_arch, "rbconfig.rb"))
end

.arch_dir_of(dir, marker) ⇒ Object 



186
187
188
189
190
 
# File 'lib/tebako/packager.rb', line 186

def arch_dir_of(dir, marker)
  return nil unless Dir.exist?(dir)

  Dir.children(dir).find { |d| File.exist?(File.join(dir, d, marker)) }
end

.build_app_image(options_manager, scenario_manager, layout_dir = nil) ⇒ Object

Deploy the application and build its DwarFS image for stitching onto a prebuilt runtime. Same layout as the bundle-mode image, plus an entry dispatcher at /local/stub.rb (the runtime's compiled-in entry). When layout_dir is given (the resolved runtime's extracted layout), the image's arch conventions are aligned to the runtime's: ruby's compiled-in search paths come from the runtime build, while the local packaging environment names arch directories after the press machine -- on macOS the arch string embeds the kernel version (arm64-darwin23 vs arm64-darwin24), so they differ whenever press and runtime were built on different macOS releases.



139
140
141
142
143
144
145
146
147
148
 
# File 'lib/tebako/packager.rb', line 139

def build_app_image(options_manager, scenario_manager, layout_dir = nil) # rubocop:disable Metrics/AbcSize
  init(options_manager.stash_dir, options_manager.data_src_dir, options_manager.data_pre_dir,
       options_manager.data_bin_dir)
  deploy(options_manager.data_src_dir, options_manager.data_pre_dir, options_manager.rv,
         options_manager.root, scenario_manager.fs_entrance, options_manager.cwd)
  align_layout_to_runtime!(options_manager.data_src_dir, layout_dir, options_manager.rv) if layout_dir
  write_entry_dispatcher(options_manager.data_src_dir, scenario_manager, options_manager.cwd)
  mkdwarfs(options_manager.deps_bin_dir, options_manager.data_bundle_file, options_manager.data_src_dir)
  options_manager.data_bundle_file
end

.check_prebuilt_env!(stash_dir, deps_bin_dir) ⇒ Object

Check that the packaging environment the prebuilt-runtime press needs (pristine Ruby stash + mkdwarfs) is in place



120
121
122
123
124
125
126
127
 
# File 'lib/tebako/packager.rb', line 120

def check_prebuilt_env!(stash_dir, deps_bin_dir)
  missing = []
  missing << "Ruby environment stash (#{stash_dir})" unless Dir.exist?(stash_dir)
  missing << "mkdwarfs (#{deps_bin_dir})" if Dir.glob(File.join(deps_bin_dir, "mkdwarfs*")).empty?
  return if missing.empty?

  Tebako.packaging_error(128, missing.join(", "))
end

.create_implib(src_dir, package_src_dir, app_name, ruby_ver) ⇒ Object

Create implib



73
74
75
76
77
78
79
80
 
# File 'lib/tebako/packager.rb', line 73

def create_implib(src_dir, package_src_dir, app_name, ruby_ver)
  a_name = File.basename(app_name, ".*")
  create_def(src_dir, a_name)
  puts "   ... creating Windows import library"
  params = ["dlltool", "-d", def_fname(src_dir, a_name), "-D", out_fname(a_name), "--output-lib",
            lib_fname(package_src_dir, ruby_ver)]
  BuildHelpers.run_with_capture(params)
end

.crt_pass1_patch(ostype, mount_point, ruby_ver) ⇒ Object 



38
39
40
41
42
43
44
45
46
47
 
# File 'lib/tebako/packager/pass1_patch.rb', line 38

def crt_pass1_patch(ostype, mount_point, ruby_ver)
  scmb = ScenarioManagerBase.new(ostype)
  if scmb.macos?
    Pass1DarwinPatch.new(mount_point, ruby_ver)
  elsif scmb.msys?
    Pass1MSysPatch.new(mount_point, ruby_ver)
  else
    Pass1Patch.new(mount_point, ruby_ver)
  end
end

.crt_pass2_patch(ostype, deps_lib_dir, ruby_ver) ⇒ Object 



36
37
38
39
40
41
42
43
 
# File 'lib/tebako/packager/pass2_patch_crt.rb', line 36

def crt_pass2_patch(ostype, deps_lib_dir, ruby_ver)
  scmb = ScenarioManagerBase.new(ostype)
  if scmb.msys?
    Pass2MSysPatch.new(ostype, deps_lib_dir, ruby_ver)
  else
    Pass2NonMSysPatch.new(ostype, deps_lib_dir, ruby_ver)
  end
end

.deploy(target_dir, pre_dir, ruby_ver, fs_root, fs_entrance, cwd) ⇒ Object

Deploy



83
84
85
86
87
88
89
90
 
# File 'lib/tebako/packager.rb', line 83

def deploy(target_dir, pre_dir, ruby_ver, fs_root, fs_entrance, cwd) # rubocop:disable Metrics/ParameterLists
  puts "-- Running deploy script"

  deploy_helper = Tebako::DeployHelper.new(fs_root, fs_entrance, target_dir, pre_dir)
  deploy_helper.configure(ruby_ver, cwd)
  deploy_helper.deploy
  Tebako::Stripper.strip(deploy_helper, target_dir)
end

.do_patch(patch_map, root) ⇒ Object 



92
93
94
 
# File 'lib/tebako/packager.rb', line 92

def do_patch(patch_map, root)
  patch_map.each { |fname, mapping| PatchHelpers.patch_file("#{root}/#{fname}", mapping) }
end

.finalize(src_dir, app_name, ruby_ver, patchelf, output_type) ⇒ Object 



96
97
98
99
100
101
102
103
104
105
106
107
 
# File 'lib/tebako/packager.rb', line 96

def finalize(src_dir, app_name, ruby_ver, patchelf, output_type)
  puts "-- Running finalize script"

  RubyBuilder.new(ruby_ver, src_dir).target_build(output_type)
  exe_suffix = ScenarioManagerBase.new.exe_suffix
  src_name = File.join(src_dir, "ruby#{exe_suffix}")
  patchelf(src_name, patchelf)
  package_name = "#{app_name}#{exe_suffix}"
  # strip_or_copy(os_type, src_name, package_name)
  Tebako::Stripper.strip_file(src_name, package_name)
  puts "Created tebako #{output_type} at \"#{package_name}\""
end

.first_dir(dir) ⇒ Object 



192
193
194
195
196
 
# File 'lib/tebako/packager.rb', line 192

def first_dir(dir)
  return nil unless Dir.exist?(dir)

  Dir.children(dir).find { |d| File.directory?(File.join(dir, d)) }
end

.init(stash_dir, src_dir, pre_dir, bin_dir) ⇒ Object

Init



110
111
112
113
114
115
116
 
# File 'lib/tebako/packager.rb', line 110

def init(stash_dir, src_dir, pre_dir, bin_dir)
  puts "-- Running init script"

  puts "   ... creating packaging environment at #{src_dir}"
  PatchHelpers.recreate([src_dir, pre_dir, bin_dir])
  FileUtils.cp_r "#{stash_dir}/.", src_dir
end

.mkdwarfs(deps_bin_dir, data_bin_file, data_src_dir, descriptor = nil) ⇒ Object 



213
214
215
216
217
218
219
 
# File 'lib/tebako/packager.rb', line 213

def mkdwarfs(deps_bin_dir, data_bin_file, data_src_dir, descriptor = nil)
  puts "-- Running mkdwarfs script"
  FileUtils.chmod("a+x", Dir.glob(File.join(deps_bin_dir, "mkdwarfs*")))
  params = [File.join(deps_bin_dir, "mkdwarfs"), "-o", data_bin_file, "-i", data_src_dir, "--no-progress"]
  params << "--header" << descriptor if descriptor
  BuildHelpers.run_with_capture_v(params)
end

.pass1(ostype, ruby_source_dir, mount_point, src_dir, ruby_ver) ⇒ Object

Pass1 Executed before Ruby build, patching ensures that Ruby itself is linked statically



223
224
225
226
227
228
229
230
231
232
233
 
# File 'lib/tebako/packager.rb', line 223

def pass1(ostype, ruby_source_dir, mount_point, src_dir, ruby_ver)
  puts "-- Running pass1 script"
  PatchHelpers.recreate(src_dir)

  # Roll all known patches
  # Just in case we are recovering after some error
  PatchHelpers.restore_and_save_files(FILES_TO_RESTORE, ruby_source_dir, strict: false)

  patch = crt_pass1_patch(ostype, mount_point, ruby_ver)
  do_patch(patch.patch_map, ruby_source_dir)
end

.pass1a(ruby_source_dir) ⇒ Object

Pass1A Patch gem_prelude.rb



237
238
239
240
241
 
# File 'lib/tebako/packager.rb', line 237

def pass1a(ruby_source_dir)
  puts "-- Running pass1a script"
  patch = Pass1APatch.new
  do_patch(patch.patch_map, ruby_source_dir)
end

.pass2(ostype, ruby_source_dir, deps_lib_dir, ruby_ver) ⇒ Object

Pass2 Creates packaging environment, patching ensures that tebako package is linked statically



245
246
247
248
249
250
 
# File 'lib/tebako/packager.rb', line 245

def pass2(ostype, ruby_source_dir, deps_lib_dir, ruby_ver)
  puts "-- Running pass2 script"

  patch = crt_pass2_patch(ostype, deps_lib_dir, ruby_ver)
  do_patch(patch.patch_map, ruby_source_dir)
end

.stash(src_dir, stash_dir, ruby_source_dir, ruby_ver) ⇒ Object

Stash Created and saves pristine Ruby environment that is used to deploy applications for packaging



254
255
256
257
258
259
260
261
 
# File 'lib/tebako/packager.rb', line 254

def stash(src_dir, stash_dir, ruby_source_dir, ruby_ver)
  puts "-- Running stash script"
  RubyBuilder.new(ruby_ver, ruby_source_dir).toolchain_build

  puts "   ... saving pristine Ruby environment to #{stash_dir}"
  PatchHelpers.recreate(stash_dir)
  FileUtils.cp_r "#{src_dir}/.", stash_dir
end

.write_entry_dispatcher(data_src_dir, scenario_manager, cwd) ⇒ Object

The prebuilt runtime packages are pressed in 'runtime' mode, so their compiled-in entry point is /local/stub.rb. A stitched package mounts the application image as the root filesystem; the dispatcher written here receives control from the runtime and loads the real entry point (replicating the bundle-mode working directory when --cwd was given).



203
204
205
206
207
208
209
210
211
 
# File 'lib/tebako/packager.rb', line 203

def write_entry_dispatcher(data_src_dir, scenario_manager, cwd)
  dispatcher = +""
  dispatcher << "Dir.chdir(\"#{scenario_manager.fs_mount_point}/#{cwd}\")\n" unless cwd.nil?
  dispatcher << "load \"#{scenario_manager.fs_mount_point}#{scenario_manager.fs_entry_point}\"\n"

  local_dir = File.join(data_src_dir, "local")
  FileUtils.mkdir_p(local_dir)
  File.write(File.join(local_dir, "stub.rb"), dispatcher)
end