Class: MilkTea::PackageRegistryStore

Inherits:
Object
  • Object
show all
Defined in:
lib/milk_tea/packages/registry_store.rb

Defined Under Namespace

Classes: Result

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(root: self.class.default_root, upstream_root: self.class.default_upstream_root) ⇒ PackageRegistryStore

Returns a new instance of PackageRegistryStore.



48
49
50
51
 
# File 'lib/milk_tea/packages/registry_store.rb', line 48

def initialize(root: self.class.default_root, upstream_root: self.class.default_upstream_root)
  @root = File.expand_path(root)
  @upstream_root = normalize_optional_root(upstream_root)
end

Instance Attribute Details

#root ⇒ Object (readonly)

Returns the value of attribute root.



46
47
48
 
# File 'lib/milk_tea/packages/registry_store.rb', line 46

def root
  @root
end

#upstream_root ⇒ Object (readonly)

Returns the value of attribute upstream_root.



46
47
48
 
# File 'lib/milk_tea/packages/registry_store.rb', line 46

def upstream_root
  @upstream_root
end

Class Method Details

.default_root(env: ENV, home: nil) ⇒ Object 



18
19
20
21
22
23
24
25
26
27
28
29
30
 
# File 'lib/milk_tea/packages/registry_store.rb', line 18

def self.default_root(env: ENV, home: nil)
  explicit_root = env.fetch("MILK_TEA_PACKAGE_REGISTRY", "").to_s.strip
  return File.expand_path(explicit_root) unless explicit_root.empty?

  xdg_data_home = env.fetch("XDG_DATA_HOME", "").to_s.strip
  base_root = if xdg_data_home.empty?
    File.join(home || Dir.home, ".local", "share")
  else
    File.expand_path(xdg_data_home)
  end

  File.join(base_root, "milk_tea", "registry")
end

.default_upstream_root(env: ENV) ⇒ Object 



32
33
34
35
36
37
 
# File 'lib/milk_tea/packages/registry_store.rb', line 32

def self.default_upstream_root(env: ENV)
  explicit_root = env.fetch("MILK_TEA_PACKAGE_REGISTRY_UPSTREAM", "").to_s.strip
  return nil if explicit_root.empty?

  http_url?(explicit_root) ? explicit_root : File.expand_path(explicit_root)
end

.http_url?(value) ⇒ Boolean

Returns:

  • (Boolean) 


39
40
41
42
43
44
 
# File 'lib/milk_tea/packages/registry_store.rb', line 39

def self.http_url?(value)
  uri = URI.parse(value.to_s)
  %w[http https].include?(uri.scheme)
rescue URI::InvalidURIError
  false
end

Instance Method Details

#add_directory_to_archive(tar, root_path, path) ⇒ Object 



276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
 
# File 'lib/milk_tea/packages/registry_store.rb', line 276

def add_directory_to_archive(tar, root_path, path)
  Dir.children(path).sort.each do |entry|
    next if entry.start_with?(".")

    child_path = File.join(path, entry)
    relative_path = child_path.delete_prefix(root_path + File::SEPARATOR)
    stat = File.lstat(child_path)
    if stat.directory?
      tar.mkdir(relative_path, stat.mode)
      add_directory_to_archive(tar, root_path, child_path)
    elsif stat.file?
      tar.add_file(relative_path, stat.mode) do |file|
        File.open(child_path, "rb") do |source|
          IO.copy_stream(source, file)
        end
      end
    end
  end
end

#archive_path_for_root(root, package_name, package_version) ⇒ Object 



184
185
186
187
 
# File 'lib/milk_tea/packages/registry_store.rb', line 184

def archive_path_for_root(root, package_name, package_version)
  package_dir = File.join(root, "packages", normalize_component(package_name, "package name"))
  File.join(package_dir, "#{normalize_component(package_version, "package version")}.tar.gz")
end

#available_versions(package_name) ⇒ Object 



79
80
81
82
83
84
85
86
87
88
89
90
91
 
# File 'lib/milk_tea/packages/registry_store.rb', line 79

def available_versions(package_name)
  versions = package_versions_for_root(@root, package_name)
  if upstream_configured?
    upstream_versions = if upstream_http?
                          package_versions_for_http(package_name)
                        else
                          package_versions_for_root(@upstream_root, package_name)
                        end
    versions = upstream_versions + versions
  end

  versions.uniq
end

#cleanup_registry_entry!(root, package_name, package_version, package_root) ⇒ Object 



346
347
348
349
 
# File 'lib/milk_tea/packages/registry_store.rb', line 346

def cleanup_registry_entry!(root, package_name, package_version, package_root)
  FileUtils.rm_rf(package_root)
  FileUtils.rm_f(archive_path_for_root(root, package_name, package_version))
end

#extract_archive_to_root(archive_body, destination_root, scratch_dir) ⇒ Object 



302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
 
# File 'lib/milk_tea/packages/registry_store.rb', line 302

def extract_archive_to_root(archive_body, destination_root, scratch_dir)
  archive_path = File.join(scratch_dir, "package.tar.gz")
  extract_root = File.join(File.dirname(archive_path), "extract")
  File.binwrite(archive_path, archive_body)
  FileUtils.mkdir_p(extract_root)

  Zlib::GzipReader.open(archive_path) do |gzip|
    Gem::Package::TarReader.new(gzip) do |tar|
      tar.each do |entry|
        relative_path = entry.full_name
        next if relative_path.nil? || relative_path.empty?

        destination_path = File.expand_path(relative_path, extract_root)
        unless destination_path == extract_root || destination_path.start_with?(extract_root + File::SEPARATOR)
          raise PackageRegistryStoreError, "registry package archive contains an invalid path #{relative_path.inspect}"
        end

        if entry.directory?
          FileUtils.mkdir_p(destination_path)
        elsif entry.file?
          FileUtils.mkdir_p(File.dirname(destination_path))
          File.open(destination_path, "wb", entry.header.mode) do |file|
            IO.copy_stream(entry, file)
          end
        end
      end
    end
  end

  File.rename(extract_root, destination_root)
end

#extract_package_version(package_or_identity, version) ⇒ Object 



158
159
160
161
162
163
164
 
# File 'lib/milk_tea/packages/registry_store.rb', line 158

def extract_package_version(package_or_identity, version)
  if package_or_identity.respond_to?(:package_name) && package_or_identity.respond_to?(:version)
    [package_or_identity.package_name, package_or_identity.version]
  else
    [package_or_identity, version]
  end
end

#http_get_response(url, redirects_remaining = 5) ⇒ Object 



367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
 
# File 'lib/milk_tea/packages/registry_store.rb', line 367

def http_get_response(url, redirects_remaining = 5)
  uri = URI.parse(url)
  response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http|
    http.request(Net::HTTP::Get.new(uri.request_uri))
  end

  if response.is_a?(Net::HTTPRedirection)
    raise PackageRegistryStoreError, "too many HTTP redirects while fetching #{url}" if redirects_remaining <= 0

    location = response["location"]
    raise PackageRegistryStoreError, "missing redirect location while fetching #{url}" if location.to_s.strip.empty?

    return http_get_response(URI.join(url, location).to_s, redirects_remaining - 1)
  end

  response
rescue URI::InvalidURIError => e
  raise PackageRegistryStoreError, "invalid upstream registry URL #{url.inspect}: #{e.message}"
rescue SocketError, IOError, SystemCallError => e
  raise PackageRegistryStoreError, "failed to fetch #{url}: #{e.message}"
end

#join_upstream_url(*segments) ⇒ Object 



359
360
361
362
363
364
365
 
# File 'lib/milk_tea/packages/registry_store.rb', line 359

def join_upstream_url(*segments)
  encoded_segments = segments.map do |segment|
    CGI.escape(segment.to_s).gsub("+", "%20")
  end
  base = @upstream_root.to_s.sub(%r{/+\z}, "")
  "#{base}/#{encoded_segments.join("/")}"
end

#manifest_path_for(package_or_identity, version = nil) ⇒ Object 



59
60
61
 
# File 'lib/milk_tea/packages/registry_store.rb', line 59

def manifest_path_for(package_or_identity, version = nil)
  File.join(package_root_for(package_or_identity, version), "package.toml")
end

#materialize_package_copy!(source_root, destination_root) ⇒ Object 



334
335
336
337
338
339
340
341
342
343
344
 
# File 'lib/milk_tea/packages/registry_store.rb', line 334

def materialize_package_copy!(source_root, destination_root)
  staging_root = "#{destination_root}.tmp.#{$$}.#{Thread.current.object_id}"

  FileUtils.rm_rf(staging_root)
  FileUtils.mkdir_p(File.dirname(destination_root))
  FileUtils.copy_entry(source_root, staging_root)
  File.rename(staging_root, destination_root)
rescue StandardError
  FileUtils.rm_rf(staging_root)
  raise
end

#normalize_component(value, label) ⇒ Object

Raises:



166
167
168
169
170
171
172
173
174
175
176
 
# File 'lib/milk_tea/packages/registry_store.rb', line 166

def normalize_component(value, label)
  text = value.to_s.strip
  raise PackageRegistryStoreError, "#{label} cannot be empty" if text.empty?

  separators = [File::SEPARATOR, File::ALT_SEPARATOR].compact
  if separators.any? { |separator| text.include?(separator) } || text == "." || text == ".."
    raise PackageRegistryStoreError, "#{label} contains an unsupported path separator: #{text.inspect}"
  end

  text
end

#normalize_optional_root(root) ⇒ Object 



394
395
396
397
398
399
 
# File 'lib/milk_tea/packages/registry_store.rb', line 394

def normalize_optional_root(root)
  text = root.to_s.strip
  return nil if text.empty?

  self.class.http_url?(text) ? text.sub(%r{/+\z}, "") : File.expand_path(text)
end

#package_archive_url(package_name, package_version) ⇒ Object 



355
356
357
 
# File 'lib/milk_tea/packages/registry_store.rb', line 355

def package_archive_url(package_name, package_version)
  join_upstream_url("packages", package_name, "#{package_version}.tar.gz")
end

#package_root_for(package_or_identity, version = nil) ⇒ Object 



53
54
55
56
57
 
# File 'lib/milk_tea/packages/registry_store.rb', line 53

def package_root_for(package_or_identity, version = nil)
  package_name, package_version = extract_package_version(package_or_identity, version)

  File.join(@root, "packages", normalize_component(package_name, "package name"), normalize_component(package_version, "package version"))
end

#package_root_for_root(root, package_or_identity, version = nil) ⇒ Object 



178
179
180
181
182
 
# File 'lib/milk_tea/packages/registry_store.rb', line 178

def package_root_for_root(root, package_or_identity, version = nil)
  package_name, package_version = extract_package_version(package_or_identity, version)

  File.join(root, "packages", normalize_component(package_name, "package name"), normalize_component(package_version, "package version"))
end

#package_versions_for_http(package_name) ⇒ Object 



209
210
211
212
213
214
215
 
# File 'lib/milk_tea/packages/registry_store.rb', line 209

def package_versions_for_http(package_name)
  response = http_get_response(package_versions_url(package_name))
  return [] if response.is_a?(Net::HTTPNotFound)
  raise_http_error!(response, "list registry package versions") unless response.is_a?(Net::HTTPSuccess)

  response.body.lines.map(&:strip).reject(&:empty?)
end

#package_versions_for_root(root, package_name) ⇒ Object 



193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
 
# File 'lib/milk_tea/packages/registry_store.rb', line 193

def package_versions_for_root(root, package_name)
  return [] if root.nil?

  package_dir = File.join(root, "packages", normalize_component(package_name, "package name"))
  return [] unless File.directory?(package_dir)

  Dir.children(package_dir).sort.each_with_object([]) do |entry, versions|
    next if entry.start_with?(".") || entry == "versions.txt" || entry.end_with?(".tar.gz")

    manifest_path = File.join(package_dir, entry, "package.toml")
    versions << entry if File.file?(manifest_path)
  end
rescue SystemCallError => e
  raise PackageRegistryStoreError, "failed to inspect registry package versions under #{package_dir}: #{e.message}"
end

#package_versions_url(package_name) ⇒ Object 



351
352
353
 
# File 'lib/milk_tea/packages/registry_store.rb', line 351

def package_versions_url(package_name)
  join_upstream_url("packages", package_name, "versions.txt")
end

#publish(path, target: :local) ⇒ Object 



93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
 
# File 'lib/milk_tea/packages/registry_store.rb', line 93

def publish(path, target: :local)
  manifest = PackageManifest.load(path)
  package_version = manifest.package_version.to_s.strip
  if package_version.empty?
    raise PackageRegistryStoreError,
          "package #{manifest.package_name} at #{manifest.manifest_path} must declare package.version before publishing"
  end

  target_root = registry_root_for_target(target)
  package_root = package_root_for_root(target_root, manifest.package_name, package_version)
  if File.exist?(package_root)
    raise PackageRegistryStoreError,
          "package #{manifest.package_name} version #{package_version} already published in registry: #{package_root}"
  end

  materialize_package_copy!(manifest.root_dir, package_root)
  write_registry_artifacts_with_rollback!(target_root, manifest.package_name, package_version, package_root)

  Result.new(package_name: manifest.package_name, version: package_version, path: package_root)
rescue PackageManifestError => e
  raise PackageRegistryStoreError, e.message
rescue SystemCallError => e
  raise PackageRegistryStoreError, "failed to publish package to #{target_root || @root}: #{e.message}"
end

#published?(package_or_identity, version = nil) ⇒ Boolean

Returns:

  • (Boolean) 


63
64
65
 
# File 'lib/milk_tea/packages/registry_store.rb', line 63

def published?(package_or_identity, version = nil)
  File.file?(manifest_path_for(package_or_identity, version))
end

#published_root_for(package_or_identity, version = nil, sync: false) ⇒ Object 



67
68
69
70
71
72
73
74
75
76
77
 
# File 'lib/milk_tea/packages/registry_store.rb', line 67

def published_root_for(package_or_identity, version = nil, sync: false)
  package_name, package_version = extract_package_version(package_or_identity, version)
  sync(package_name, package_version) if sync && !published?(package_name, package_version) && upstream_configured?
  manifest_path = manifest_path_for(package_name, package_version)
  unless File.file?(manifest_path)
    raise PackageRegistryStoreError,
          "registry package #{package_name} version #{package_version} not found in #{@root}"
  end

  package_root_for(package_name, package_version)
end

#raise_http_error!(response, action) ⇒ Object

Raises:



389
390
391
392
 
# File 'lib/milk_tea/packages/registry_store.rb', line 389

def raise_http_error!(response, action)
  raise PackageRegistryStoreError,
        "failed to #{action} from upstream registry #{@upstream_root}: HTTP #{response.code} #{response.message}"
end

#registry_root_for_target(target) ⇒ Object 



401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
 
# File 'lib/milk_tea/packages/registry_store.rb', line 401

def registry_root_for_target(target)
  case target
  when :local
    @root
  when :upstream
    raise PackageRegistryStoreError, "registry upstream is not configured" unless upstream_configured?
    if upstream_http?
      raise PackageRegistryStoreError,
            "publishing directly to an HTTP registry upstream is not supported; publish to a filesystem mirror instead"
    end

    @upstream_root
  else
    raise PackageRegistryStoreError, "unknown registry publish target #{target.inspect}"
  end
end

#sync(package_or_identity, version = nil) ⇒ Object 



118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
 
# File 'lib/milk_tea/packages/registry_store.rb', line 118

def sync(package_or_identity, version = nil)
  package_name, package_version = extract_package_version(package_or_identity, version)
  raise PackageRegistryStoreError, "registry upstream is not configured" unless upstream_configured?

  local_root = package_root_for(package_name, package_version)
  return Result.new(package_name:, version: package_version, path: local_root) if published?(package_name, package_version)

  return sync_from_http(package_name, package_version, local_root) if upstream_http?

  upstream_package_root = package_root_for_root(@upstream_root, package_name, package_version)
  upstream_manifest_path = File.join(upstream_package_root, "package.toml")
  unless File.file?(upstream_manifest_path)
    raise PackageRegistryStoreError,
          "registry package #{package_name} version #{package_version} not found in upstream registry #{@upstream_root}"
  end

  upstream_manifest = PackageManifest.load(upstream_package_root)
  if upstream_manifest.package_name != package_name || upstream_manifest.package_version != package_version
    raise PackageRegistryStoreError,
          "upstream registry package at #{upstream_manifest.manifest_path} resolved to #{upstream_manifest.package_name}@#{upstream_manifest.package_version.inspect}; expected #{package_name}@#{package_version}"
  end

  materialize_package_copy!(upstream_package_root, local_root)
  write_registry_artifacts_with_rollback!(@root, package_name, package_version, local_root)

  Result.new(package_name:, version: package_version, path: local_root)
rescue PackageManifestError => e
  raise PackageRegistryStoreError, e.message
rescue SystemCallError => e
  raise PackageRegistryStoreError, "failed to sync package from upstream registry #{@upstream_root}: #{e.message}"
end

#sync_from_http(package_name, package_version, local_root) ⇒ Object 



217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
 
# File 'lib/milk_tea/packages/registry_store.rb', line 217

def sync_from_http(package_name, package_version, local_root)
  response = http_get_response(package_archive_url(package_name, package_version))
  if response.is_a?(Net::HTTPNotFound)
    raise PackageRegistryStoreError,
          "registry package #{package_name} version #{package_version} not found in upstream registry #{@upstream_root}"
  end
  raise_http_error!(response, "download registry package archive") unless response.is_a?(Net::HTTPSuccess)

  FileUtils.rm_rf(local_root) if File.exist?(local_root)
  FileUtils.mkdir_p(File.dirname(local_root))

  Dir.mktmpdir("milk-tea-http-registry", File.dirname(local_root)) do |tmpdir|
    extract_archive_to_root(response.body, local_root, tmpdir)
  end

  validate_synced_package!(local_root, package_name, package_version)
  write_registry_artifacts_with_rollback!(@root, package_name, package_version, local_root)

  Result.new(package_name:, version: package_version, path: local_root)
rescue PackageManifestError => e
  raise PackageRegistryStoreError, e.message
rescue SystemCallError => e
  raise PackageRegistryStoreError, "failed to sync package from upstream registry #{@upstream_root}: #{e.message}"
end

#upstream_configured? ⇒ Boolean

Returns:

  • (Boolean) 


150
151
152
 
# File 'lib/milk_tea/packages/registry_store.rb', line 150

def upstream_configured?
  !@upstream_root.nil?
end

#upstream_http? ⇒ Boolean

Returns:

  • (Boolean) 


154
155
156
 
# File 'lib/milk_tea/packages/registry_store.rb', line 154

def upstream_http?
  self.class.http_url?(@upstream_root)
end

#validate_synced_package!(package_root, package_name, package_version) ⇒ Object 



242
243
244
245
246
247
248
 
# File 'lib/milk_tea/packages/registry_store.rb', line 242

def validate_synced_package!(package_root, package_name, package_version)
  manifest = PackageManifest.load(package_root)
  if manifest.package_name != package_name || manifest.package_version != package_version
    raise PackageRegistryStoreError,
          "upstream registry package at #{manifest.manifest_path} resolved to #{manifest.package_name}@#{manifest.package_version.inspect}; expected #{package_name}@#{package_version}"
  end
end

#versions_index_path_for_root(root, package_name) ⇒ Object 



189
190
191
 
# File 'lib/milk_tea/packages/registry_store.rb', line 189

def versions_index_path_for_root(root, package_name)
  File.join(root, "packages", normalize_component(package_name, "package name"), "versions.txt")
end

#write_package_archive!(archive_path, package_root) ⇒ Object 



262
263
264
265
266
267
268
269
270
271
272
273
274
 
# File 'lib/milk_tea/packages/registry_store.rb', line 262

def write_package_archive!(archive_path, package_root)
  tar_buffer = StringIO.new(+"")
  Gem::Package::TarWriter.new(tar_buffer) do |tar|
    add_directory_to_archive(tar, package_root, package_root)
  end
  tar_buffer.rewind

  PackageAtomicWrite.open(archive_path, binmode: true) do |file|
    gzip = Zlib::GzipWriter.new(file)
    gzip.write(tar_buffer.string)
    gzip.finish
  end
end

#write_registry_artifacts!(root, package_name, package_version, package_root) ⇒ Object 



250
251
252
253
 
# File 'lib/milk_tea/packages/registry_store.rb', line 250

def write_registry_artifacts!(root, package_name, package_version, package_root)
  write_package_archive!(archive_path_for_root(root, package_name, package_version), package_root)
  write_versions_index!(root, package_name)
end

#write_registry_artifacts_with_rollback!(root, package_name, package_version, package_root) ⇒ Object 



255
256
257
258
259
260
 
# File 'lib/milk_tea/packages/registry_store.rb', line 255

def write_registry_artifacts_with_rollback!(root, package_name, package_version, package_root)
  write_registry_artifacts!(root, package_name, package_version, package_root)
rescue StandardError
  cleanup_registry_entry!(root, package_name, package_version, package_root)
  raise
end

#write_versions_index!(root, package_name) ⇒ Object 



296
297
298
299
300
 
# File 'lib/milk_tea/packages/registry_store.rb', line 296

def write_versions_index!(root, package_name)
  path = versions_index_path_for_root(root, package_name)
  versions = package_versions_for_root(root, package_name)
  PackageAtomicWrite.write(path, versions.join("\n") + (versions.empty? ? "" : "\n"))
end