Class: Kamal::Cli::Proxy

Inherits:
Base
  • Object
show all
Defined in:
lib/kamal/cli/proxy.rb

Defined Under Namespace

Classes: Drift, LoadbalancerClaim, LoadbalancerReboot, Reboot

Constant Summary

Constants inherited from Base

Base::AUTOMATIC_DEPLOY_LOCK_MESSAGE, Base::HOOK_MUTEX, Base::VERBOSITY

Instance Method Summary collapse

Methods inherited from Base

dynamic_command_class, exit_on_failure?, #initialize

Constructor Details

This class inherits a constructor from Kamal::Cli::Base

Instance Method Details

#boot ⇒ Object 



3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
 
# File 'lib/kamal/cli/proxy.rb', line 3

def boot
  modify(lock: true) do
    on(KAMAL.hosts) do |host|
      execute *KAMAL.docker.create_network
    rescue SSHKit::Command::Failed => e
      raise unless e.message.include?("already exists")
    end

    # Skip proxy on loadbalancer host - the loadbalancer will handle it
    proxy_hosts = KAMAL.proxy_hosts
    if KAMAL.config.proxy.loadbalancer_on_proxy_host?
      proxy_hosts = proxy_hosts - [ KAMAL.config.proxy.effective_loadbalancer ]
    end

    drifted_hosts = Concurrent::Array.new
    stale_hosts = Concurrent::Array.new
    auto_reboot = KAMAL.config.proxy.reboot_on_deploy?

    on(proxy_hosts) do |host|
      execute *KAMAL.registry.

      proxy = KAMAL.proxy(host)
      drift = Kamal::Cli::Proxy::Drift.new(host, self)

      if drift.drifted? && auto_reboot
        # Leave the old proxy serving until its serial reboot slot below.
        drifted_hosts << host.to_s
      else
        stale_hosts << host.to_s if drift.drifted?

        version = capture_with_info(*proxy.version).strip.presence

        if version && Kamal::Utils.older_version?(version, Kamal::Configuration::Proxy::Run::MINIMUM_VERSION)
          raise "kamal-proxy version #{version} is too old, run `kamal proxy reboot` in order to update to at least #{Kamal::Configuration::Proxy::Run::MINIMUM_VERSION}"
        end

        if (run_config = proxy.proxy_run_config)&.secrets?
          execute *proxy.ensure_proxy_directory
          upload! run_config.secrets_io, run_config.secrets_path, mode: "0600"
        else
          # A host keeps no secrets it no longer needs.
          execute *proxy.remove_proxy_secrets_file, raise_on_non_zero_exit: false
        end

        execute *proxy.ensure_apps_config_directory
        execute *proxy.start_holder_or_run if proxy.port_holder?
        execute *proxy.start_or_run(digest: drift.expected_digest)
      end
    end

    if stale_hosts.any?
      say "kamal-proxy on #{stale_hosts.sort.join(", ")} is running with a configuration that no longer matches the deploy config. " \
          "Automatic reboot is disabled (proxy: reboot_on_deploy: false) - run `kamal proxy reboot` to apply the new configuration.", :yellow
    end

    drifted_hosts.sort.each do |host|
      say "kamal-proxy configuration changed, rebooting on #{host}...", :magenta
      run_hook "pre-proxy-reboot", hosts: host
      on(host) do |h|
        Kamal::Cli::Proxy::Reboot.new(h, self).run
      end
      run_hook "post-proxy-reboot", hosts: host
    end

    if KAMAL.config.proxy.load_balancing?
      lb_drifted = Concurrent::Array.new
      lb_stale = Concurrent::Array.new

      on(KAMAL.config.proxy.effective_loadbalancer) do |host|
        info "Starting loadbalancer on #{host}..."
        execute *KAMAL.registry.

        # The load balancer terminates TLS and owns the cache, so its host
        # needs the proxy secrets (acme credentials, cache store) just like
        # the proxy hosts do.
        if (lb_run = KAMAL.loadbalancer_config.run).secrets?
          execute *KAMAL.loadbalancer.ensure_proxy_directory
          upload! lb_run.secrets_io, lb_run.secrets_path, mode: "0600"
        else
          execute *KAMAL.loadbalancer.remove_proxy_secrets_file, raise_on_non_zero_exit: false
        end

        execute *KAMAL.loadbalancer.ensure_apps_config_directory

        # TLS terminates at the load balancer, so the TLS material the app
        # hosts get - custom certificates and the mTLS client CA - must
        # reach this host too; the LB container reads it through the same
        # apps-config mount the per-host proxies use.
        KAMAL.config.roles.select(&:running_proxy?).each do |role|
          Kamal::Cli::App::SslCertificates.new(host, role, self).run
        end

        # The same drift detection the proxy hosts get: a loadbalancer booted
        # with a different config digest reboots below (or warns, when
        # automatic reboot is off) instead of serving a stale config forever.
        container_id = capture_with_info(*KAMAL.loadbalancer.container_id, raise_on_non_zero_exit: false).strip
        current_digest = capture_with_info(*KAMAL.loadbalancer.config_digest, raise_on_non_zero_exit: false).strip

        if container_id.present? && current_digest != KAMAL.loadbalancer_config.run_config_digest
          if auto_reboot
            # Leave the old loadbalancer serving until its reboot below.
            lb_drifted << host.to_s
          else
            lb_stale << host.to_s
            execute *KAMAL.loadbalancer.start_or_run
          end
        else
          Kamal::Cli::Proxy::LoadbalancerClaim.new(host, self).claim_run_config
          execute *KAMAL.loadbalancer.start_or_run
        end
      end

      if lb_stale.any?
        say "The loadbalancer on #{lb_stale.sort.join(", ")} is running with a configuration that no longer matches the deploy config. " \
            "Automatic reboot is disabled (proxy: reboot_on_deploy: false) - run `kamal proxy reboot` to apply the new configuration.", :yellow
      end

      lb_drifted.sort.each do |host|
        say "Loadbalancer configuration changed, rebooting on #{host}...", :magenta
        run_hook "pre-loadbalancer-reboot", hosts: host
        on(host) do |h|
          Kamal::Cli::Proxy::LoadbalancerReboot.new(h, self).run
        end
        run_hook "post-loadbalancer-reboot", hosts: host
      end
    end
  end
end

#boot_config(subcommand) ⇒ Object 



144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
 
# File 'lib/kamal/cli/proxy.rb', line 144

def boot_config(subcommand)
  say "The proxy boot_config command is deprecated - set the config in the deploy YAML at proxy/run instead", :yellow
  proxy_boot_config = KAMAL.config.proxy_boot

  case subcommand
  when "set"
    boot_options = [
      *(proxy_boot_config.publish_args(options[:http_port], options[:https_port], options[:publish_host_ip]) if options[:publish]),
      *(proxy_boot_config.logging_args(options[:log_max_size])),
      *("--expose=#{options[:metrics_port]}" if options[:metrics_port]),
      *options[:docker_options].map { |option| "--#{option}" }
    ]

    image = [
      options[:registry].presence,
      options[:repository].presence || proxy_boot_config.repository_name,
      proxy_boot_config.image_name
    ].compact.join("/")

    image_version = options[:image_version]

    run_command_options = { debug: options[:debug] || nil, "metrics-port": options[:metrics_port] }.compact
    run_command = "kamal-proxy run #{Kamal::Utils.optionize(run_command_options).join(" ")}" if run_command_options.any?

    on(KAMAL.proxy_hosts) do |host|
      proxy = KAMAL.proxy(host)
      execute(*proxy.ensure_proxy_directory)
      if boot_options != proxy_boot_config.default_boot_options
        upload! StringIO.new(boot_options.join(" ")), proxy_boot_config.options_file
      else
        execute *proxy.reset_boot_options, raise_on_non_zero_exit: false
      end

      if image != proxy_boot_config.image_default
        upload! StringIO.new(image), proxy_boot_config.image_file
      else
        execute *proxy.reset_image, raise_on_non_zero_exit: false
      end

      if image_version
        upload! StringIO.new(image_version), proxy_boot_config.image_version_file
      else
        execute *proxy.reset_image_version, raise_on_non_zero_exit: false
      end

      if run_command
        upload! StringIO.new(run_command), proxy_boot_config.run_command_file
      else
        execute *proxy.reset_run_command, raise_on_non_zero_exit: false
      end
    end
  when "get"

    on(KAMAL.proxy_hosts) do |host|
      puts "Host #{host}: #{capture_with_info(*KAMAL.proxy(host).boot_config)}"
    end
  when "reset"
    on(KAMAL.proxy_hosts) do |host|
      proxy = KAMAL.proxy(host)
      execute *proxy.reset_boot_options, raise_on_non_zero_exit: false
      execute *proxy.reset_image, raise_on_non_zero_exit: false
      execute *proxy.reset_image_version, raise_on_non_zero_exit: false
      execute *proxy.reset_run_command, raise_on_non_zero_exit: false
    end
  else
    raise ArgumentError, "Unknown boot_config subcommand #{subcommand}"
  end
end

#cache(subcommand) ⇒ Object 



455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
 
# File 'lib/kamal/cli/proxy.rb', line 455

def cache(subcommand)
  count, json, path_prefix = options[:count], options[:json], options[:path_prefix]

  case subcommand
  when "stats"
    # The cache lives where its policy applies: at the loadbalancer when
    # load balancing (cache is edge-only in the layering contract),
    # otherwise on each proxy host.
    if KAMAL.config.proxy.load_balancing?
      on(KAMAL.config.proxy.effective_loadbalancer) do |host|
        puts_by_host host, capture_with_info(*KAMAL.loadbalancer.cache_stats(count: count, json: json)), type: "Loadbalancer"
      end
    else
      on(KAMAL.proxy_hosts) do |host|
        puts_by_host host, capture_with_info(*KAMAL.proxy(host).cache_stats(count: count, json: json)), type: "Proxy"
      end
    end
  when "purge"
    if KAMAL.config.proxy.load_balancing?
      # The loadbalancer registers one service under the bare service name.
      on(KAMAL.config.proxy.effective_loadbalancer) do |host|
        execute *KAMAL.auditor.record("Purged the response cache"), verbosity: :debug
        puts_by_host host, capture_with_info(*KAMAL.loadbalancer.cache_purge(KAMAL.config.service, path_prefix: path_prefix)), type: "Loadbalancer"
      end
    else
      # Each proxied role registered its own service, so purge walks them.
      on(KAMAL.proxy_hosts) do |host|
        execute *KAMAL.auditor.record("Purged the response cache"), verbosity: :debug

        KAMAL.roles_on(host).select(&:running_proxy?).each do |role|
          puts_by_host host, capture_with_info(*KAMAL.proxy(host).cache_purge(role.container_prefix, path_prefix: path_prefix)), type: "Proxy"
        end
      end
    end
  else
    puts "Unknown cache subcommand: #{subcommand}. Available: stats, purge"
  end
end

#details ⇒ Object 



339
340
341
342
343
344
345
346
347
348
 
# File 'lib/kamal/cli/proxy.rb', line 339

def details
  quiet = options[:quiet]
  on(KAMAL.proxy_hosts) { |host| puts_by_host host, capture_with_info(*KAMAL.proxy(host).info), type: "Proxy", quiet: quiet }

  if KAMAL.config.proxy.load_balancing?
    on(KAMAL.config.proxy.effective_loadbalancer) do |host|
      puts_by_host host, capture_with_info(*KAMAL.loadbalancer.info), type: "Loadbalancer"
    end
  end
end

#domains(subcommand) ⇒ Object 



495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
 
# File 'lib/kamal/cli/proxy.rb', line 495

def domains(subcommand)
  case subcommand
  when "refresh", "list", "stats"
    # TLS terminates at the load balancer when load balancing, so the
    # dynamic domain set lives there rather than on the per-host proxies.
    if KAMAL.config.proxy.load_balancing?
      on(KAMAL.config.proxy.effective_loadbalancer) do |host|
        puts_by_host host, capture_with_info(*KAMAL.loadbalancer.domains(subcommand)), type: "Loadbalancer"
      end
    else
      on(KAMAL.proxy_hosts) do |host|
        puts_by_host host, capture_with_info(*KAMAL.proxy(host).domains(subcommand)), type: "Proxy"
      end
    end
  else
    puts "Unknown domains subcommand: #{subcommand}. Available: refresh, list, stats"
  end
end

#loadbalancer(status) ⇒ Object 



391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
 
# File 'lib/kamal/cli/proxy.rb', line 391

def loadbalancer(status)
  case status
  when "info"
    if KAMAL.config.proxy.load_balancing?
      on(KAMAL.config.proxy.effective_loadbalancer) do |host|
        puts "Loadbalancer status on #{host}:"
        puts capture_with_info(*KAMAL.loadbalancer.info)
      end
    else
      puts "Load balancing is not configured"
    end
  when "start"
    if KAMAL.config.proxy.load_balancing?
      on(KAMAL.config.proxy.effective_loadbalancer) do |host|
        execute *KAMAL.registry.
        execute *KAMAL.loadbalancer.start_or_run
      end
    else
      puts "Load balancing is not configured"
    end
  when "stop"
    if KAMAL.config.proxy.load_balancing?
      on(KAMAL.config.proxy.effective_loadbalancer) do |host|
        execute *KAMAL.loadbalancer.stop, raise_on_non_zero_exit: false
      end
    else
      puts "Load balancing is not configured"
    end
  when "logs"
    if KAMAL.config.proxy.load_balancing?
      on(KAMAL.config.proxy.effective_loadbalancer) do |host|
        puts_by_host host, capture(*KAMAL.loadbalancer.logs(timestamps: true)), type: "Loadbalancer"
      end
    else
      puts "Load balancing is not configured"
    end
  when "deploy"
    if KAMAL.config.proxy.load_balancing?
      targets = []
      KAMAL.config.roles.each do |role|
        next unless role.running_proxy?

        role.hosts.each do |host|
          targets << host
        end
      end

      on(KAMAL.config.proxy.effective_loadbalancer) do |host|
        Kamal::Cli::Proxy::LoadbalancerClaim.new(host, self).claim_service
        info "Deploying to loadbalancer on #{host} with targets: #{targets.join(', ')}"
        execute *KAMAL.loadbalancer.deploy(targets: targets)
      end
    else
      puts "Load balancing is not configured"
    end
  else
    puts "Unknown loadbalancer subcommand: #{status}. Available: info, start, stop, logs, deploy"
  end
end

#logs ⇒ Object 



356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
 
# File 'lib/kamal/cli/proxy.rb', line 356

def logs
  grep = options[:grep]
  timestamps = !options[:skip_timestamps]

  if options[:follow]
    run_locally do
      proxy = KAMAL.proxy(KAMAL.primary_host)
      info "Following logs on #{KAMAL.primary_host}..."
      info proxy.follow_logs(host: KAMAL.primary_host, timestamps: timestamps, grep: grep)
      exec proxy.follow_logs(host: KAMAL.primary_host, timestamps: timestamps, grep: grep)
    end
  else
    since = options[:since]
    lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set

    on(KAMAL.proxy_hosts) do |host|
      puts_by_host host, capture(*KAMAL.proxy(host).logs(timestamps: timestamps, since: since, lines: lines, grep: grep)), type: "Proxy"
    end
  end
end

#reboot ⇒ Object 



216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
 
# File 'lib/kamal/cli/proxy.rb', line 216

def reboot
  confirming "This will cause a brief outage on each host. Are you sure?" do
    modify(lock: true) do
      # Skip proxy on loadbalancer host - it will be handled by loadbalancer reboot
      proxy_hosts = KAMAL.proxy_hosts
      if KAMAL.config.proxy.loadbalancer_on_proxy_host?
        proxy_hosts = proxy_hosts - [ KAMAL.config.proxy.effective_loadbalancer ]
      end

      host_groups = options[:rolling] ? proxy_hosts : [ proxy_hosts ]
      host_groups.each do |hosts|
        next if Array(hosts).empty?

        host_list = Array(hosts).join(",")
        run_hook "pre-proxy-reboot", hosts: host_list
        on(hosts) do |host|
          info "Rebooting kamal-proxy on #{host}..."
          Kamal::Cli::Proxy::Reboot.new(host, self).run
        end
        run_hook "post-proxy-reboot", hosts: host_list
      end

      if KAMAL.config.proxy.load_balancing?
        lb_host = KAMAL.config.proxy.effective_loadbalancer
        run_hook "pre-loadbalancer-reboot", hosts: lb_host

        on(lb_host) do |host|
          Kamal::Cli::Proxy::LoadbalancerReboot.new(host, self).run
        end

        run_hook "post-loadbalancer-reboot", hosts: lb_host
      end
    end
  end
end

#remove ⇒ Object 



379
380
381
382
383
384
385
386
387
388
 
# File 'lib/kamal/cli/proxy.rb', line 379

def remove
  modify(lock: true) do
    if removal_allowed?(options[:force])
      stop
      remove_container
      remove_image
      remove_proxy_directory
    end
  end
end

#remove_container ⇒ Object 



515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
 
# File 'lib/kamal/cli/proxy.rb', line 515

def remove_container
  modify(lock: true) do
    on(KAMAL.proxy_hosts) do
      execute *KAMAL.auditor.record("Removed proxy container"), verbosity: :debug
      execute *KAMAL.proxy(host).remove_container
    end

    if KAMAL.config.proxy.load_balancing?
      on(KAMAL.config.proxy.effective_loadbalancer) do
        execute *KAMAL.auditor.record("Removed loadbalancer container"), verbosity: :debug
        execute *KAMAL.loadbalancer.remove_container
      end
    end
  end
end

#remove_image ⇒ Object 



532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
 
# File 'lib/kamal/cli/proxy.rb', line 532

def remove_image
  modify(lock: true) do
    on(KAMAL.proxy_hosts) do
      execute *KAMAL.auditor.record("Removed proxy image"), verbosity: :debug
      execute *KAMAL.proxy(host).remove_image
    end

    if KAMAL.config.proxy.load_balancing?
      on(KAMAL.config.proxy.effective_loadbalancer) do
        execute *KAMAL.auditor.record("Removed loadbalancer image"), verbosity: :debug
        execute *KAMAL.loadbalancer.remove_image
      end
    end
  end
end

#remove_proxy_directory ⇒ Object 



549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
 
# File 'lib/kamal/cli/proxy.rb', line 549

def remove_proxy_directory
  modify(lock: true) do
    on(KAMAL.proxy_hosts) do
      execute *KAMAL.proxy(host).remove_proxy_directory, raise_on_non_zero_exit: false
    end

    # Otherwise the ownership registry outlives the load balancer and a later
    # boot would fail against an owner that no longer exists.
    if KAMAL.config.proxy.load_balancing?
      on(KAMAL.config.proxy.effective_loadbalancer) do
        execute *KAMAL.loadbalancer.remove_directory, raise_on_non_zero_exit: false
      end
    end
  end
end

#restart ⇒ Object 



331
332
333
334
335
336
 
# File 'lib/kamal/cli/proxy.rb', line 331

def restart
  modify(lock: true) do
    stop
    start
  end
end

#start ⇒ Object 



294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
 
# File 'lib/kamal/cli/proxy.rb', line 294

def start
  modify(lock: true) do
    on(KAMAL.proxy_hosts) do |host|
      execute *KAMAL.auditor.record("Started proxy"), verbosity: :debug
      execute *KAMAL.proxy(host).start
    end

    if KAMAL.config.proxy.load_balancing?
      on(KAMAL.config.proxy.effective_loadbalancer) do
        execute *KAMAL.auditor.record("Started loadbalancer"), verbosity: :debug
        execute *KAMAL.loadbalancer.start, raise_on_non_zero_exit: false
      end
    end
  end
end

#stop ⇒ Object 



311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
 
# File 'lib/kamal/cli/proxy.rb', line 311

def stop
  modify(lock: true) do
    on(KAMAL.proxy_hosts) do |host|
      execute *KAMAL.auditor.record("Stopped proxy"), verbosity: :debug
      execute *KAMAL.proxy(host).stop, raise_on_non_zero_exit: false
    end

    # `docker container prune` only collects stopped containers, so leaving the
    # loadbalancer running would also leave `kamal proxy remove` unable to
    # remove it.
    if KAMAL.config.proxy.load_balancing?
      on(KAMAL.config.proxy.effective_loadbalancer) do
        execute *KAMAL.auditor.record("Stopped loadbalancer"), verbosity: :debug
        execute *KAMAL.loadbalancer.stop, raise_on_non_zero_exit: false
      end
    end
  end
end

#upgrade ⇒ Object 



255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
 
# File 'lib/kamal/cli/proxy.rb', line 255

def upgrade
  invoke_options = { "version" => KAMAL.config.latest_tag }.merge(options)

  confirming "This will cause a brief outage on each host. Are you sure?" do
    host_groups = options[:rolling] ? KAMAL.hosts : [ KAMAL.hosts ]
    host_groups.each do |hosts|
      host_list = Array(hosts).join(",")
      say "Upgrading proxy on #{host_list}...", :magenta
      run_hook "pre-proxy-reboot", hosts: host_list
      on(hosts) do |host|
        proxy = KAMAL.proxy(host)
        execute *KAMAL.auditor.record("Rebooted proxy"), verbosity: :debug
        execute *KAMAL.registry.

        info "Stopping and removing Traefik on #{host}, if running..."
        execute *proxy.cleanup_traefik

        info "Stopping and removing kamal-proxy on #{host}, if running..."
        execute *proxy.stop, raise_on_non_zero_exit: false
        execute *proxy.remove_container
        execute *proxy.remove_image
      end

      KAMAL.with_specific_hosts(hosts) do
        invoke "kamal:cli:proxy:boot", [], invoke_options
        reset_invocation(Kamal::Cli::Proxy)
        invoke "kamal:cli:app:boot", [], invoke_options
        reset_invocation(Kamal::Cli::App)
        invoke "kamal:cli:prune:all", [], invoke_options
        reset_invocation(Kamal::Cli::Prune)
      end

      run_hook "post-proxy-reboot", hosts: host_list
      say "Upgraded proxy on #{host_list}", :magenta
    end
  end
end