Class: Ignis::AI::Optim::LRScheduler::CosineAnnealing

Inherits:
Object
  • Object
show all
Defined in:
lib/nnw/ai/optim/lr_scheduler.rb

Overview

Cosine annealing: decay LR following a cosine curve. LR drops from base_lr to min_lr over total_steps.

Instance Method Summary collapse

Constructor Details

#initialize(optimizer, total_steps:, min_lr: 0.0) ⇒ CosineAnnealing

Returns a new instance of CosineAnnealing.

Parameters:

  • optimizer (Base)
  • total_steps (Integer)
  • min_lr (Float) (defaults to: 0.0) 


47
48
49
50
51
52
 
# File 'lib/nnw/ai/optim/lr_scheduler.rb', line 47

def initialize(optimizer, total_steps:, min_lr: 0.0)
  @optimizer = optimizer
  @total_steps = total_steps
  @min_lr = min_lr.to_f
  @base_lr = optimizer.lr
end

Instance Method Details

#current_lr ⇒ Float

Returns:

  • (Float) 


69
70
71
 
# File 'lib/nnw/ai/optim/lr_scheduler.rb', line 69

def current_lr
  @optimizer.lr
end

#step ⇒ Float

Returns:

  • (Float) 


55
56
57
58
59
60
61
62
63
64
65
66
 
# File 'lib/nnw/ai/optim/lr_scheduler.rb', line 55

def step
  current = @optimizer.step_count
  if current >= @total_steps
    @optimizer.lr = @min_lr
    return @min_lr
  end

  progress = current.to_f / @total_steps
  new_lr = @min_lr + 0.5 * (@base_lr - @min_lr) * (1.0 + Math.cos(Math::PI * progress))
  @optimizer.lr = new_lr
  new_lr
end