Module: Ignis::AI::Transformer

Defined in:
lib/nnw/ai/transformer/block.rb,
lib/nnw/ai/transformer/model.rb,
lib/nnw/ai/transformer/modern.rb,
lib/nnw/ai/transformer/swiglu.rb,
lib/nnw/ai/transformer/attention.rb,
lib/nnw/ai/transformer/feed_forward.rb

Defined Under Namespace

Classes: Block, FeedForward, Model, ModernBlock, ModernModel, MultiHeadAttention, RopeGqaAttention, SwiGLU

Class Method Summary collapse

Class Method Details

.compute_inv_freq(head_dim, base, scaling = nil) ⇒ Array<Float>

Compute the RoPE inverse-frequency table [head_dim/2], optionally applying a scaling scheme. Supports the “llama3” rope_type (Llama-3.1/3.2): low frequencies are divided by ‘factor`, high frequencies are kept, and a smooth interpolation bridges the two — matching HF’s _compute_llama3_parameters.

Parameters:

  • head_dim (Integer)
  • base (Float) —

    rope_theta

  • scaling (Hash, nil) (defaults to: nil) —

    e.g. factor:, low_freq_factor:, high_freq_factor:, original_max_position_embeddings: (symbol or string keys)

Returns:

  • (Array<Float>) —

    length head_dim/2



20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
 
# File 'lib/nnw/ai/transformer/modern.rb', line 20

def self.compute_inv_freq(head_dim, base, scaling = nil)
  half = head_dim / 2
  freqs = (0...half).map { |i| base.to_f**(-2.0 * i / head_dim) }
  return freqs unless scaling
  g = ->(k) { scaling[k] || scaling[k.to_s] }
  return freqs unless (g.call(:rope_type) || g.call(:type)) == "llama3"

  factor   = g.call(:factor).to_f
  low_ff   = g.call(:low_freq_factor).to_f
  high_ff  = g.call(:high_freq_factor).to_f
  old_ctx  = g.call(:original_max_position_embeddings).to_f
  low_wl   = old_ctx / low_ff
  high_wl  = old_ctx / high_ff
  freqs.map do |f|
    wl = 2.0 * Math::PI / f
    if wl > low_wl
      f / factor
    elsif wl < high_wl
      f
    else
      smooth = (old_ctx / wl - low_ff) / (high_ff - low_ff)
      (1.0 - smooth) * (f / factor) + smooth * f
    end
  end
end