Class: Ignis::AI::NN::Dropout

Inherits:
Module
  • Object
show all
Defined in:
lib/nnw/ai/nn/dropout.rb

Overview

Dropout: randomly zeros elements during training. In eval mode, passes input through unchanged. Uses inverted dropout: scale by 1/(1-p) during training.

Instance Attribute Summary

Attributes inherited from Module

#training

Instance Method Summary collapse

Methods inherited from Module

#call, #eval!, #load_state_dict, #named_parameters, #num_parameters, #parameters, #state_dict, #to, #train!, #zero_grad!

Constructor Details

#initialize(p: 0.1) ⇒ Dropout

Returns a new instance of Dropout.

Parameters:

  • p (Float) (defaults to: 0.1) —

    probability of zeroing an element (0.0 to 1.0)



13
14
15
16
 
# File 'lib/nnw/ai/nn/dropout.rb', line 13

def initialize(p: 0.1)
  super()
  @p = p
end

Instance Method Details

#forward(x) ⇒ Tensor

Forward pass. In training mode: apply random mask and scale by 1/(1-p). In eval mode: pass through.

Parameters:

  • x (Tensor)

Returns:

  • (Tensor) 


23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
 
# File 'lib/nnw/ai/nn/dropout.rb', line 23

def forward(x)
  return x unless @training
  return x if @p == 0.0

  # Generate bernoulli mask on host, transfer to GPU
  n = x.numel
  scale = 1.0 / (1.0 - @p)
  mask_data = Array.new(n) { Kernel.rand >= @p ? scale : 0.0 }

  mask_nv = Ignis::Shared::NvArray.new(shape: x.shape, dtype: x.dtype, device_id: x.device_id)
  mask_nv.from_host(mask_data)

  # Elementwise multiply
  result_nv = Ignis::Shared::NvArray.new(shape: x.shape, dtype: x.dtype, device_id: x.device_id)
  result_nv.from_host(Array.new(n, 0.0))

  kernel = Ignis::JIT::Kernels::Elementwise.mul_forward
  kernel.launch(grid: [(n + 255) / 256], block: [256],
                args: [x.data, mask_nv, result_nv, n])

  result = Tensor.new(data: result_nv, requires_grad: x.requires_grad, is_leaf: false)

  if x.requires_grad
    Tape.record(result, inputs: [x]) do |grad|
      # Backward: multiply grad by same mask
      grad_in = Ignis::Shared::NvArray.new(shape: grad.shape, dtype: grad.dtype,
                                           device_id: grad.device_id)
      grad_in.from_host(Array.new(grad.numel, 0.0))
      mk = Ignis::JIT::Kernels::Elementwise.mul_forward
      mk.launch(grid: [(grad.numel + 255) / 256], block: [256],
                args: [grad, mask_nv, grad_in, grad.numel])
      [grad_in]
    end
  end

  result
end

#to_s ⇒ String

Returns:

  • (String) 


62
63
64
 
# File 'lib/nnw/ai/nn/dropout.rb', line 62

def to_s
  "Dropout(p=#{@p})"
end