Module: ToyDriftGrad

Defined in:
lib/toy/train/toy_drift_grad.rb

Class Method Summary collapse

Class Method Details

.emit_drift_event(sess, t, snap_mat, step, t_now) ⇒ Object

Emit one ‘drift` event for tensor t, comparing against snap_mat. Single-pass cos + L2 loop over the f64 flats.



107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
 
# File 'lib/toy/train/toy_drift_grad.rb', line 107

def self.emit_drift_event(sess, t, snap_mat, step, t_now)
  n = TinyNN.tnn_tensor_nelements(t)
  cur = Mat.new(1, n)
  TinyNN.tnn_download_to_f64_array(sess, t, cur.flat, n)

  dot = 0.0
  sum_sq_s = 0.0
  sum_sq_c = 0.0
  sum_sq_diff = 0.0
  i = 0
  while i < n
    sv = snap_mat.flat[i]
    cv = cur.flat[i]
    dot = dot + sv * cv
    sum_sq_s = sum_sq_s + sv * sv
    sum_sq_c = sum_sq_c + cv * cv
    d = sv - cv
    sum_sq_diff = sum_sq_diff + d * d
    i = i + 1
  end
  norm_s = sum_sq_s ** 0.5
  norm_c = sum_sq_c ** 0.5
  cos_to_init = 0.0
  if norm_s > 0.0 && norm_c > 0.0
    cos_to_init = dot / (norm_s * norm_c)
  end
  l2_to_init = sum_sq_diff ** 0.5

  name = TinyNN.tnn_tensor_name(t)
  ev = SpinelKit::Json::Builder.new
  ev.add_str("kind",  "drift")
  ev.add_str("phase", "train")
  ev.add_num("t",           t_now)
  ev.add_num("step",        step)
  ev.add_str("param",       name)
  ev.add_num("cos_to_init", cos_to_init)
  ev.add_num("l2_to_init",  l2_to_init)
  TinyNN.tnn_events_emit(ev.dump)
end

.emit_grad_event(sess, t, step, t_now) ⇒ Object

Emit one ‘grad` event for tensor t. Call after tnn_compute_backward but before the next step’s reset (so the grad tensor is still live).



149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
 
# File 'lib/toy/train/toy_drift_grad.rb', line 149

def self.emit_grad_event(sess, t, step, t_now)
  g = TinyNN.tnn_tensor_grad(sess, t)
  if g == nil || g == TinyNN.tnn_null_ptr
    return
  end
  n = TinyNN.tnn_tensor_nelements(g)
  TinyNN.tnn_download(sess, g)
  sum_sq   = TinyNN.tnn_scratch_sum_sq_f32(sess, n)
  sum_abs  = TinyNN.tnn_scratch_sum_abs_f32(sess, n)
  nan      = TinyNN.tnn_scratch_nan_count_f32(sess, n)
  l2       = sum_sq ** 0.5
  abs_mean = sum_abs / n.to_f
  ne0 = TinyNN.tnn_tensor_ne0(g)
  ne1 = TinyNN.tnn_tensor_ne1(g)
  name = TinyNN.tnn_tensor_name(t)
  ev = SpinelKit::Json::Builder.new
  ev.add_str("kind",  "grad")
  ev.add_str("phase", "train")
  ev.add_num("t",         t_now)
  ev.add_num("step",      step)
  ev.add_str("param",     name)
  ev.add_raw("shape",     "[" + ne0.to_s + "," + ne1.to_s + "]")
  ev.add_num("l2",        l2)
  ev.add_num("abs_mean",  abs_mean)
  ev.add_num("nan_count", nan)
  TinyNN.tnn_events_emit(ev.dump)
end

.params(sess) ⇒ Object

Walk session graph + leaves, return ordered array of (ptr, name) for every PARAM-flagged tensor (flags bit 4 set). The flag is ‘ggml_set_param`-marked, so this exactly matches the params the training graph differentiates over.



43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
 
# File 'lib/toy/train/toy_drift_grad.rb', line 43

def self.params(sess)
  ptrs = [TinyNN.tnn_null_ptr]; ptrs.pop
  seen = [TinyNN.tnn_null_ptr]; seen.pop

  # Collect every distinct tensor reachable from the graph: nodes +
  # their src dependencies. Filter to PARAM-flagged.
  n_nodes = TinyNN.tnn_graph_n_nodes(sess)
  i = 0
  while i < n_nodes
    node = TinyNN.tnn_graph_node(sess, i)
    # The node itself
    seen.push(node)
    # Its srcs (leaves usually carry the PARAM flag)
    si = 0
    while si < 10
      src = TinyNN.tnn_tensor_src(node, si)
      if src == nil || src == TinyNN.tnn_null_ptr
        si = 10
      else
        # Already seen?
        dup = false
        k = 0
        while k < seen.length
          if seen[k] == src
            dup = true
            k = seen.length
          else
            k = k + 1
          end
        end
        if !dup
          seen.push(src)
        end
        si = si + 1
      end
    end
    i = i + 1
  end

  # Filter by PARAM flag.
  i = 0
  while i < seen.length
    t = seen[i]
    flags = TinyNN.tnn_tensor_flags(t)
    if (flags & 4) != 0
      ptrs.push(t)
    end
    i = i + 1
  end
  ptrs
end

.snapshot_one(sess, t) ⇒ Object

Snapshot ONE param’s current values into a fresh Mat. Caller keeps these in a main-scope Array<Mat> (a separate per-step Array would force Spinel to declare sp_Mat_ptr_array, which it can’t).



98
99
100
101
102
103
 
# File 'lib/toy/train/toy_drift_grad.rb', line 98

def self.snapshot_one(sess, t)
  n = TinyNN.tnn_tensor_nelements(t)
  m = Mat.new(1, n)
  TinyNN.tnn_download_to_f64_array(sess, t, m.flat, n)
  m
end