Class: Mat

Inherits:
Object
  • Object
show all
Defined in:
lib/toy/models/transformer.rb

Overview

Mat: 2D float matrix, flat-storage. Indexed as flat[i * ncols + j].

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(nrows, ncols) ⇒ Mat

Returns a new instance of Mat.



71
72
73
74
75
 
# File 'lib/toy/models/transformer.rb', line 71

def initialize(nrows, ncols)
  @nrows = nrows
  @ncols = ncols
  @flat  = Array.new(nrows * ncols, 0.0)
end

Instance Attribute Details

#flat ⇒ Object

Returns the value of attribute flat.



69
70
71
 
# File 'lib/toy/models/transformer.rb', line 69

def flat
  @flat
end

#ncols ⇒ Object

Returns the value of attribute ncols.



69
70
71
 
# File 'lib/toy/models/transformer.rb', line 69

def ncols
  @ncols
end

#nrows ⇒ Object

Returns the value of attribute nrows.



69
70
71
 
# File 'lib/toy/models/transformer.rb', line 69

def nrows
  @nrows
end

Instance Method Details

#add!(other) ⇒ Object 



212
213
214
215
216
217
218
219
220
221
222
 
# File 'lib/toy/models/transformer.rb', line 212

def add!(other)
  _trace = TinyNN.tnn_trace_begin("Mat.add!")
  n = @nrows * @ncols
  i = 0
  while i < n
    @flat[i] += other.flat[i]
    i += 1
  end
  TinyNN.tnn_trace_end("Mat.add!", _trace)
  self
end

#fill_random(scale) ⇒ Object 



77
78
79
80
81
82
83
84
85
86
87
 
# File 'lib/toy/models/transformer.rb', line 77

def fill_random(scale)
  # Spinel's `rand` (no args) is the C `rand()` returning a large int,
  # not Ruby's [0.0, 1.0) float. Use `rand(N)` which behaves the same
  # in both: an integer in [0, N).
  n = @nrows * @ncols
  i = 0
  while i < n
    @flat[i] = (rand(2000).to_f / 1000.0 - 1.0) * scale
    i += 1
  end
end

#fill_zero ⇒ Object 



89
90
91
92
93
94
95
96
 
# File 'lib/toy/models/transformer.rb', line 89

def fill_zero
  n = @nrows * @ncols
  i = 0
  while i < n
    @flat[i] = 0.0
    i += 1
  end
end

#get(i, j) ⇒ Object 



236
237
238
 
# File 'lib/toy/models/transformer.rb', line 236

def get(i, j)
  @flat[i * @ncols + j]
end

#info ⇒ Object

“Mat[5, 768] min=-2.34 max=1.97 mean=0.012” — shape + a few summary statistics. Useful for “is my activation drifting?” sanity checks.



255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
 
# File 'lib/toy/models/transformer.rb', line 255

def info
  n = @nrows * @ncols
  if n == 0
    return "Mat[" + @nrows.to_s + ", " + @ncols.to_s + "] (empty)"
  end
  mn = @flat[0]
  mx = @flat[0]
  sum = 0.0
  i = 0
  while i < n
    v = @flat[i]
    if v < mn
      mn = v
    end
    if v > mx
      mx = v
    end
    sum = sum + v
    i += 1
  end
  mean = sum / n.to_f
  "Mat[" + @nrows.to_s + ", " + @ncols.to_s +
    "] min=" + mn.to_s + " max=" + mx.to_s + " mean=" + mean.to_s
end

#matmul(other) ⇒ Object

(m × n) · (n × p) → (m × p) — using local accumulator (faster than repeated indexed writes since each += would re-load the receiver).



100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
 
# File 'lib/toy/models/transformer.rb', line 100

def matmul(other)
  _trace = TinyNN.tnn_trace_begin("Mat.matmul")
  m = @nrows
  n = @ncols
  p = other.ncols
  if MAT_SHAPES_ON; puts "MAT_SHAPE matmul " + m.to_s + " " + n.to_s + " " + p.to_s; end
  out = Mat.new(m, p)
  i = 0
  while i < m
    j = 0
    while j < p
      s = 0.0
      k = 0
      while k < n
        s = s + @flat[i * n + k] * other.flat[k * p + j]
        k += 1
      end
      out.flat[i * p + j] = s
      j += 1
    end
    i += 1
  end
  TinyNN.tnn_trace_end("Mat.matmul", _trace)
  out
end

#matmul_t(other) ⇒ Object

self · otherᵀ where other has the same column count as self.



127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
 
# File 'lib/toy/models/transformer.rb', line 127

def matmul_t(other)
  _trace = TinyNN.tnn_trace_begin("Mat.matmul_t")
  m = @nrows
  n = @ncols
  p = other.nrows                # other is (p × n) so otherᵀ is (n × p)
  if MAT_SHAPES_ON; puts "MAT_SHAPE matmul_t " + m.to_s + " " + n.to_s + " " + p.to_s; end
  out = Mat.new(m, p)
  i = 0
  while i < m
    j = 0
    while j < p
      s = 0.0
      k = 0
      while k < n
        s = s + @flat[i * n + k] * other.flat[j * n + k]
        k += 1
      end
      out.flat[i * p + j] = s
      j += 1
    end
    i += 1
  end
  TinyNN.tnn_trace_end("Mat.matmul_t", _trace)
  out
end

#plus(other) ⇒ Object

Element-wise sum. Renamed from ‘add` to dodge a Spinel polymorphic-dispatch with Tep::Router#add (different arity, but Spinel’s arg-type-narrowing in 4024216 doesn’t catch the arity mismatch). add! (mutating, suffix-banged) is unaffected since method NAME (not signature) is what collides.



199
200
201
202
203
204
205
206
207
208
209
210
 
# File 'lib/toy/models/transformer.rb', line 199

def plus(other)
  _trace = TinyNN.tnn_trace_begin("Mat.plus")
  out = Mat.new(@nrows, @ncols)
  n = @nrows * @ncols
  i = 0
  while i < n
    out.flat[i] = @flat[i] + other.flat[i]
    i += 1
  end
  TinyNN.tnn_trace_end("Mat.plus", _trace)
  out
end

#scale!(s) ⇒ Object 



224
225
226
227
228
229
230
231
232
233
234
 
# File 'lib/toy/models/transformer.rb', line 224

def scale!(s)
  _trace = TinyNN.tnn_trace_begin("Mat.scale!")
  n = @nrows * @ncols
  i = 0
  while i < n
    @flat[i] = @flat[i] * s
    i += 1
  end
  TinyNN.tnn_trace_end("Mat.scale!", _trace)
  self
end

#set_at(i, j, v) ⇒ Object 



240
241
242
 
# File 'lib/toy/models/transformer.rb', line 240

def set_at(i, j, v)
  @flat[i * @ncols + j] = v
end

#shape ⇒ Object

“[5, 768]” — concise shape string for prints and logs.



249
250
251
 
# File 'lib/toy/models/transformer.rb', line 249

def shape
  "[" + @nrows.to_s + ", " + @ncols.to_s + "]"
end

#t_matmul(other) ⇒ Object

selfᵀ · other where self is (n × m) and other is (n × p) → (m × p)



154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
 
# File 'lib/toy/models/transformer.rb', line 154

def t_matmul(other)
  _trace = TinyNN.tnn_trace_begin("Mat.t_matmul")
  n = @nrows
  m = @ncols
  p = other.ncols
  if MAT_SHAPES_ON; puts "MAT_SHAPE t_matmul " + n.to_s + " " + m.to_s + " " + p.to_s; end
  out = Mat.new(m, p)
  i = 0
  while i < m
    j = 0
    while j < p
      s = 0.0
      k = 0
      while k < n
        s = s + @flat[k * m + i] * other.flat[k * p + j]
        k += 1
      end
      out.flat[i * p + j] = s
      j += 1
    end
    i += 1
  end
  TinyNN.tnn_trace_end("Mat.t_matmul", _trace)
  out
end

#transpose ⇒ Object 



180
181
182
183
184
185
186
187
188
189
190
191
192
 
# File 'lib/toy/models/transformer.rb', line 180

def transpose
  out = Mat.new(@ncols, @nrows)
  i = 0
  while i < @nrows
    j = 0
    while j < @ncols
      out.flat[j * @nrows + i] = @flat[i * @ncols + j]
      j += 1
    end
    i += 1
  end
  out
end