Class: Toy::Core::CLI::Eval

Inherits:
Object
  • Object
show all
Defined in:
lib/toy/core/cli/eval.rb

Constant Summary collapse

FORMAT = 
"toy/eval-v1"
DEFAULT_TOP_K =

parity w/ lib/toy/run/eval.rb TOP_K default

5
RUNNER_TARGET =

NOTE: target name MUST equal the output path — ToyRoot.ensure_built runs ‘make <RUNNER_TARGET>` and File.join(root, RUNNER_TARGET) is the binary. `make libexec/toy-eval` outputs libexec/toy-eval.

"libexec/toy-eval"
LMC_RUNNER_TARGET =

LMC subcommand (‘toy eval lmc –ckpt A –other B [–alphas …]`). CPU-only this slice; a cuda LMC twin is a later slice.

"libexec/toy-eval-lmc"
DEFAULT_ALPHAS = 
"0,0.25,0.5,0.75,1.0"

Instance Method Summary collapse

Constructor Details

#initialize(argv) ⇒ Eval

Returns a new instance of Eval.



47
48
49
50
51
52
53
54
55
56
 
# File 'lib/toy/core/cli/eval.rb', line 47

def initialize(argv)
  @argv = argv
  @json = false
  @model = nil
  @top_k = DEFAULT_TOP_K
  @device = "cpu"
  @lmc_a = nil
  @lmc_b = nil
  @alphas = DEFAULT_ALPHAS
end

Instance Method Details

#run ⇒ Object 



58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
 
# File 'lib/toy/core/cli/eval.rb', line 58

def run
  # LMC subcommand dispatch — `toy eval lmc ...`. The single-model path
  # below is BYTE-UNCHANGED for the non-lmc case.
  return run_lmc(@argv.drop(1)) if @argv.first == "lmc"

  parsed = parse_args
  return parsed unless parsed == true

  # metal is accepted by the parser but only buildable in a macOS
  # build — gate it HERE, before any build/Open3, so the gx10 build
  # never tries to compile the (Apple-only) metal runner.
  if @device == "metal" && RUBY_PLATFORM !~ /darwin/
    return fail_out("metal is only available in a macOS build")
  end

  path = File.expand_path(@model)
  unless File.file?(path)
    return bad_input("no such file: #{path}")
  end

  root = ToyRoot.locate_root
  unless root
    return fail_out(
      "could not locate toy's install root. Set TOY_HOME to a toy " \
      "checkout (one with a Makefile + tinynn/tinynn_ggml.c), or run " \
      "from inside the toy source tree. Then `toy install` to build " \
      "the backend."
    )
  end

  # Per-device binary: the CPU target stays byte-unchanged (its link
  # line never sees the CUDA archive); cuda builds the sibling runner.
  target = case @device
           when "cuda"  then "libexec/toy-eval-cuda"
           when "metal" then "libexec/toy-eval-metal"
           else RUNNER_TARGET
           end

  ok, err = ToyRoot.ensure_built(root, target, quiet: @json)
  return fail_out(err) unless ok

  runner = File.join(root, target)
  unless File.file?(runner) && File.executable?(runner)
    return fail_out(
      "runner missing after build: #{runner}. Run `toy install` to " \
      "build the backend, then retry."
    )
  end

  env = { "GGUF" => path, "TOP_K" => @top_k.to_s, "DEVICE" => @device }
  # Metal: disable ggml's residency-set optimization so the runner exits
  # cleanly. See lib/toy/core/cli/infer.rb for the full rationale — the
  # ggml-metal static-destructor teardown asserts the residency set is
  # empty and aborts at exit; disabling it keeps compute byte-identical.
  env["GGML_METAL_NO_RESIDENCY"] = "1" if @device == "metal"
  out, status = Open3.capture2e(env, runner)
  unless status.success?
    tail = out.lines.last(20).join
    return fail_out("runner exited #{status.exitstatus}:\n#{tail}")
  end

  lines = out.lines.map(&:chomp).select { |l| l.start_with?("logprob:") }
  if lines.empty?
    tail = out.lines.last(20).join
    return fail_out("runner produced no `logprob:` line; output was:\n#{tail}")
  end

  emit(path, lines)
end