Class: Toy::Core::CLI::Infer

Inherits:
Object
  • Object
show all
Defined in:
lib/toy/core/cli/infer.rb

Constant Summary collapse

FORMAT = 
"toy/infer-v1"
DEFAULT_PROMPT =

parity w/ lib/toy/run/infer.rb

"Once upon a time"
DEFAULT_N =

parity w/ runner N_NEW default

16
RUNNER_TARGET =

NOTE: target name MUST equal the output path — ToyRoot.ensure_built runs ‘make <RUNNER_TARGET>` and File.join(root, RUNNER_TARGET) is the binary. `make libexec/toy-infer` outputs libexec/toy-infer.

"libexec/toy-infer"

Instance Method Summary collapse

Constructor Details

#initialize(argv) ⇒ Infer

Returns a new instance of Infer.



44
45
46
47
48
49
50
51
52
 
# File 'lib/toy/core/cli/infer.rb', line 44

def initialize(argv)
  @argv = argv
  @json = false
  @model = nil
  @prompt = DEFAULT_PROMPT
  @prompt_ids = nil # numeric ids for tokenizer-less models
  @n = DEFAULT_N
  @device = "cpu"
end

Instance Method Details

#run ⇒ Object 



54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
 
# File 'lib/toy/core/cli/infer.rb', line 54

def run
  parsed = parse_args
  return parsed unless parsed == true

  # metal is accepted by the parser but only buildable in a macOS
  # build — gate it HERE, before any build/Open3, so the gx10 build
  # never tries to compile the (Apple-only) metal runner.
  if @device == "metal" && RUBY_PLATFORM !~ /darwin/
    return fail_out("metal is only available in a macOS build")
  end

  path = File.expand_path(@model)
  unless File.file?(path)
    return bad_input("no such file: #{path}")
  end

  root = ToyRoot.locate_root
  unless root
    return fail_out(
      "could not locate toy's install root. Set TOY_HOME to a toy " \
      "checkout (one with a Makefile + tinynn/tinynn_ggml.c), or run " \
      "from inside the toy source tree. Then `toy install` to build " \
      "the backend."
    )
  end

  # Per-device binary: the CPU target stays byte-unchanged (its link
  # line never sees the CUDA archive); cuda builds the sibling runner.
  target = case @device
           when "cuda"  then "libexec/toy-infer-cuda"
           when "metal" then "libexec/toy-infer-metal"
           else RUNNER_TARGET
           end

  ok, err = ToyRoot.ensure_built(root, target, quiet: @json)
  return fail_out(err) unless ok

  runner = File.join(root, target)
  unless File.file?(runner) && File.executable?(runner)
    return fail_out(
      "runner missing after build: #{runner}. Run `toy install` to " \
      "build the backend, then retry."
    )
  end

  env = { "GGUF" => path, "PROMPT" => @prompt, "N_NEW" => @n.to_s,
          "DEVICE" => @device }
  env["PROMPT_IDS"] = @prompt_ids if @prompt_ids
  # Metal: disable ggml's residency-set optimization. The runner relies
  # on process exit to reclaim GPU buffers (it never explicitly frees),
  # but ggml-metal's static-destructor teardown asserts the residency
  # set is empty (ggml-metal-device.m: GGML_ASSERT([rsets->data count]
  # == 0)) — which fires AFTER correct output, aborting at exit and
  # making the runner look like it produced nothing. Residency sets are
  # a pure GPU-residency perf hint; disabling them leaves compute (and
  # thus metal-vs-cpu token parity) byte-identical.
  env["GGML_METAL_NO_RESIDENCY"] = "1" if @device == "metal"
  out, status = Open3.capture2e(env, runner)
  unless status.success?
    tail = out.lines.last(20).join
    return fail_out("runner exited #{status.exitstatus}:\n#{tail}")
  end

  text = scan_line(out, "text: ")
  if text
    return emit(path, text: text)
  end
  ids = scan_line(out, "ids:")
  if ids
    return emit(path, ids: ids.strip)
  end

  tail = out.lines.last(20).join
  fail_out("runner produced no `text:`/`ids:` line; output was:\n#{tail}")
end