Class: Toy::Core::CLI::Serve

Inherits:
Object
  • Object
show all
Defined in:
lib/toy/core/cli/serve.rb

Constant Summary collapse

DEFAULT_PORT = 
4567
RUNNER_TARGET =

NOTE: target name MUST equal the output path — ToyRoot.ensure_built runs ‘make <RUNNER_TARGET>` and File.join(root, RUNNER_TARGET) is the binary. `make libexec/toy-serve` outputs libexec/toy-serve.

"libexec/toy-serve"

Instance Method Summary collapse

Constructor Details

#initialize(argv) ⇒ Serve

Returns a new instance of Serve.



46
47
48
49
50
51
 
# File 'lib/toy/core/cli/serve.rb', line 46

def initialize(argv)
  @argv = argv
  @model = nil
  @port = DEFAULT_PORT
  @name = nil
end

Instance Method Details

#run ⇒ Object 



53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
 
# File 'lib/toy/core/cli/serve.rb', line 53

def run
  parsed = parse_args
  return parsed unless parsed == true

  path = File.expand_path(@model)
  unless File.file?(path)
    return bad_input("no such file: #{path}")
  end

  root = ToyRoot.locate_root
  unless root
    return fail_out(
      "could not locate toy's install root. Set TOY_HOME to a toy " \
      "checkout (one with a Makefile + tinynn/tinynn_ggml.c), or run " \
      "from inside the toy source tree. Then `toy install` to build " \
      "the backend."
    )
  end

  ok, err = ToyRoot.ensure_built(root, RUNNER_TARGET)
  return fail_out(err) unless ok

  runner = File.join(root, RUNNER_TARGET)
  unless File.file?(runner) && File.executable?(runner)
    return fail_out(
      "runner missing after build: #{runner}. Run `toy install` to " \
      "build the backend, then retry."
    )
  end

  # Default the model name from the GGUF basename (the runner does
  # the same derivation; we echo a friendly name pre-exec).
  name = @name || File.basename(path).sub(/\.gguf\z/, "")

  # Resolve a run id + create runs/<id>/ in the PROJECT cwd, mirroring
  # cli/train.rb:101-113. serve uses Kernel.exec (NOT Open3) — there is
  # NO "after exec", so ALL resolution + mkdir MUST happen BEFORE the
  # exec below. The runner assumes TAO_RUN_DIR pre-exists
  # (tnn_events_open does no parent mkdir). serve has no fixed arch, so
  # the {arch} run-id token is the literal "serve" (only shapes the
  # run_id STRING, e.g. serve-20260531-001). Always-on (matches train,
  # which has no off-switch): the runner's EVENTS.length>0 guard makes
  # emission conditional on TAO_RUN_DIR being non-empty.
  project = Dir.pwd
  cfg     = Toy::Core::Config.load(project)
  run_id  = resolve_run_id(cfg.run_id_template, project, "serve")
  run_dir = File.join(project, "runs", run_id)
  begin
    FileUtils.mkdir_p(run_dir)
  rescue SystemCallError => e
    return fail_out("could not create run dir #{run_dir}: #{e.message}")
  end

  # The toy-branded line BEFORE exec; after exec the child streams
  # its own "[openai_api_llama] ready; serving" + Tep's listening
  # line. Goes to stderr so it never pollutes any piped capture.
  $stderr.puts "toy serve: starting #{name} on http://127.0.0.1:#{@port} (Ctrl-C to stop)"

  # DIVERGENCE: replace this process with the persistent runner.
  # The controlled env hash is exec's first positional arg — the
  # caller's stale MODEL_PATH/MODEL_NAME/PORT can never leak in.
  # Control never returns past this line.
  Kernel.exec(
    { "MODEL_PATH" => path, "MODEL_NAME" => name, "PORT" => @port.to_s,
      "TAO_RUN_DIR" => run_dir, "TOY_RUN_ID" => run_id },
    runner
  )
end