lex-mlx

LegionIO extension for Apple MLX local LLM inference server (mlx_lm.server).

Installation

gem 'lex-mlx'

Usage

require 'legion/extensions/mlx'

client = Legion::Extensions::Mlx::Client.new
# or with custom host:
# client = Legion::Extensions::Mlx::Client.new(host: 'http://localhost:8080')

# Chat
result = client.chat(model: 'mlx-community/Llama-3-8B-Instruct-4bit',
                     messages: [{ role: 'user', content: 'Hello!' }])

# Streaming chat
client.chat_stream(model: 'mlx-community/Llama-3-8B-Instruct-4bit',
                   messages: [{ role: 'user', content: 'Hello!' }]) do |event|
  print event[:text] if event[:type] == :delta
end

# Completions
result = client.complete(model: 'mlx-community/Llama-3-8B-Instruct-4bit',
                         prompt: 'Once upon a time')

# Embeddings
result = client.embed(model: 'mlx-community/Llama-3-8B-Instruct-4bit',
                      input: 'Hello world')

# List models
result = client.list_models

License

MIT