Module: CompletionKit::McpTools::Runs

Extended by:
Base
Defined in:
app/services/completion_kit/mcp_tools/runs.rb

Constant Summary collapse

TOOLS = 
{
  "runs_list" => {
    description: "List all runs",
    inputSchema: {type: "object", properties: {}, required: []},
    handler: :list
  },
  "runs_get" => {
    description: "Get a run by ID",
    inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
    handler: :get
  },
  "runs_create" => {
    description: "Create a run. Omit prompt_id and provide output_column to score existing outputs by grading a pre-existing dataset column instead of generating new ones.",
    inputSchema: {
      type: "object",
      properties: {
        name: {type: "string"}, prompt_id: {type: "integer"},
        dataset_id: {type: "integer"}, judge_model: {type: "string"},
        output_column: {type: "string", description: "Dataset column to grade when prompt_id is omitted; defaults to \"actual_output\"."},
        expected_column: {type: "string", description: "Dataset column holding each row's answer key / ground truth, graded by checks with compare_to \"expected\" and passed to the judge; defaults to \"expected_output\"."},
        metric_ids: {type: "array", items: {type: "integer"}},
        tag_names: {type: "array", items: {type: "string"}}
      },
      required: ["name"]
    },
    handler: :create
  },
  "runs_update" => {
    description: "Update a run",
    inputSchema: {
      type: "object",
      properties: {
        id: {type: "integer"}, name: {type: "string"},
        dataset_id: {type: "integer"}, judge_model: {type: "string"},
        output_column: {type: "string"},
        expected_column: {type: "string"},
        metric_ids: {type: "array", items: {type: "integer"}},
        tag_names: {type: "array", items: {type: "string"}}
      },
      required: ["id"]
    },
    handler: :update
  },
  "runs_delete" => {
    description: "Delete a run",
    inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
    handler: :delete
  },
  "runs_generate" => {
    description: "Start a run. Required for every run, including score-only runs (no prompt): generates responses with the prompt when there is one, otherwise copies the graded dataset column and grades it.",
    inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
    handler: :generate
  }
}.freeze

Class Method Summary collapse

Methods included from Base

call, definitions, error_result, text_result

Class Method Details

.create(args) ⇒ Object 



69
70
71
72
73
74
75
76
77
78
 
# File 'app/services/completion_kit/mcp_tools/runs.rb', line 69

def self.create(args)
  run = Run.new(args.slice("name", "prompt_id", "dataset_id", "judge_model", "output_column", "expected_column"))
  if run.save
    run.replace_metrics!(args["metric_ids"])
    run.update!(tag_names: args["tag_names"]) if args.key?("tag_names")
    text_result(run.reload.as_json)
  else
    error_result(run.errors.full_messages.join(", "))
  end
end

.delete(args) ⇒ Object 



91
92
93
94
 
# File 'app/services/completion_kit/mcp_tools/runs.rb', line 91

def self.delete(args)
  Run.find(args["id"]).destroy!
  text_result("Run #{args["id"]} deleted")
end

.generate(args) ⇒ Object 



96
97
98
99
100
101
102
103
 
# File 'app/services/completion_kit/mcp_tools/runs.rb', line 96

def self.generate(args)
  run = Run.find(args["id"])
  if run.start!
    text_result(run.reload.as_json)
  else
    text_result(run.failure_summary || run.errors.full_messages.to_sentence)
  end
end

.get(args) ⇒ Object 



65
66
67
 
# File 'app/services/completion_kit/mcp_tools/runs.rb', line 65

def self.get(args)
  text_result(Run.find(args["id"]).as_json)
end

.list(_args) ⇒ Object 



61
62
63
 
# File 'app/services/completion_kit/mcp_tools/runs.rb', line 61

def self.list(_args)
  text_result(Run.display_scoped.order(created_at: :desc).map(&:as_json))
end

.update(args) ⇒ Object 



80
81
82
83
84
85
86
87
88
89
 
# File 'app/services/completion_kit/mcp_tools/runs.rb', line 80

def self.update(args)
  run = Run.find(args["id"])
  if run.update(args.except("id", "metric_ids", "tag_names").slice("name", "dataset_id", "judge_model", "output_column", "expected_column"))
    run.replace_metrics!(args["metric_ids"]) if args.key?("metric_ids")
    run.update!(tag_names: args["tag_names"]) if args.key?("tag_names")
    text_result(run.reload.as_json)
  else
    error_result(run.errors.full_messages.join(", "))
  end
end