Class: Boxcars::Gpt4allEng

Inherits:
Engine
  • Object
show all
Includes:
UnifiedObservability
Defined in:
lib/boxcars/engine/gpt4all_eng.rb

Overview

An engine that uses local GPT4All API.

Constant Summary collapse

DEFAULT_NAME = 
"Gpt4all engine"
DEFAULT_DESCRIPTION = 
"useful for when you need to use local AI to answer questions. " \
"You should ask targeted questions"
DEFAULT_PARAMS = 
{
  model_name: "gpt4all-j-v1.3-groovy"
}.freeze

Instance Attribute Summary collapse

Attributes inherited from Engine

#batch_size, #user_id

Instance Method Summary collapse

Methods inherited from Engine

#add_usage_detail!, #aggregate_generate_usage!, #aggregate_token_usage_details!, #append_generate_choices!, #capabilities, #extract_answer, #generate, #generate_one, #generation_info, #get_num_tokens, #normalize_generate_response, #process_generate_prompt!, #process_generate_response!, #run, #supports?, #usage_nested_token_value, #usage_token_value, #validate_response!

Constructor Details

#initialize(name: DEFAULT_NAME, description: DEFAULT_DESCRIPTION, batch_size: 2, **kwargs) ⇒ Gpt4allEng

Returns a new instance of Gpt4allEng.

Raises:



25
26
27
28
29
30
31
 
# File 'lib/boxcars/engine/gpt4all_eng.rb', line 25

def initialize(name: DEFAULT_NAME, description: DEFAULT_DESCRIPTION, batch_size: 2, **kwargs)
  raise ArgumentError, "unknown keyword: :prompts" if kwargs.key?(:prompts)

  user_id = kwargs.delete(:user_id)
  @gpt4all_params = DEFAULT_PARAMS.merge(kwargs)
  super(description:, name:, batch_size:, user_id:)
end

Instance Attribute Details

#gpt4all_params ⇒ Object (readonly)

Returns the value of attribute gpt4all_params.



16
17
18
 
# File 'lib/boxcars/engine/gpt4all_eng.rb', line 16

def gpt4all_params
  @gpt4all_params
end

Instance Method Details

#client(prompt:, inputs: {}, **kwargs) ⇒ Object 



33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
 
# File 'lib/boxcars/engine/gpt4all_eng.rb', line 33

def client(prompt:, inputs: {}, **kwargs)
  ensure_gpt4all_available!
  start_time = Time.now
  response_data = { response_obj: nil, parsed_json: nil, success: false, error: nil, status_code: nil }
  current_params = @gpt4all_params.merge(kwargs)
  api_request_params, gpt4all_instance = nil
  current_prompt_object = prompt
  begin
    gpt4all_instance = Gpt4all::ConversationalAI.new
    gpt4all_instance.prepare_resources(force_download: false)
    gpt4all_instance.start_bot

    prompt_text_for_api = current_prompt_object.as_prompt(inputs:)
    prompt_text_for_api = prompt_text_for_api[:prompt] if prompt_text_for_api.is_a?(Hash) && prompt_text_for_api.key?(:prompt)
    api_request_params = { prompt: prompt_text_for_api }

    Boxcars.debug("Prompt after formatting:\n#{prompt_text_for_api}", :cyan) if Boxcars.configuration.log_prompts

    raw_response_text = gpt4all_instance.prompt(prompt_text_for_api)
    prompt_tokens = get_num_tokens(text: prompt_text_for_api.to_s)
    completion_tokens = get_num_tokens(text: raw_response_text.to_s)

    response_data[:response_obj] = raw_response_text
    response_data[:parsed_json] = {
      "text" => raw_response_text,
      "choices" => [
        {
          "text" => raw_response_text,
          "finish_reason" => "stop"
        }
      ],
      "usage" => {
        "prompt_tokens" => prompt_tokens,
        "completion_tokens" => completion_tokens,
        "total_tokens" => prompt_tokens + completion_tokens
      }
    }
    response_data[:success] = true
    response_data[:status_code] = 200
  rescue StandardError => e
    response_data[:error] = e
    response_data[:success] = false
  ensure
    gpt4all_instance&.stop_bot

    duration_ms = ((Time.now - start_time) * 1000).round
    request_context = {
      prompt: current_prompt_object,
      inputs:,
      conversation_for_api: api_request_params&.dig(:prompt),
      user_id:
    }

    track_ai_generation(
      duration_ms:,
      current_params:,
      request_context:,
      response_data:,
      provider: :gpt4all
    )
  end

  gpt4all_handle_call_outcome(response_data:)
end

#default_params ⇒ Object 



98
99
100
 
# File 'lib/boxcars/engine/gpt4all_eng.rb', line 98

def default_params
  @gpt4all_params
end