Class: Google::Cloud::Ces::V1beta::EvaluationMetricsConfig

Inherits:
Object
  • Object
show all
Extended by:
Protobuf::MessageExts::ClassMethods
Includes:
Protobuf::MessageExts
Defined in:
proto_docs/google/cloud/ces/v1beta/evaluation_metrics_config.rb

Overview

Configures the metrics for an evaluation.

Defined Under Namespace

Modules: ComparisonType Classes: ExpectationsMetMetricsConfig, GoldenMetricsConfig, HallucinationMetricsConfig, ScenarioMetricsConfig, SemanticSimilarityMetricsConfig, ToolCorrectnessMetricsConfig, UserGoalMetMetricsConfig

Instance Attribute Summary collapse

Instance Attribute Details

#golden_metrics_config ⇒ ::Google::Cloud::Ces::V1beta::EvaluationMetricsConfig::GoldenMetricsConfig

Returns Optional. Configuration for the golden metrics for the evaluation.

Returns:



31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
 
# File 'proto_docs/google/cloud/ces/v1beta/evaluation_metrics_config.rb', line 31

class EvaluationMetricsConfig
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # Configuration for similarity metrics for the evaluation.
  # To disable the metric, set the message but do not set the
  # `enable_semantic_similarity_metrics` field to true (or explicitly set it to
  # false). To unset the configuration and fallback to the default behavior,
  # omit the message entirely.
  # @!attribute [rw] enable_semantic_similarity_metrics
  #   @return [::Boolean]
  #     Optional. Whether to calculate semantic similarity metrics for the
  #     evaluation.
  class SemanticSimilarityMetricsConfig
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # Configuration for correctness metrics for the evaluation.
  # To disable the metric, set the message but do not set the
  # `enable_tool_correctness_metrics` field to true (or explicitly set it to
  # false). To unset the configuration and fallback to the default behavior,
  # omit the message entirely.
  # @!attribute [rw] enable_tool_correctness_metrics
  #   @return [::Boolean]
  #     Optional. Whether to calculate tool correctness metrics for the
  #     evaluation.
  class ToolCorrectnessMetricsConfig
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # Configuration for the hallucination metrics for the evaluation.
  # To disable the metric, set the message but do not set the
  # `enable_hallucination_metrics` field to true (or explicitly set it to
  # false). To unset the configuration and fallback to the default behavior,
  # omit the message entirely.
  # @!attribute [rw] enable_hallucination_metrics
  #   @return [::Boolean]
  #     Optional. Whether to calculate hallucination metrics for the evaluation.
  class HallucinationMetricsConfig
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # Configuration for the user goal met metrics for the evaluation.
  # To disable the metric, set the message but do not set the
  # `enable_user_goal_met_metrics` field to true (or explicitly set it to
  # false). To unset the configuration and fallback to the default behavior,
  # omit the message entirely.
  # @!attribute [rw] enable_user_goal_met_metrics
  #   @return [::Boolean]
  #     Optional. Whether to calculate the user goal met metrics for the
  #     evaluation.
  class UserGoalMetMetricsConfig
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # Configuration for the expectation level metrics for the evaluation.
  # To disable the metric, set the message but do not set the
  # `enable_expectations_met_metrics` field to true (or explicitly set it to
  # false). To unset the configuration and fallback to the default behavior,
  # omit the message entirely.
  # @!attribute [rw] enable_expectations_met_metrics
  #   @return [::Boolean]
  #     Optional. Whether to calculate the expectation level metrics for the
  #     evaluation.
  class ExpectationsMetMetricsConfig
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # Configuration for the golden metrics for the evaluation.
  # @!attribute [rw] semantic_similarity_metrics_config
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationMetricsConfig::SemanticSimilarityMetricsConfig]
  #     Optional. Global configuration for semantic similarity metrics.
  # @!attribute [rw] tool_correctness_metrics_config
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationMetricsConfig::ToolCorrectnessMetricsConfig]
  #     Optional. Configuration for turn level tool correctness metrics.
  # @!attribute [rw] step_tool_correctness_metrics_config
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationMetricsConfig::ToolCorrectnessMetricsConfig]
  #     Optional. Configuration for step level tool correctness metrics.
  class GoldenMetricsConfig
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # Configuration for the scenario metrics for the evaluation.
  # @!attribute [rw] user_goal_met_metrics_config
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationMetricsConfig::UserGoalMetMetricsConfig]
  #     Optional. Configuration for user goal met metrics.
  # @!attribute [rw] expectations_met_metrics_config
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationMetricsConfig::ExpectationsMetMetricsConfig]
  #     Optional. Configuration for expectation level metrics.
  class ScenarioMetricsConfig
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # Supported comparison types for checking the agent's response.
  module ComparisonType
    # Unspecified comparison type. Behavior defaults to SEMANTIC_SIMILARITY
    # for agent responses and tool calls.
    COMPARISON_TYPE_UNSPECIFIED = 0

    # Exact string match.
    EQUALS = 1

    # Substring match (checks if the expected string is contained in the
    # actual response).
    CONTAINS = 2

    # Semantic similarity match (evaluates meaning similarity using an LLM).
    SEMANTIC_SIMILARITY = 3
  end
end

#scenario_metrics_config ⇒ ::Google::Cloud::Ces::V1beta::EvaluationMetricsConfig::ScenarioMetricsConfig

Returns Optional. Configuration for the scenario metrics for the evaluation.

Returns:



31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
 
# File 'proto_docs/google/cloud/ces/v1beta/evaluation_metrics_config.rb', line 31

class EvaluationMetricsConfig
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # Configuration for similarity metrics for the evaluation.
  # To disable the metric, set the message but do not set the
  # `enable_semantic_similarity_metrics` field to true (or explicitly set it to
  # false). To unset the configuration and fallback to the default behavior,
  # omit the message entirely.
  # @!attribute [rw] enable_semantic_similarity_metrics
  #   @return [::Boolean]
  #     Optional. Whether to calculate semantic similarity metrics for the
  #     evaluation.
  class SemanticSimilarityMetricsConfig
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # Configuration for correctness metrics for the evaluation.
  # To disable the metric, set the message but do not set the
  # `enable_tool_correctness_metrics` field to true (or explicitly set it to
  # false). To unset the configuration and fallback to the default behavior,
  # omit the message entirely.
  # @!attribute [rw] enable_tool_correctness_metrics
  #   @return [::Boolean]
  #     Optional. Whether to calculate tool correctness metrics for the
  #     evaluation.
  class ToolCorrectnessMetricsConfig
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # Configuration for the hallucination metrics for the evaluation.
  # To disable the metric, set the message but do not set the
  # `enable_hallucination_metrics` field to true (or explicitly set it to
  # false). To unset the configuration and fallback to the default behavior,
  # omit the message entirely.
  # @!attribute [rw] enable_hallucination_metrics
  #   @return [::Boolean]
  #     Optional. Whether to calculate hallucination metrics for the evaluation.
  class HallucinationMetricsConfig
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # Configuration for the user goal met metrics for the evaluation.
  # To disable the metric, set the message but do not set the
  # `enable_user_goal_met_metrics` field to true (or explicitly set it to
  # false). To unset the configuration and fallback to the default behavior,
  # omit the message entirely.
  # @!attribute [rw] enable_user_goal_met_metrics
  #   @return [::Boolean]
  #     Optional. Whether to calculate the user goal met metrics for the
  #     evaluation.
  class UserGoalMetMetricsConfig
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # Configuration for the expectation level metrics for the evaluation.
  # To disable the metric, set the message but do not set the
  # `enable_expectations_met_metrics` field to true (or explicitly set it to
  # false). To unset the configuration and fallback to the default behavior,
  # omit the message entirely.
  # @!attribute [rw] enable_expectations_met_metrics
  #   @return [::Boolean]
  #     Optional. Whether to calculate the expectation level metrics for the
  #     evaluation.
  class ExpectationsMetMetricsConfig
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # Configuration for the golden metrics for the evaluation.
  # @!attribute [rw] semantic_similarity_metrics_config
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationMetricsConfig::SemanticSimilarityMetricsConfig]
  #     Optional. Global configuration for semantic similarity metrics.
  # @!attribute [rw] tool_correctness_metrics_config
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationMetricsConfig::ToolCorrectnessMetricsConfig]
  #     Optional. Configuration for turn level tool correctness metrics.
  # @!attribute [rw] step_tool_correctness_metrics_config
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationMetricsConfig::ToolCorrectnessMetricsConfig]
  #     Optional. Configuration for step level tool correctness metrics.
  class GoldenMetricsConfig
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # Configuration for the scenario metrics for the evaluation.
  # @!attribute [rw] user_goal_met_metrics_config
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationMetricsConfig::UserGoalMetMetricsConfig]
  #     Optional. Configuration for user goal met metrics.
  # @!attribute [rw] expectations_met_metrics_config
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationMetricsConfig::ExpectationsMetMetricsConfig]
  #     Optional. Configuration for expectation level metrics.
  class ScenarioMetricsConfig
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # Supported comparison types for checking the agent's response.
  module ComparisonType
    # Unspecified comparison type. Behavior defaults to SEMANTIC_SIMILARITY
    # for agent responses and tool calls.
    COMPARISON_TYPE_UNSPECIFIED = 0

    # Exact string match.
    EQUALS = 1

    # Substring match (checks if the expected string is contained in the
    # actual response).
    CONTAINS = 2

    # Semantic similarity match (evaluates meaning similarity using an LLM).
    SEMANTIC_SIMILARITY = 3
  end
end