Class: Google::Cloud::Ces::V1beta::EvaluationResult

Inherits:
Object
  • Object
show all
Extended by:
Protobuf::MessageExts::ClassMethods
Includes:
Protobuf::MessageExts
Defined in:
proto_docs/google/cloud/ces/v1beta/evaluation.rb

Overview

An evaluation result represents the output of running an Evaluation.

Defined Under Namespace

Modules: ExecutionState, Outcome Classes: EvaluationExpectationResult, GoldenExpectationOutcome, GoldenResult, HallucinationResult, OverallToolInvocationResult, ScenarioExpectationOutcome, ScenarioResult, ScenarioRubricOutcome, SemanticSimilarityResult, SpanLatency, TaskCompletionResult, ToolCallLatency, UserGoalSatisfactionResult

Instance Attribute Summary collapse

Instance Attribute Details

#app_version ⇒ ::String (readonly)

Returns Output only. The app version used to generate the conversation that resulted in this result. Format: projects/{project}/locations/{location}/apps/{app}/versions/{version}.

Returns:

  • (::String) —

    Output only. The app version used to generate the conversation that resulted in this result. Format: projects/{project}/locations/{location}/apps/{app}/versions/{version}



549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
 
# File 'proto_docs/google/cloud/ces/v1beta/evaluation.rb', line 549

class EvaluationResult
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # Specifies the expectation and the result of that expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
  #     Output only. The result of the tool call expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_tool_response`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_tool_response
  #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
  #     Output only. The result of the tool response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_response`, `observed_tool_call`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The result of the agent response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`, `observed_tool_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_transfer
  #   @return [::Google::Cloud::Ces::V1beta::AgentTransfer]
  #     Output only. The result of the agent transfer expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_transfer`, `observed_tool_call`, `observed_tool_response`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::GoldenExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the expectation.
  # @!attribute [r] semantic_similarity_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
  #     Output only. The result of the semantic similarity check.
  # @!attribute [r] tool_invocation_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome::ToolInvocationResult]
  #     Output only. The result of the tool invocation check.
  class GoldenExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of the tool invocation check.
    # @!attribute [r] parameter_correctness_score
    #   @return [::Float]
    #     Output only. The tool invocation parameter correctness score. This
    #     indicates the percent of parameters from the expected tool call that
    #     were also present in the actual tool call.
    # @!attribute [r] outcome
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
    #     Output only. The outcome of the tool invocation check. This is
    #     determined by comparing the parameter_correctness_score to the
    #     threshold. If the score is equal to or above the threshold, the outcome
    #     will be PASS. Otherwise, the outcome will be FAIL.
    # @!attribute [r] explanation
    #   @return [::String]
    #     Output only. A free text explanation for the tool invocation result.
    class ToolInvocationResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The result of a single evaluation expectation.
  # @!attribute [r] evaluation_expectation
  #   @return [::String]
  #     Output only. The evaluation expectation.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluation_expectation}`
  # @!attribute [r] prompt
  #   @return [::String]
  #     Output only. The prompt that was used for the evaluation.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the evaluation expectation.
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the result.
  class EvaluationExpectationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a golden evaluation.
  # @!attribute [r] turn_replay_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenResult::TurnReplayResult>]
  #     Output only. The result of running each turn of the golden conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class GoldenResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of running a single turn of the golden conversation.
    # @!attribute [r] conversation
    #   @return [::String]
    #     Output only. The conversation that was generated for this turn.
    # @!attribute [r] expectation_outcome
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome>]
    #     Output only. The outcome of each expectation.
    # @!attribute [r] hallucination_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult]
    #     Output only. The result of the hallucination check.
    # @!attribute [r] tool_invocation_score
    #   @deprecated This field is deprecated and may be removed in the next major version update.
    #   @return [::Float]
    #     Output only. Deprecated. Use OverallToolInvocationResult instead.
    # @!attribute [r] tool_ordered_invocation_score
    #   @return [::Float]
    #     Output only. The overall tool ordered invocation score for this turn.
    #     This indicates the overall percent of tools from the expected turn that
    #     were actually invoked in the expected order.
    # @!attribute [r] turn_latency
    #   @return [::Google::Protobuf::Duration]
    #     Output only. Duration of the turn.
    # @!attribute [r] tool_call_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
    #     Output only. The latency of each tool call in the turn.
    # @!attribute [r] semantic_similarity_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
    #     Output only. The result of the semantic similarity check.
    # @!attribute [r] overall_tool_invocation_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::OverallToolInvocationResult]
    #     Output only. The result of the overall tool invocation check.
    # @!attribute [r] error_info
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationErrorInfo]
    #     Output only. Information about the error that occurred during this
    #     turn.
    # @!attribute [r] span_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
    #     Output only. The latency of spans in the turn.
    class TurnReplayResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of the evaluation against the rubric.
  # @!attribute [r] rubric
  #   @return [::String]
  #     Output only. The rubric that was used to evaluate the conversation.
  # @!attribute [r] score
  #   @return [::Float]
  #     Output only. The score of the conversation against the rubric.
  # @!attribute [r] score_explanation
  #   @return [::String]
  #     Output only. The rater's response to the rubric.
  class ScenarioRubricOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of a scenario expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome::ObservedToolCall]
  #     Output only. The observed tool call.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The observed agent response.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::ScenarioExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the ScenarioExpectation.
  class ScenarioExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The observed tool call and response.
    # @!attribute [r] tool_call
    #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
    #     Output only. The observed tool call.
    # @!attribute [r] tool_response
    #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
    #     Output only. The observed tool response.
    class ObservedToolCall
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of a scenario evaluation.
  # @!attribute [r] conversation
  #   @return [::String]
  #     Output only. The conversation that was generated in the scenario.
  # @!attribute [r] task
  #   @return [::String]
  #     Output only. The task that was used when running the scenario for this
  #     result.
  # @!attribute [r] user_facts
  #   @return [::Array<::Google::Cloud::Ces::V1beta::Evaluation::Scenario::UserFact>]
  #     Output only. The user facts that were used by the scenario for this
  #     result.
  # @!attribute [r] expectation_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome>]
  #     Output only. The outcome of each expectation.
  # @!attribute [r] rubric_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioRubricOutcome>]
  #     Output only. The outcome of the rubric.
  # @!attribute [r] hallucination_result
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult>]
  #     Output only. The result of the hallucination check.
  #     There will be one hallucination result for each turn in the conversation.
  # @!attribute [r] task_completion_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::TaskCompletionResult]
  #     Output only. The result of the task completion check.
  # @!attribute [r] tool_call_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
  #     Output only. The latency of each tool call execution in the conversation.
  # @!attribute [r] user_goal_satisfaction_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::UserGoalSatisfactionResult]
  #     Output only. The result of the user goal satisfaction check.
  # @!attribute [r] all_expectations_satisfied
  #   @return [::Boolean]
  #     Output only. Whether all expectations were satisfied for this turn.
  # @!attribute [r] task_completed
  #   @return [::Boolean]
  #     Output only. Whether the task was completed for this turn. This is a
  #     composite of all expectations satisfied, no hallucinations, and user goal
  #     satisfaction.
  # @!attribute [r] span_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
  #     Output only. The latency of spans in the conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class ScenarioResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The latency of a span execution.
  # @!attribute [r] resource
  #   @return [::String]
  #     Output only. The resource name of the guardrail or tool spans.
  #
  #     Note: The following fields are mutually exclusive: `resource`, `toolset`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] toolset
  #   @return [::Google::Cloud::Ces::V1beta::ToolsetTool]
  #     Output only. The toolset tool identifier.
  #
  #     Note: The following fields are mutually exclusive: `toolset`, `resource`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] model
  #   @return [::String]
  #     Output only. The name of the LLM span.
  #
  #     Note: The following fields are mutually exclusive: `model`, `resource`, `toolset`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] callback
  #   @return [::String]
  #     Output only. The name of the user callback span.
  #
  #     Note: The following fields are mutually exclusive: `callback`, `resource`, `toolset`, `model`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] type
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency::Type]
  #     Output only. The type of span.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the span. Applicable to tool and
  #     guardrail spans.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of span.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of span.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of span.
  class SpanLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The type of span.
    # Additional values may be added in the future.
    module Type
      # Default value. This value is unused.
      TYPE_UNSPECIFIED = 0

      # Tool call span.
      TOOL = 1

      # User callback span.
      USER_CALLBACK = 2

      # Guardrail span.
      GUARDRAIL = 3

      # LLM span.
      LLM = 4
    end
  end

  # The latency of a tool call execution.
  # @!attribute [r] tool
  #   @return [::String]
  #     Output only. The name of the tool that got executed.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/tools/{tool}`.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the tool.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of the tool call execution.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of the tool call execution.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of the tool call execution.
  class ToolCallLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the hallucination check for a single turn.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The hallucination score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Justified
  #     Score 0: Not Justified
  #     Score -1: No Claim To Assess
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the hallucination score.
  class HallucinationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a user goal satisfaction check for a conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The user task satisfaction score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: User Task Satisfied
  #     Score 0: User Task Not Satisfied
  #     Score -1: User Task Unspecified
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the user task satisfaction score.
  class UserGoalSatisfactionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the semantic similarity check.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The semantic similarity score. Can be 0, 1, 2, 3, or 4.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 4: Fully Consistent
  #     Score 3: Mostly Consistent
  #     Score 2: Partially Consistent (Minor Omissions)
  #     Score 1: Largely Inconsistent (Major Omissions)
  #     Score 0: Completely Inconsistent / Contradictory
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the semantic similarity score.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the semantic similarity check. This is
  #     determined by comparing the score to the
  #     semantic_similarity_success_threshold. If the score is equal to or above
  #     the threshold, the outcome will be PASS. Otherwise, the outcome will be
  #     FAIL.
  class SemanticSimilarityResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the overall tool invocation check.
  # @!attribute [rw] tool_invocation_score
  #   @return [::Float]
  #     The overall tool invocation score for this turn. This indicates the
  #     overall percent of tools from the expected turn that were actually
  #     invoked.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the tool invocation check. This is determined
  #     by comparing the tool_invocation_score to the
  #     overall_tool_invocation_correctness_threshold. If the
  #     score is equal to or above the threshold, the outcome will be PASS.
  #     Otherwise, the outcome will be FAIL.
  class OverallToolInvocationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the task completion check for the conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The task completion score. Can be -1, 0, 1
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Task Completed
  #     Score 0: Task Not Completed
  #     Score -1: User Goal Undefined
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the task completion score.
  class TaskCompletionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of the evaluation or expectation.
  module Outcome
    # Evaluation outcome is not specified.
    OUTCOME_UNSPECIFIED = 0

    # Evaluation/Expectation passed. In the case of an evaluation, this means
    # that all expectations were met.
    PASS = 1

    # Evaluation/Expectation failed. In the case of an evaluation, this means
    # that at least one expectation was not met.
    FAIL = 2

    # Evaluation/Expectation was skipped.
    SKIPPED = 3
  end

  # The state of the evaluation result execution.
  module ExecutionState
    # Evaluation result execution state is not specified.
    EXECUTION_STATE_UNSPECIFIED = 0

    # Evaluation result execution is running.
    RUNNING = 1

    # Evaluation result execution has completed.
    COMPLETED = 2

    # Evaluation result execution failed due to an internal error.
    ERROR = 3
  end
end

#app_version_display_name ⇒ ::String (readonly)

Returns Output only. The display name of the app_version that the evaluation ran against.

Returns:

  • (::String) —

    Output only. The display name of the app_version that the evaluation ran against.



549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
 
# File 'proto_docs/google/cloud/ces/v1beta/evaluation.rb', line 549

class EvaluationResult
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # Specifies the expectation and the result of that expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
  #     Output only. The result of the tool call expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_tool_response`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_tool_response
  #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
  #     Output only. The result of the tool response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_response`, `observed_tool_call`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The result of the agent response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`, `observed_tool_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_transfer
  #   @return [::Google::Cloud::Ces::V1beta::AgentTransfer]
  #     Output only. The result of the agent transfer expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_transfer`, `observed_tool_call`, `observed_tool_response`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::GoldenExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the expectation.
  # @!attribute [r] semantic_similarity_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
  #     Output only. The result of the semantic similarity check.
  # @!attribute [r] tool_invocation_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome::ToolInvocationResult]
  #     Output only. The result of the tool invocation check.
  class GoldenExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of the tool invocation check.
    # @!attribute [r] parameter_correctness_score
    #   @return [::Float]
    #     Output only. The tool invocation parameter correctness score. This
    #     indicates the percent of parameters from the expected tool call that
    #     were also present in the actual tool call.
    # @!attribute [r] outcome
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
    #     Output only. The outcome of the tool invocation check. This is
    #     determined by comparing the parameter_correctness_score to the
    #     threshold. If the score is equal to or above the threshold, the outcome
    #     will be PASS. Otherwise, the outcome will be FAIL.
    # @!attribute [r] explanation
    #   @return [::String]
    #     Output only. A free text explanation for the tool invocation result.
    class ToolInvocationResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The result of a single evaluation expectation.
  # @!attribute [r] evaluation_expectation
  #   @return [::String]
  #     Output only. The evaluation expectation.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluation_expectation}`
  # @!attribute [r] prompt
  #   @return [::String]
  #     Output only. The prompt that was used for the evaluation.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the evaluation expectation.
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the result.
  class EvaluationExpectationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a golden evaluation.
  # @!attribute [r] turn_replay_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenResult::TurnReplayResult>]
  #     Output only. The result of running each turn of the golden conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class GoldenResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of running a single turn of the golden conversation.
    # @!attribute [r] conversation
    #   @return [::String]
    #     Output only. The conversation that was generated for this turn.
    # @!attribute [r] expectation_outcome
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome>]
    #     Output only. The outcome of each expectation.
    # @!attribute [r] hallucination_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult]
    #     Output only. The result of the hallucination check.
    # @!attribute [r] tool_invocation_score
    #   @deprecated This field is deprecated and may be removed in the next major version update.
    #   @return [::Float]
    #     Output only. Deprecated. Use OverallToolInvocationResult instead.
    # @!attribute [r] tool_ordered_invocation_score
    #   @return [::Float]
    #     Output only. The overall tool ordered invocation score for this turn.
    #     This indicates the overall percent of tools from the expected turn that
    #     were actually invoked in the expected order.
    # @!attribute [r] turn_latency
    #   @return [::Google::Protobuf::Duration]
    #     Output only. Duration of the turn.
    # @!attribute [r] tool_call_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
    #     Output only. The latency of each tool call in the turn.
    # @!attribute [r] semantic_similarity_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
    #     Output only. The result of the semantic similarity check.
    # @!attribute [r] overall_tool_invocation_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::OverallToolInvocationResult]
    #     Output only. The result of the overall tool invocation check.
    # @!attribute [r] error_info
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationErrorInfo]
    #     Output only. Information about the error that occurred during this
    #     turn.
    # @!attribute [r] span_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
    #     Output only. The latency of spans in the turn.
    class TurnReplayResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of the evaluation against the rubric.
  # @!attribute [r] rubric
  #   @return [::String]
  #     Output only. The rubric that was used to evaluate the conversation.
  # @!attribute [r] score
  #   @return [::Float]
  #     Output only. The score of the conversation against the rubric.
  # @!attribute [r] score_explanation
  #   @return [::String]
  #     Output only. The rater's response to the rubric.
  class ScenarioRubricOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of a scenario expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome::ObservedToolCall]
  #     Output only. The observed tool call.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The observed agent response.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::ScenarioExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the ScenarioExpectation.
  class ScenarioExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The observed tool call and response.
    # @!attribute [r] tool_call
    #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
    #     Output only. The observed tool call.
    # @!attribute [r] tool_response
    #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
    #     Output only. The observed tool response.
    class ObservedToolCall
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of a scenario evaluation.
  # @!attribute [r] conversation
  #   @return [::String]
  #     Output only. The conversation that was generated in the scenario.
  # @!attribute [r] task
  #   @return [::String]
  #     Output only. The task that was used when running the scenario for this
  #     result.
  # @!attribute [r] user_facts
  #   @return [::Array<::Google::Cloud::Ces::V1beta::Evaluation::Scenario::UserFact>]
  #     Output only. The user facts that were used by the scenario for this
  #     result.
  # @!attribute [r] expectation_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome>]
  #     Output only. The outcome of each expectation.
  # @!attribute [r] rubric_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioRubricOutcome>]
  #     Output only. The outcome of the rubric.
  # @!attribute [r] hallucination_result
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult>]
  #     Output only. The result of the hallucination check.
  #     There will be one hallucination result for each turn in the conversation.
  # @!attribute [r] task_completion_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::TaskCompletionResult]
  #     Output only. The result of the task completion check.
  # @!attribute [r] tool_call_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
  #     Output only. The latency of each tool call execution in the conversation.
  # @!attribute [r] user_goal_satisfaction_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::UserGoalSatisfactionResult]
  #     Output only. The result of the user goal satisfaction check.
  # @!attribute [r] all_expectations_satisfied
  #   @return [::Boolean]
  #     Output only. Whether all expectations were satisfied for this turn.
  # @!attribute [r] task_completed
  #   @return [::Boolean]
  #     Output only. Whether the task was completed for this turn. This is a
  #     composite of all expectations satisfied, no hallucinations, and user goal
  #     satisfaction.
  # @!attribute [r] span_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
  #     Output only. The latency of spans in the conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class ScenarioResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The latency of a span execution.
  # @!attribute [r] resource
  #   @return [::String]
  #     Output only. The resource name of the guardrail or tool spans.
  #
  #     Note: The following fields are mutually exclusive: `resource`, `toolset`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] toolset
  #   @return [::Google::Cloud::Ces::V1beta::ToolsetTool]
  #     Output only. The toolset tool identifier.
  #
  #     Note: The following fields are mutually exclusive: `toolset`, `resource`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] model
  #   @return [::String]
  #     Output only. The name of the LLM span.
  #
  #     Note: The following fields are mutually exclusive: `model`, `resource`, `toolset`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] callback
  #   @return [::String]
  #     Output only. The name of the user callback span.
  #
  #     Note: The following fields are mutually exclusive: `callback`, `resource`, `toolset`, `model`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] type
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency::Type]
  #     Output only. The type of span.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the span. Applicable to tool and
  #     guardrail spans.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of span.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of span.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of span.
  class SpanLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The type of span.
    # Additional values may be added in the future.
    module Type
      # Default value. This value is unused.
      TYPE_UNSPECIFIED = 0

      # Tool call span.
      TOOL = 1

      # User callback span.
      USER_CALLBACK = 2

      # Guardrail span.
      GUARDRAIL = 3

      # LLM span.
      LLM = 4
    end
  end

  # The latency of a tool call execution.
  # @!attribute [r] tool
  #   @return [::String]
  #     Output only. The name of the tool that got executed.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/tools/{tool}`.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the tool.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of the tool call execution.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of the tool call execution.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of the tool call execution.
  class ToolCallLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the hallucination check for a single turn.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The hallucination score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Justified
  #     Score 0: Not Justified
  #     Score -1: No Claim To Assess
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the hallucination score.
  class HallucinationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a user goal satisfaction check for a conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The user task satisfaction score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: User Task Satisfied
  #     Score 0: User Task Not Satisfied
  #     Score -1: User Task Unspecified
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the user task satisfaction score.
  class UserGoalSatisfactionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the semantic similarity check.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The semantic similarity score. Can be 0, 1, 2, 3, or 4.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 4: Fully Consistent
  #     Score 3: Mostly Consistent
  #     Score 2: Partially Consistent (Minor Omissions)
  #     Score 1: Largely Inconsistent (Major Omissions)
  #     Score 0: Completely Inconsistent / Contradictory
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the semantic similarity score.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the semantic similarity check. This is
  #     determined by comparing the score to the
  #     semantic_similarity_success_threshold. If the score is equal to or above
  #     the threshold, the outcome will be PASS. Otherwise, the outcome will be
  #     FAIL.
  class SemanticSimilarityResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the overall tool invocation check.
  # @!attribute [rw] tool_invocation_score
  #   @return [::Float]
  #     The overall tool invocation score for this turn. This indicates the
  #     overall percent of tools from the expected turn that were actually
  #     invoked.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the tool invocation check. This is determined
  #     by comparing the tool_invocation_score to the
  #     overall_tool_invocation_correctness_threshold. If the
  #     score is equal to or above the threshold, the outcome will be PASS.
  #     Otherwise, the outcome will be FAIL.
  class OverallToolInvocationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the task completion check for the conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The task completion score. Can be -1, 0, 1
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Task Completed
  #     Score 0: Task Not Completed
  #     Score -1: User Goal Undefined
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the task completion score.
  class TaskCompletionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of the evaluation or expectation.
  module Outcome
    # Evaluation outcome is not specified.
    OUTCOME_UNSPECIFIED = 0

    # Evaluation/Expectation passed. In the case of an evaluation, this means
    # that all expectations were met.
    PASS = 1

    # Evaluation/Expectation failed. In the case of an evaluation, this means
    # that at least one expectation was not met.
    FAIL = 2

    # Evaluation/Expectation was skipped.
    SKIPPED = 3
  end

  # The state of the evaluation result execution.
  module ExecutionState
    # Evaluation result execution state is not specified.
    EXECUTION_STATE_UNSPECIFIED = 0

    # Evaluation result execution is running.
    RUNNING = 1

    # Evaluation result execution has completed.
    COMPLETED = 2

    # Evaluation result execution failed due to an internal error.
    ERROR = 3
  end
end

#changelog ⇒ ::String (readonly)

Returns Output only. The changelog of the app version that the evaluation ran against. This is populated if user runs evaluation on latest/draft.

Returns:

  • (::String) —

    Output only. The changelog of the app version that the evaluation ran against. This is populated if user runs evaluation on latest/draft.



549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
 
# File 'proto_docs/google/cloud/ces/v1beta/evaluation.rb', line 549

class EvaluationResult
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # Specifies the expectation and the result of that expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
  #     Output only. The result of the tool call expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_tool_response`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_tool_response
  #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
  #     Output only. The result of the tool response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_response`, `observed_tool_call`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The result of the agent response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`, `observed_tool_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_transfer
  #   @return [::Google::Cloud::Ces::V1beta::AgentTransfer]
  #     Output only. The result of the agent transfer expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_transfer`, `observed_tool_call`, `observed_tool_response`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::GoldenExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the expectation.
  # @!attribute [r] semantic_similarity_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
  #     Output only. The result of the semantic similarity check.
  # @!attribute [r] tool_invocation_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome::ToolInvocationResult]
  #     Output only. The result of the tool invocation check.
  class GoldenExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of the tool invocation check.
    # @!attribute [r] parameter_correctness_score
    #   @return [::Float]
    #     Output only. The tool invocation parameter correctness score. This
    #     indicates the percent of parameters from the expected tool call that
    #     were also present in the actual tool call.
    # @!attribute [r] outcome
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
    #     Output only. The outcome of the tool invocation check. This is
    #     determined by comparing the parameter_correctness_score to the
    #     threshold. If the score is equal to or above the threshold, the outcome
    #     will be PASS. Otherwise, the outcome will be FAIL.
    # @!attribute [r] explanation
    #   @return [::String]
    #     Output only. A free text explanation for the tool invocation result.
    class ToolInvocationResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The result of a single evaluation expectation.
  # @!attribute [r] evaluation_expectation
  #   @return [::String]
  #     Output only. The evaluation expectation.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluation_expectation}`
  # @!attribute [r] prompt
  #   @return [::String]
  #     Output only. The prompt that was used for the evaluation.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the evaluation expectation.
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the result.
  class EvaluationExpectationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a golden evaluation.
  # @!attribute [r] turn_replay_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenResult::TurnReplayResult>]
  #     Output only. The result of running each turn of the golden conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class GoldenResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of running a single turn of the golden conversation.
    # @!attribute [r] conversation
    #   @return [::String]
    #     Output only. The conversation that was generated for this turn.
    # @!attribute [r] expectation_outcome
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome>]
    #     Output only. The outcome of each expectation.
    # @!attribute [r] hallucination_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult]
    #     Output only. The result of the hallucination check.
    # @!attribute [r] tool_invocation_score
    #   @deprecated This field is deprecated and may be removed in the next major version update.
    #   @return [::Float]
    #     Output only. Deprecated. Use OverallToolInvocationResult instead.
    # @!attribute [r] tool_ordered_invocation_score
    #   @return [::Float]
    #     Output only. The overall tool ordered invocation score for this turn.
    #     This indicates the overall percent of tools from the expected turn that
    #     were actually invoked in the expected order.
    # @!attribute [r] turn_latency
    #   @return [::Google::Protobuf::Duration]
    #     Output only. Duration of the turn.
    # @!attribute [r] tool_call_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
    #     Output only. The latency of each tool call in the turn.
    # @!attribute [r] semantic_similarity_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
    #     Output only. The result of the semantic similarity check.
    # @!attribute [r] overall_tool_invocation_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::OverallToolInvocationResult]
    #     Output only. The result of the overall tool invocation check.
    # @!attribute [r] error_info
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationErrorInfo]
    #     Output only. Information about the error that occurred during this
    #     turn.
    # @!attribute [r] span_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
    #     Output only. The latency of spans in the turn.
    class TurnReplayResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of the evaluation against the rubric.
  # @!attribute [r] rubric
  #   @return [::String]
  #     Output only. The rubric that was used to evaluate the conversation.
  # @!attribute [r] score
  #   @return [::Float]
  #     Output only. The score of the conversation against the rubric.
  # @!attribute [r] score_explanation
  #   @return [::String]
  #     Output only. The rater's response to the rubric.
  class ScenarioRubricOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of a scenario expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome::ObservedToolCall]
  #     Output only. The observed tool call.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The observed agent response.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::ScenarioExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the ScenarioExpectation.
  class ScenarioExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The observed tool call and response.
    # @!attribute [r] tool_call
    #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
    #     Output only. The observed tool call.
    # @!attribute [r] tool_response
    #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
    #     Output only. The observed tool response.
    class ObservedToolCall
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of a scenario evaluation.
  # @!attribute [r] conversation
  #   @return [::String]
  #     Output only. The conversation that was generated in the scenario.
  # @!attribute [r] task
  #   @return [::String]
  #     Output only. The task that was used when running the scenario for this
  #     result.
  # @!attribute [r] user_facts
  #   @return [::Array<::Google::Cloud::Ces::V1beta::Evaluation::Scenario::UserFact>]
  #     Output only. The user facts that were used by the scenario for this
  #     result.
  # @!attribute [r] expectation_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome>]
  #     Output only. The outcome of each expectation.
  # @!attribute [r] rubric_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioRubricOutcome>]
  #     Output only. The outcome of the rubric.
  # @!attribute [r] hallucination_result
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult>]
  #     Output only. The result of the hallucination check.
  #     There will be one hallucination result for each turn in the conversation.
  # @!attribute [r] task_completion_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::TaskCompletionResult]
  #     Output only. The result of the task completion check.
  # @!attribute [r] tool_call_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
  #     Output only. The latency of each tool call execution in the conversation.
  # @!attribute [r] user_goal_satisfaction_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::UserGoalSatisfactionResult]
  #     Output only. The result of the user goal satisfaction check.
  # @!attribute [r] all_expectations_satisfied
  #   @return [::Boolean]
  #     Output only. Whether all expectations were satisfied for this turn.
  # @!attribute [r] task_completed
  #   @return [::Boolean]
  #     Output only. Whether the task was completed for this turn. This is a
  #     composite of all expectations satisfied, no hallucinations, and user goal
  #     satisfaction.
  # @!attribute [r] span_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
  #     Output only. The latency of spans in the conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class ScenarioResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The latency of a span execution.
  # @!attribute [r] resource
  #   @return [::String]
  #     Output only. The resource name of the guardrail or tool spans.
  #
  #     Note: The following fields are mutually exclusive: `resource`, `toolset`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] toolset
  #   @return [::Google::Cloud::Ces::V1beta::ToolsetTool]
  #     Output only. The toolset tool identifier.
  #
  #     Note: The following fields are mutually exclusive: `toolset`, `resource`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] model
  #   @return [::String]
  #     Output only. The name of the LLM span.
  #
  #     Note: The following fields are mutually exclusive: `model`, `resource`, `toolset`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] callback
  #   @return [::String]
  #     Output only. The name of the user callback span.
  #
  #     Note: The following fields are mutually exclusive: `callback`, `resource`, `toolset`, `model`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] type
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency::Type]
  #     Output only. The type of span.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the span. Applicable to tool and
  #     guardrail spans.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of span.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of span.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of span.
  class SpanLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The type of span.
    # Additional values may be added in the future.
    module Type
      # Default value. This value is unused.
      TYPE_UNSPECIFIED = 0

      # Tool call span.
      TOOL = 1

      # User callback span.
      USER_CALLBACK = 2

      # Guardrail span.
      GUARDRAIL = 3

      # LLM span.
      LLM = 4
    end
  end

  # The latency of a tool call execution.
  # @!attribute [r] tool
  #   @return [::String]
  #     Output only. The name of the tool that got executed.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/tools/{tool}`.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the tool.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of the tool call execution.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of the tool call execution.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of the tool call execution.
  class ToolCallLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the hallucination check for a single turn.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The hallucination score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Justified
  #     Score 0: Not Justified
  #     Score -1: No Claim To Assess
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the hallucination score.
  class HallucinationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a user goal satisfaction check for a conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The user task satisfaction score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: User Task Satisfied
  #     Score 0: User Task Not Satisfied
  #     Score -1: User Task Unspecified
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the user task satisfaction score.
  class UserGoalSatisfactionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the semantic similarity check.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The semantic similarity score. Can be 0, 1, 2, 3, or 4.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 4: Fully Consistent
  #     Score 3: Mostly Consistent
  #     Score 2: Partially Consistent (Minor Omissions)
  #     Score 1: Largely Inconsistent (Major Omissions)
  #     Score 0: Completely Inconsistent / Contradictory
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the semantic similarity score.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the semantic similarity check. This is
  #     determined by comparing the score to the
  #     semantic_similarity_success_threshold. If the score is equal to or above
  #     the threshold, the outcome will be PASS. Otherwise, the outcome will be
  #     FAIL.
  class SemanticSimilarityResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the overall tool invocation check.
  # @!attribute [rw] tool_invocation_score
  #   @return [::Float]
  #     The overall tool invocation score for this turn. This indicates the
  #     overall percent of tools from the expected turn that were actually
  #     invoked.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the tool invocation check. This is determined
  #     by comparing the tool_invocation_score to the
  #     overall_tool_invocation_correctness_threshold. If the
  #     score is equal to or above the threshold, the outcome will be PASS.
  #     Otherwise, the outcome will be FAIL.
  class OverallToolInvocationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the task completion check for the conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The task completion score. Can be -1, 0, 1
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Task Completed
  #     Score 0: Task Not Completed
  #     Score -1: User Goal Undefined
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the task completion score.
  class TaskCompletionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of the evaluation or expectation.
  module Outcome
    # Evaluation outcome is not specified.
    OUTCOME_UNSPECIFIED = 0

    # Evaluation/Expectation passed. In the case of an evaluation, this means
    # that all expectations were met.
    PASS = 1

    # Evaluation/Expectation failed. In the case of an evaluation, this means
    # that at least one expectation was not met.
    FAIL = 2

    # Evaluation/Expectation was skipped.
    SKIPPED = 3
  end

  # The state of the evaluation result execution.
  module ExecutionState
    # Evaluation result execution state is not specified.
    EXECUTION_STATE_UNSPECIFIED = 0

    # Evaluation result execution is running.
    RUNNING = 1

    # Evaluation result execution has completed.
    COMPLETED = 2

    # Evaluation result execution failed due to an internal error.
    ERROR = 3
  end
end

#changelog_create_time ⇒ ::Google::Protobuf::Timestamp (readonly)

Returns Output only. The create time of the changelog of the app version that the evaluation ran against. This is populated if user runs evaluation on latest/draft.

Returns:

  • (::Google::Protobuf::Timestamp) —

    Output only. The create time of the changelog of the app version that the evaluation ran against. This is populated if user runs evaluation on latest/draft.



549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
 
# File 'proto_docs/google/cloud/ces/v1beta/evaluation.rb', line 549

class EvaluationResult
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # Specifies the expectation and the result of that expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
  #     Output only. The result of the tool call expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_tool_response`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_tool_response
  #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
  #     Output only. The result of the tool response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_response`, `observed_tool_call`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The result of the agent response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`, `observed_tool_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_transfer
  #   @return [::Google::Cloud::Ces::V1beta::AgentTransfer]
  #     Output only. The result of the agent transfer expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_transfer`, `observed_tool_call`, `observed_tool_response`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::GoldenExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the expectation.
  # @!attribute [r] semantic_similarity_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
  #     Output only. The result of the semantic similarity check.
  # @!attribute [r] tool_invocation_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome::ToolInvocationResult]
  #     Output only. The result of the tool invocation check.
  class GoldenExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of the tool invocation check.
    # @!attribute [r] parameter_correctness_score
    #   @return [::Float]
    #     Output only. The tool invocation parameter correctness score. This
    #     indicates the percent of parameters from the expected tool call that
    #     were also present in the actual tool call.
    # @!attribute [r] outcome
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
    #     Output only. The outcome of the tool invocation check. This is
    #     determined by comparing the parameter_correctness_score to the
    #     threshold. If the score is equal to or above the threshold, the outcome
    #     will be PASS. Otherwise, the outcome will be FAIL.
    # @!attribute [r] explanation
    #   @return [::String]
    #     Output only. A free text explanation for the tool invocation result.
    class ToolInvocationResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The result of a single evaluation expectation.
  # @!attribute [r] evaluation_expectation
  #   @return [::String]
  #     Output only. The evaluation expectation.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluation_expectation}`
  # @!attribute [r] prompt
  #   @return [::String]
  #     Output only. The prompt that was used for the evaluation.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the evaluation expectation.
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the result.
  class EvaluationExpectationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a golden evaluation.
  # @!attribute [r] turn_replay_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenResult::TurnReplayResult>]
  #     Output only. The result of running each turn of the golden conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class GoldenResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of running a single turn of the golden conversation.
    # @!attribute [r] conversation
    #   @return [::String]
    #     Output only. The conversation that was generated for this turn.
    # @!attribute [r] expectation_outcome
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome>]
    #     Output only. The outcome of each expectation.
    # @!attribute [r] hallucination_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult]
    #     Output only. The result of the hallucination check.
    # @!attribute [r] tool_invocation_score
    #   @deprecated This field is deprecated and may be removed in the next major version update.
    #   @return [::Float]
    #     Output only. Deprecated. Use OverallToolInvocationResult instead.
    # @!attribute [r] tool_ordered_invocation_score
    #   @return [::Float]
    #     Output only. The overall tool ordered invocation score for this turn.
    #     This indicates the overall percent of tools from the expected turn that
    #     were actually invoked in the expected order.
    # @!attribute [r] turn_latency
    #   @return [::Google::Protobuf::Duration]
    #     Output only. Duration of the turn.
    # @!attribute [r] tool_call_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
    #     Output only. The latency of each tool call in the turn.
    # @!attribute [r] semantic_similarity_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
    #     Output only. The result of the semantic similarity check.
    # @!attribute [r] overall_tool_invocation_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::OverallToolInvocationResult]
    #     Output only. The result of the overall tool invocation check.
    # @!attribute [r] error_info
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationErrorInfo]
    #     Output only. Information about the error that occurred during this
    #     turn.
    # @!attribute [r] span_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
    #     Output only. The latency of spans in the turn.
    class TurnReplayResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of the evaluation against the rubric.
  # @!attribute [r] rubric
  #   @return [::String]
  #     Output only. The rubric that was used to evaluate the conversation.
  # @!attribute [r] score
  #   @return [::Float]
  #     Output only. The score of the conversation against the rubric.
  # @!attribute [r] score_explanation
  #   @return [::String]
  #     Output only. The rater's response to the rubric.
  class ScenarioRubricOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of a scenario expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome::ObservedToolCall]
  #     Output only. The observed tool call.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The observed agent response.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::ScenarioExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the ScenarioExpectation.
  class ScenarioExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The observed tool call and response.
    # @!attribute [r] tool_call
    #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
    #     Output only. The observed tool call.
    # @!attribute [r] tool_response
    #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
    #     Output only. The observed tool response.
    class ObservedToolCall
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of a scenario evaluation.
  # @!attribute [r] conversation
  #   @return [::String]
  #     Output only. The conversation that was generated in the scenario.
  # @!attribute [r] task
  #   @return [::String]
  #     Output only. The task that was used when running the scenario for this
  #     result.
  # @!attribute [r] user_facts
  #   @return [::Array<::Google::Cloud::Ces::V1beta::Evaluation::Scenario::UserFact>]
  #     Output only. The user facts that were used by the scenario for this
  #     result.
  # @!attribute [r] expectation_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome>]
  #     Output only. The outcome of each expectation.
  # @!attribute [r] rubric_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioRubricOutcome>]
  #     Output only. The outcome of the rubric.
  # @!attribute [r] hallucination_result
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult>]
  #     Output only. The result of the hallucination check.
  #     There will be one hallucination result for each turn in the conversation.
  # @!attribute [r] task_completion_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::TaskCompletionResult]
  #     Output only. The result of the task completion check.
  # @!attribute [r] tool_call_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
  #     Output only. The latency of each tool call execution in the conversation.
  # @!attribute [r] user_goal_satisfaction_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::UserGoalSatisfactionResult]
  #     Output only. The result of the user goal satisfaction check.
  # @!attribute [r] all_expectations_satisfied
  #   @return [::Boolean]
  #     Output only. Whether all expectations were satisfied for this turn.
  # @!attribute [r] task_completed
  #   @return [::Boolean]
  #     Output only. Whether the task was completed for this turn. This is a
  #     composite of all expectations satisfied, no hallucinations, and user goal
  #     satisfaction.
  # @!attribute [r] span_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
  #     Output only. The latency of spans in the conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class ScenarioResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The latency of a span execution.
  # @!attribute [r] resource
  #   @return [::String]
  #     Output only. The resource name of the guardrail or tool spans.
  #
  #     Note: The following fields are mutually exclusive: `resource`, `toolset`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] toolset
  #   @return [::Google::Cloud::Ces::V1beta::ToolsetTool]
  #     Output only. The toolset tool identifier.
  #
  #     Note: The following fields are mutually exclusive: `toolset`, `resource`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] model
  #   @return [::String]
  #     Output only. The name of the LLM span.
  #
  #     Note: The following fields are mutually exclusive: `model`, `resource`, `toolset`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] callback
  #   @return [::String]
  #     Output only. The name of the user callback span.
  #
  #     Note: The following fields are mutually exclusive: `callback`, `resource`, `toolset`, `model`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] type
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency::Type]
  #     Output only. The type of span.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the span. Applicable to tool and
  #     guardrail spans.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of span.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of span.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of span.
  class SpanLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The type of span.
    # Additional values may be added in the future.
    module Type
      # Default value. This value is unused.
      TYPE_UNSPECIFIED = 0

      # Tool call span.
      TOOL = 1

      # User callback span.
      USER_CALLBACK = 2

      # Guardrail span.
      GUARDRAIL = 3

      # LLM span.
      LLM = 4
    end
  end

  # The latency of a tool call execution.
  # @!attribute [r] tool
  #   @return [::String]
  #     Output only. The name of the tool that got executed.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/tools/{tool}`.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the tool.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of the tool call execution.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of the tool call execution.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of the tool call execution.
  class ToolCallLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the hallucination check for a single turn.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The hallucination score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Justified
  #     Score 0: Not Justified
  #     Score -1: No Claim To Assess
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the hallucination score.
  class HallucinationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a user goal satisfaction check for a conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The user task satisfaction score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: User Task Satisfied
  #     Score 0: User Task Not Satisfied
  #     Score -1: User Task Unspecified
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the user task satisfaction score.
  class UserGoalSatisfactionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the semantic similarity check.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The semantic similarity score. Can be 0, 1, 2, 3, or 4.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 4: Fully Consistent
  #     Score 3: Mostly Consistent
  #     Score 2: Partially Consistent (Minor Omissions)
  #     Score 1: Largely Inconsistent (Major Omissions)
  #     Score 0: Completely Inconsistent / Contradictory
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the semantic similarity score.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the semantic similarity check. This is
  #     determined by comparing the score to the
  #     semantic_similarity_success_threshold. If the score is equal to or above
  #     the threshold, the outcome will be PASS. Otherwise, the outcome will be
  #     FAIL.
  class SemanticSimilarityResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the overall tool invocation check.
  # @!attribute [rw] tool_invocation_score
  #   @return [::Float]
  #     The overall tool invocation score for this turn. This indicates the
  #     overall percent of tools from the expected turn that were actually
  #     invoked.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the tool invocation check. This is determined
  #     by comparing the tool_invocation_score to the
  #     overall_tool_invocation_correctness_threshold. If the
  #     score is equal to or above the threshold, the outcome will be PASS.
  #     Otherwise, the outcome will be FAIL.
  class OverallToolInvocationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the task completion check for the conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The task completion score. Can be -1, 0, 1
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Task Completed
  #     Score 0: Task Not Completed
  #     Score -1: User Goal Undefined
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the task completion score.
  class TaskCompletionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of the evaluation or expectation.
  module Outcome
    # Evaluation outcome is not specified.
    OUTCOME_UNSPECIFIED = 0

    # Evaluation/Expectation passed. In the case of an evaluation, this means
    # that all expectations were met.
    PASS = 1

    # Evaluation/Expectation failed. In the case of an evaluation, this means
    # that at least one expectation was not met.
    FAIL = 2

    # Evaluation/Expectation was skipped.
    SKIPPED = 3
  end

  # The state of the evaluation result execution.
  module ExecutionState
    # Evaluation result execution state is not specified.
    EXECUTION_STATE_UNSPECIFIED = 0

    # Evaluation result execution is running.
    RUNNING = 1

    # Evaluation result execution has completed.
    COMPLETED = 2

    # Evaluation result execution failed due to an internal error.
    ERROR = 3
  end
end

#config ⇒ ::Google::Cloud::Ces::V1beta::EvaluationConfig (readonly)

Returns Output only. The configuration used in the evaluation run that resulted in this result.

Returns:



549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
 
# File 'proto_docs/google/cloud/ces/v1beta/evaluation.rb', line 549

class EvaluationResult
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # Specifies the expectation and the result of that expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
  #     Output only. The result of the tool call expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_tool_response`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_tool_response
  #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
  #     Output only. The result of the tool response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_response`, `observed_tool_call`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The result of the agent response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`, `observed_tool_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_transfer
  #   @return [::Google::Cloud::Ces::V1beta::AgentTransfer]
  #     Output only. The result of the agent transfer expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_transfer`, `observed_tool_call`, `observed_tool_response`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::GoldenExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the expectation.
  # @!attribute [r] semantic_similarity_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
  #     Output only. The result of the semantic similarity check.
  # @!attribute [r] tool_invocation_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome::ToolInvocationResult]
  #     Output only. The result of the tool invocation check.
  class GoldenExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of the tool invocation check.
    # @!attribute [r] parameter_correctness_score
    #   @return [::Float]
    #     Output only. The tool invocation parameter correctness score. This
    #     indicates the percent of parameters from the expected tool call that
    #     were also present in the actual tool call.
    # @!attribute [r] outcome
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
    #     Output only. The outcome of the tool invocation check. This is
    #     determined by comparing the parameter_correctness_score to the
    #     threshold. If the score is equal to or above the threshold, the outcome
    #     will be PASS. Otherwise, the outcome will be FAIL.
    # @!attribute [r] explanation
    #   @return [::String]
    #     Output only. A free text explanation for the tool invocation result.
    class ToolInvocationResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The result of a single evaluation expectation.
  # @!attribute [r] evaluation_expectation
  #   @return [::String]
  #     Output only. The evaluation expectation.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluation_expectation}`
  # @!attribute [r] prompt
  #   @return [::String]
  #     Output only. The prompt that was used for the evaluation.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the evaluation expectation.
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the result.
  class EvaluationExpectationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a golden evaluation.
  # @!attribute [r] turn_replay_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenResult::TurnReplayResult>]
  #     Output only. The result of running each turn of the golden conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class GoldenResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of running a single turn of the golden conversation.
    # @!attribute [r] conversation
    #   @return [::String]
    #     Output only. The conversation that was generated for this turn.
    # @!attribute [r] expectation_outcome
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome>]
    #     Output only. The outcome of each expectation.
    # @!attribute [r] hallucination_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult]
    #     Output only. The result of the hallucination check.
    # @!attribute [r] tool_invocation_score
    #   @deprecated This field is deprecated and may be removed in the next major version update.
    #   @return [::Float]
    #     Output only. Deprecated. Use OverallToolInvocationResult instead.
    # @!attribute [r] tool_ordered_invocation_score
    #   @return [::Float]
    #     Output only. The overall tool ordered invocation score for this turn.
    #     This indicates the overall percent of tools from the expected turn that
    #     were actually invoked in the expected order.
    # @!attribute [r] turn_latency
    #   @return [::Google::Protobuf::Duration]
    #     Output only. Duration of the turn.
    # @!attribute [r] tool_call_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
    #     Output only. The latency of each tool call in the turn.
    # @!attribute [r] semantic_similarity_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
    #     Output only. The result of the semantic similarity check.
    # @!attribute [r] overall_tool_invocation_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::OverallToolInvocationResult]
    #     Output only. The result of the overall tool invocation check.
    # @!attribute [r] error_info
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationErrorInfo]
    #     Output only. Information about the error that occurred during this
    #     turn.
    # @!attribute [r] span_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
    #     Output only. The latency of spans in the turn.
    class TurnReplayResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of the evaluation against the rubric.
  # @!attribute [r] rubric
  #   @return [::String]
  #     Output only. The rubric that was used to evaluate the conversation.
  # @!attribute [r] score
  #   @return [::Float]
  #     Output only. The score of the conversation against the rubric.
  # @!attribute [r] score_explanation
  #   @return [::String]
  #     Output only. The rater's response to the rubric.
  class ScenarioRubricOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of a scenario expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome::ObservedToolCall]
  #     Output only. The observed tool call.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The observed agent response.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::ScenarioExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the ScenarioExpectation.
  class ScenarioExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The observed tool call and response.
    # @!attribute [r] tool_call
    #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
    #     Output only. The observed tool call.
    # @!attribute [r] tool_response
    #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
    #     Output only. The observed tool response.
    class ObservedToolCall
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of a scenario evaluation.
  # @!attribute [r] conversation
  #   @return [::String]
  #     Output only. The conversation that was generated in the scenario.
  # @!attribute [r] task
  #   @return [::String]
  #     Output only. The task that was used when running the scenario for this
  #     result.
  # @!attribute [r] user_facts
  #   @return [::Array<::Google::Cloud::Ces::V1beta::Evaluation::Scenario::UserFact>]
  #     Output only. The user facts that were used by the scenario for this
  #     result.
  # @!attribute [r] expectation_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome>]
  #     Output only. The outcome of each expectation.
  # @!attribute [r] rubric_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioRubricOutcome>]
  #     Output only. The outcome of the rubric.
  # @!attribute [r] hallucination_result
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult>]
  #     Output only. The result of the hallucination check.
  #     There will be one hallucination result for each turn in the conversation.
  # @!attribute [r] task_completion_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::TaskCompletionResult]
  #     Output only. The result of the task completion check.
  # @!attribute [r] tool_call_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
  #     Output only. The latency of each tool call execution in the conversation.
  # @!attribute [r] user_goal_satisfaction_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::UserGoalSatisfactionResult]
  #     Output only. The result of the user goal satisfaction check.
  # @!attribute [r] all_expectations_satisfied
  #   @return [::Boolean]
  #     Output only. Whether all expectations were satisfied for this turn.
  # @!attribute [r] task_completed
  #   @return [::Boolean]
  #     Output only. Whether the task was completed for this turn. This is a
  #     composite of all expectations satisfied, no hallucinations, and user goal
  #     satisfaction.
  # @!attribute [r] span_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
  #     Output only. The latency of spans in the conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class ScenarioResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The latency of a span execution.
  # @!attribute [r] resource
  #   @return [::String]
  #     Output only. The resource name of the guardrail or tool spans.
  #
  #     Note: The following fields are mutually exclusive: `resource`, `toolset`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] toolset
  #   @return [::Google::Cloud::Ces::V1beta::ToolsetTool]
  #     Output only. The toolset tool identifier.
  #
  #     Note: The following fields are mutually exclusive: `toolset`, `resource`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] model
  #   @return [::String]
  #     Output only. The name of the LLM span.
  #
  #     Note: The following fields are mutually exclusive: `model`, `resource`, `toolset`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] callback
  #   @return [::String]
  #     Output only. The name of the user callback span.
  #
  #     Note: The following fields are mutually exclusive: `callback`, `resource`, `toolset`, `model`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] type
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency::Type]
  #     Output only. The type of span.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the span. Applicable to tool and
  #     guardrail spans.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of span.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of span.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of span.
  class SpanLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The type of span.
    # Additional values may be added in the future.
    module Type
      # Default value. This value is unused.
      TYPE_UNSPECIFIED = 0

      # Tool call span.
      TOOL = 1

      # User callback span.
      USER_CALLBACK = 2

      # Guardrail span.
      GUARDRAIL = 3

      # LLM span.
      LLM = 4
    end
  end

  # The latency of a tool call execution.
  # @!attribute [r] tool
  #   @return [::String]
  #     Output only. The name of the tool that got executed.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/tools/{tool}`.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the tool.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of the tool call execution.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of the tool call execution.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of the tool call execution.
  class ToolCallLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the hallucination check for a single turn.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The hallucination score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Justified
  #     Score 0: Not Justified
  #     Score -1: No Claim To Assess
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the hallucination score.
  class HallucinationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a user goal satisfaction check for a conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The user task satisfaction score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: User Task Satisfied
  #     Score 0: User Task Not Satisfied
  #     Score -1: User Task Unspecified
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the user task satisfaction score.
  class UserGoalSatisfactionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the semantic similarity check.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The semantic similarity score. Can be 0, 1, 2, 3, or 4.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 4: Fully Consistent
  #     Score 3: Mostly Consistent
  #     Score 2: Partially Consistent (Minor Omissions)
  #     Score 1: Largely Inconsistent (Major Omissions)
  #     Score 0: Completely Inconsistent / Contradictory
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the semantic similarity score.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the semantic similarity check. This is
  #     determined by comparing the score to the
  #     semantic_similarity_success_threshold. If the score is equal to or above
  #     the threshold, the outcome will be PASS. Otherwise, the outcome will be
  #     FAIL.
  class SemanticSimilarityResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the overall tool invocation check.
  # @!attribute [rw] tool_invocation_score
  #   @return [::Float]
  #     The overall tool invocation score for this turn. This indicates the
  #     overall percent of tools from the expected turn that were actually
  #     invoked.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the tool invocation check. This is determined
  #     by comparing the tool_invocation_score to the
  #     overall_tool_invocation_correctness_threshold. If the
  #     score is equal to or above the threshold, the outcome will be PASS.
  #     Otherwise, the outcome will be FAIL.
  class OverallToolInvocationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the task completion check for the conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The task completion score. Can be -1, 0, 1
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Task Completed
  #     Score 0: Task Not Completed
  #     Score -1: User Goal Undefined
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the task completion score.
  class TaskCompletionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of the evaluation or expectation.
  module Outcome
    # Evaluation outcome is not specified.
    OUTCOME_UNSPECIFIED = 0

    # Evaluation/Expectation passed. In the case of an evaluation, this means
    # that all expectations were met.
    PASS = 1

    # Evaluation/Expectation failed. In the case of an evaluation, this means
    # that at least one expectation was not met.
    FAIL = 2

    # Evaluation/Expectation was skipped.
    SKIPPED = 3
  end

  # The state of the evaluation result execution.
  module ExecutionState
    # Evaluation result execution state is not specified.
    EXECUTION_STATE_UNSPECIFIED = 0

    # Evaluation result execution is running.
    RUNNING = 1

    # Evaluation result execution has completed.
    COMPLETED = 2

    # Evaluation result execution failed due to an internal error.
    ERROR = 3
  end
end

#create_time ⇒ ::Google::Protobuf::Timestamp (readonly)

Returns Output only. Timestamp when the evaluation result was created.

Returns:



549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
 
# File 'proto_docs/google/cloud/ces/v1beta/evaluation.rb', line 549

class EvaluationResult
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # Specifies the expectation and the result of that expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
  #     Output only. The result of the tool call expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_tool_response`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_tool_response
  #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
  #     Output only. The result of the tool response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_response`, `observed_tool_call`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The result of the agent response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`, `observed_tool_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_transfer
  #   @return [::Google::Cloud::Ces::V1beta::AgentTransfer]
  #     Output only. The result of the agent transfer expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_transfer`, `observed_tool_call`, `observed_tool_response`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::GoldenExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the expectation.
  # @!attribute [r] semantic_similarity_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
  #     Output only. The result of the semantic similarity check.
  # @!attribute [r] tool_invocation_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome::ToolInvocationResult]
  #     Output only. The result of the tool invocation check.
  class GoldenExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of the tool invocation check.
    # @!attribute [r] parameter_correctness_score
    #   @return [::Float]
    #     Output only. The tool invocation parameter correctness score. This
    #     indicates the percent of parameters from the expected tool call that
    #     were also present in the actual tool call.
    # @!attribute [r] outcome
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
    #     Output only. The outcome of the tool invocation check. This is
    #     determined by comparing the parameter_correctness_score to the
    #     threshold. If the score is equal to or above the threshold, the outcome
    #     will be PASS. Otherwise, the outcome will be FAIL.
    # @!attribute [r] explanation
    #   @return [::String]
    #     Output only. A free text explanation for the tool invocation result.
    class ToolInvocationResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The result of a single evaluation expectation.
  # @!attribute [r] evaluation_expectation
  #   @return [::String]
  #     Output only. The evaluation expectation.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluation_expectation}`
  # @!attribute [r] prompt
  #   @return [::String]
  #     Output only. The prompt that was used for the evaluation.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the evaluation expectation.
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the result.
  class EvaluationExpectationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a golden evaluation.
  # @!attribute [r] turn_replay_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenResult::TurnReplayResult>]
  #     Output only. The result of running each turn of the golden conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class GoldenResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of running a single turn of the golden conversation.
    # @!attribute [r] conversation
    #   @return [::String]
    #     Output only. The conversation that was generated for this turn.
    # @!attribute [r] expectation_outcome
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome>]
    #     Output only. The outcome of each expectation.
    # @!attribute [r] hallucination_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult]
    #     Output only. The result of the hallucination check.
    # @!attribute [r] tool_invocation_score
    #   @deprecated This field is deprecated and may be removed in the next major version update.
    #   @return [::Float]
    #     Output only. Deprecated. Use OverallToolInvocationResult instead.
    # @!attribute [r] tool_ordered_invocation_score
    #   @return [::Float]
    #     Output only. The overall tool ordered invocation score for this turn.
    #     This indicates the overall percent of tools from the expected turn that
    #     were actually invoked in the expected order.
    # @!attribute [r] turn_latency
    #   @return [::Google::Protobuf::Duration]
    #     Output only. Duration of the turn.
    # @!attribute [r] tool_call_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
    #     Output only. The latency of each tool call in the turn.
    # @!attribute [r] semantic_similarity_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
    #     Output only. The result of the semantic similarity check.
    # @!attribute [r] overall_tool_invocation_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::OverallToolInvocationResult]
    #     Output only. The result of the overall tool invocation check.
    # @!attribute [r] error_info
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationErrorInfo]
    #     Output only. Information about the error that occurred during this
    #     turn.
    # @!attribute [r] span_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
    #     Output only. The latency of spans in the turn.
    class TurnReplayResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of the evaluation against the rubric.
  # @!attribute [r] rubric
  #   @return [::String]
  #     Output only. The rubric that was used to evaluate the conversation.
  # @!attribute [r] score
  #   @return [::Float]
  #     Output only. The score of the conversation against the rubric.
  # @!attribute [r] score_explanation
  #   @return [::String]
  #     Output only. The rater's response to the rubric.
  class ScenarioRubricOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of a scenario expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome::ObservedToolCall]
  #     Output only. The observed tool call.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The observed agent response.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::ScenarioExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the ScenarioExpectation.
  class ScenarioExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The observed tool call and response.
    # @!attribute [r] tool_call
    #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
    #     Output only. The observed tool call.
    # @!attribute [r] tool_response
    #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
    #     Output only. The observed tool response.
    class ObservedToolCall
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of a scenario evaluation.
  # @!attribute [r] conversation
  #   @return [::String]
  #     Output only. The conversation that was generated in the scenario.
  # @!attribute [r] task
  #   @return [::String]
  #     Output only. The task that was used when running the scenario for this
  #     result.
  # @!attribute [r] user_facts
  #   @return [::Array<::Google::Cloud::Ces::V1beta::Evaluation::Scenario::UserFact>]
  #     Output only. The user facts that were used by the scenario for this
  #     result.
  # @!attribute [r] expectation_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome>]
  #     Output only. The outcome of each expectation.
  # @!attribute [r] rubric_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioRubricOutcome>]
  #     Output only. The outcome of the rubric.
  # @!attribute [r] hallucination_result
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult>]
  #     Output only. The result of the hallucination check.
  #     There will be one hallucination result for each turn in the conversation.
  # @!attribute [r] task_completion_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::TaskCompletionResult]
  #     Output only. The result of the task completion check.
  # @!attribute [r] tool_call_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
  #     Output only. The latency of each tool call execution in the conversation.
  # @!attribute [r] user_goal_satisfaction_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::UserGoalSatisfactionResult]
  #     Output only. The result of the user goal satisfaction check.
  # @!attribute [r] all_expectations_satisfied
  #   @return [::Boolean]
  #     Output only. Whether all expectations were satisfied for this turn.
  # @!attribute [r] task_completed
  #   @return [::Boolean]
  #     Output only. Whether the task was completed for this turn. This is a
  #     composite of all expectations satisfied, no hallucinations, and user goal
  #     satisfaction.
  # @!attribute [r] span_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
  #     Output only. The latency of spans in the conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class ScenarioResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The latency of a span execution.
  # @!attribute [r] resource
  #   @return [::String]
  #     Output only. The resource name of the guardrail or tool spans.
  #
  #     Note: The following fields are mutually exclusive: `resource`, `toolset`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] toolset
  #   @return [::Google::Cloud::Ces::V1beta::ToolsetTool]
  #     Output only. The toolset tool identifier.
  #
  #     Note: The following fields are mutually exclusive: `toolset`, `resource`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] model
  #   @return [::String]
  #     Output only. The name of the LLM span.
  #
  #     Note: The following fields are mutually exclusive: `model`, `resource`, `toolset`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] callback
  #   @return [::String]
  #     Output only. The name of the user callback span.
  #
  #     Note: The following fields are mutually exclusive: `callback`, `resource`, `toolset`, `model`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] type
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency::Type]
  #     Output only. The type of span.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the span. Applicable to tool and
  #     guardrail spans.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of span.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of span.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of span.
  class SpanLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The type of span.
    # Additional values may be added in the future.
    module Type
      # Default value. This value is unused.
      TYPE_UNSPECIFIED = 0

      # Tool call span.
      TOOL = 1

      # User callback span.
      USER_CALLBACK = 2

      # Guardrail span.
      GUARDRAIL = 3

      # LLM span.
      LLM = 4
    end
  end

  # The latency of a tool call execution.
  # @!attribute [r] tool
  #   @return [::String]
  #     Output only. The name of the tool that got executed.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/tools/{tool}`.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the tool.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of the tool call execution.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of the tool call execution.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of the tool call execution.
  class ToolCallLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the hallucination check for a single turn.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The hallucination score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Justified
  #     Score 0: Not Justified
  #     Score -1: No Claim To Assess
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the hallucination score.
  class HallucinationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a user goal satisfaction check for a conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The user task satisfaction score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: User Task Satisfied
  #     Score 0: User Task Not Satisfied
  #     Score -1: User Task Unspecified
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the user task satisfaction score.
  class UserGoalSatisfactionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the semantic similarity check.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The semantic similarity score. Can be 0, 1, 2, 3, or 4.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 4: Fully Consistent
  #     Score 3: Mostly Consistent
  #     Score 2: Partially Consistent (Minor Omissions)
  #     Score 1: Largely Inconsistent (Major Omissions)
  #     Score 0: Completely Inconsistent / Contradictory
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the semantic similarity score.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the semantic similarity check. This is
  #     determined by comparing the score to the
  #     semantic_similarity_success_threshold. If the score is equal to or above
  #     the threshold, the outcome will be PASS. Otherwise, the outcome will be
  #     FAIL.
  class SemanticSimilarityResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the overall tool invocation check.
  # @!attribute [rw] tool_invocation_score
  #   @return [::Float]
  #     The overall tool invocation score for this turn. This indicates the
  #     overall percent of tools from the expected turn that were actually
  #     invoked.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the tool invocation check. This is determined
  #     by comparing the tool_invocation_score to the
  #     overall_tool_invocation_correctness_threshold. If the
  #     score is equal to or above the threshold, the outcome will be PASS.
  #     Otherwise, the outcome will be FAIL.
  class OverallToolInvocationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the task completion check for the conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The task completion score. Can be -1, 0, 1
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Task Completed
  #     Score 0: Task Not Completed
  #     Score -1: User Goal Undefined
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the task completion score.
  class TaskCompletionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of the evaluation or expectation.
  module Outcome
    # Evaluation outcome is not specified.
    OUTCOME_UNSPECIFIED = 0

    # Evaluation/Expectation passed. In the case of an evaluation, this means
    # that all expectations were met.
    PASS = 1

    # Evaluation/Expectation failed. In the case of an evaluation, this means
    # that at least one expectation was not met.
    FAIL = 2

    # Evaluation/Expectation was skipped.
    SKIPPED = 3
  end

  # The state of the evaluation result execution.
  module ExecutionState
    # Evaluation result execution state is not specified.
    EXECUTION_STATE_UNSPECIFIED = 0

    # Evaluation result execution is running.
    RUNNING = 1

    # Evaluation result execution has completed.
    COMPLETED = 2

    # Evaluation result execution failed due to an internal error.
    ERROR = 3
  end
end

#display_name ⇒ ::String

Returns Required. Display name of the Evaluation Result. Unique within an Evaluation. By default, it has the following format: " result

  • ".

Returns:

  • (::String) —

    Required. Display name of the Evaluation Result. Unique within an Evaluation. By default, it has the following format: " result

    • ".


549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
 
# File 'proto_docs/google/cloud/ces/v1beta/evaluation.rb', line 549

class EvaluationResult
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # Specifies the expectation and the result of that expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
  #     Output only. The result of the tool call expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_tool_response`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_tool_response
  #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
  #     Output only. The result of the tool response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_response`, `observed_tool_call`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The result of the agent response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`, `observed_tool_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_transfer
  #   @return [::Google::Cloud::Ces::V1beta::AgentTransfer]
  #     Output only. The result of the agent transfer expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_transfer`, `observed_tool_call`, `observed_tool_response`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::GoldenExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the expectation.
  # @!attribute [r] semantic_similarity_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
  #     Output only. The result of the semantic similarity check.
  # @!attribute [r] tool_invocation_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome::ToolInvocationResult]
  #     Output only. The result of the tool invocation check.
  class GoldenExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of the tool invocation check.
    # @!attribute [r] parameter_correctness_score
    #   @return [::Float]
    #     Output only. The tool invocation parameter correctness score. This
    #     indicates the percent of parameters from the expected tool call that
    #     were also present in the actual tool call.
    # @!attribute [r] outcome
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
    #     Output only. The outcome of the tool invocation check. This is
    #     determined by comparing the parameter_correctness_score to the
    #     threshold. If the score is equal to or above the threshold, the outcome
    #     will be PASS. Otherwise, the outcome will be FAIL.
    # @!attribute [r] explanation
    #   @return [::String]
    #     Output only. A free text explanation for the tool invocation result.
    class ToolInvocationResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The result of a single evaluation expectation.
  # @!attribute [r] evaluation_expectation
  #   @return [::String]
  #     Output only. The evaluation expectation.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluation_expectation}`
  # @!attribute [r] prompt
  #   @return [::String]
  #     Output only. The prompt that was used for the evaluation.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the evaluation expectation.
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the result.
  class EvaluationExpectationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a golden evaluation.
  # @!attribute [r] turn_replay_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenResult::TurnReplayResult>]
  #     Output only. The result of running each turn of the golden conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class GoldenResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of running a single turn of the golden conversation.
    # @!attribute [r] conversation
    #   @return [::String]
    #     Output only. The conversation that was generated for this turn.
    # @!attribute [r] expectation_outcome
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome>]
    #     Output only. The outcome of each expectation.
    # @!attribute [r] hallucination_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult]
    #     Output only. The result of the hallucination check.
    # @!attribute [r] tool_invocation_score
    #   @deprecated This field is deprecated and may be removed in the next major version update.
    #   @return [::Float]
    #     Output only. Deprecated. Use OverallToolInvocationResult instead.
    # @!attribute [r] tool_ordered_invocation_score
    #   @return [::Float]
    #     Output only. The overall tool ordered invocation score for this turn.
    #     This indicates the overall percent of tools from the expected turn that
    #     were actually invoked in the expected order.
    # @!attribute [r] turn_latency
    #   @return [::Google::Protobuf::Duration]
    #     Output only. Duration of the turn.
    # @!attribute [r] tool_call_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
    #     Output only. The latency of each tool call in the turn.
    # @!attribute [r] semantic_similarity_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
    #     Output only. The result of the semantic similarity check.
    # @!attribute [r] overall_tool_invocation_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::OverallToolInvocationResult]
    #     Output only. The result of the overall tool invocation check.
    # @!attribute [r] error_info
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationErrorInfo]
    #     Output only. Information about the error that occurred during this
    #     turn.
    # @!attribute [r] span_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
    #     Output only. The latency of spans in the turn.
    class TurnReplayResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of the evaluation against the rubric.
  # @!attribute [r] rubric
  #   @return [::String]
  #     Output only. The rubric that was used to evaluate the conversation.
  # @!attribute [r] score
  #   @return [::Float]
  #     Output only. The score of the conversation against the rubric.
  # @!attribute [r] score_explanation
  #   @return [::String]
  #     Output only. The rater's response to the rubric.
  class ScenarioRubricOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of a scenario expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome::ObservedToolCall]
  #     Output only. The observed tool call.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The observed agent response.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::ScenarioExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the ScenarioExpectation.
  class ScenarioExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The observed tool call and response.
    # @!attribute [r] tool_call
    #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
    #     Output only. The observed tool call.
    # @!attribute [r] tool_response
    #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
    #     Output only. The observed tool response.
    class ObservedToolCall
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of a scenario evaluation.
  # @!attribute [r] conversation
  #   @return [::String]
  #     Output only. The conversation that was generated in the scenario.
  # @!attribute [r] task
  #   @return [::String]
  #     Output only. The task that was used when running the scenario for this
  #     result.
  # @!attribute [r] user_facts
  #   @return [::Array<::Google::Cloud::Ces::V1beta::Evaluation::Scenario::UserFact>]
  #     Output only. The user facts that were used by the scenario for this
  #     result.
  # @!attribute [r] expectation_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome>]
  #     Output only. The outcome of each expectation.
  # @!attribute [r] rubric_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioRubricOutcome>]
  #     Output only. The outcome of the rubric.
  # @!attribute [r] hallucination_result
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult>]
  #     Output only. The result of the hallucination check.
  #     There will be one hallucination result for each turn in the conversation.
  # @!attribute [r] task_completion_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::TaskCompletionResult]
  #     Output only. The result of the task completion check.
  # @!attribute [r] tool_call_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
  #     Output only. The latency of each tool call execution in the conversation.
  # @!attribute [r] user_goal_satisfaction_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::UserGoalSatisfactionResult]
  #     Output only. The result of the user goal satisfaction check.
  # @!attribute [r] all_expectations_satisfied
  #   @return [::Boolean]
  #     Output only. Whether all expectations were satisfied for this turn.
  # @!attribute [r] task_completed
  #   @return [::Boolean]
  #     Output only. Whether the task was completed for this turn. This is a
  #     composite of all expectations satisfied, no hallucinations, and user goal
  #     satisfaction.
  # @!attribute [r] span_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
  #     Output only. The latency of spans in the conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class ScenarioResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The latency of a span execution.
  # @!attribute [r] resource
  #   @return [::String]
  #     Output only. The resource name of the guardrail or tool spans.
  #
  #     Note: The following fields are mutually exclusive: `resource`, `toolset`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] toolset
  #   @return [::Google::Cloud::Ces::V1beta::ToolsetTool]
  #     Output only. The toolset tool identifier.
  #
  #     Note: The following fields are mutually exclusive: `toolset`, `resource`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] model
  #   @return [::String]
  #     Output only. The name of the LLM span.
  #
  #     Note: The following fields are mutually exclusive: `model`, `resource`, `toolset`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] callback
  #   @return [::String]
  #     Output only. The name of the user callback span.
  #
  #     Note: The following fields are mutually exclusive: `callback`, `resource`, `toolset`, `model`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] type
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency::Type]
  #     Output only. The type of span.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the span. Applicable to tool and
  #     guardrail spans.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of span.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of span.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of span.
  class SpanLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The type of span.
    # Additional values may be added in the future.
    module Type
      # Default value. This value is unused.
      TYPE_UNSPECIFIED = 0

      # Tool call span.
      TOOL = 1

      # User callback span.
      USER_CALLBACK = 2

      # Guardrail span.
      GUARDRAIL = 3

      # LLM span.
      LLM = 4
    end
  end

  # The latency of a tool call execution.
  # @!attribute [r] tool
  #   @return [::String]
  #     Output only. The name of the tool that got executed.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/tools/{tool}`.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the tool.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of the tool call execution.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of the tool call execution.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of the tool call execution.
  class ToolCallLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the hallucination check for a single turn.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The hallucination score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Justified
  #     Score 0: Not Justified
  #     Score -1: No Claim To Assess
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the hallucination score.
  class HallucinationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a user goal satisfaction check for a conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The user task satisfaction score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: User Task Satisfied
  #     Score 0: User Task Not Satisfied
  #     Score -1: User Task Unspecified
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the user task satisfaction score.
  class UserGoalSatisfactionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the semantic similarity check.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The semantic similarity score. Can be 0, 1, 2, 3, or 4.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 4: Fully Consistent
  #     Score 3: Mostly Consistent
  #     Score 2: Partially Consistent (Minor Omissions)
  #     Score 1: Largely Inconsistent (Major Omissions)
  #     Score 0: Completely Inconsistent / Contradictory
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the semantic similarity score.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the semantic similarity check. This is
  #     determined by comparing the score to the
  #     semantic_similarity_success_threshold. If the score is equal to or above
  #     the threshold, the outcome will be PASS. Otherwise, the outcome will be
  #     FAIL.
  class SemanticSimilarityResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the overall tool invocation check.
  # @!attribute [rw] tool_invocation_score
  #   @return [::Float]
  #     The overall tool invocation score for this turn. This indicates the
  #     overall percent of tools from the expected turn that were actually
  #     invoked.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the tool invocation check. This is determined
  #     by comparing the tool_invocation_score to the
  #     overall_tool_invocation_correctness_threshold. If the
  #     score is equal to or above the threshold, the outcome will be PASS.
  #     Otherwise, the outcome will be FAIL.
  class OverallToolInvocationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the task completion check for the conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The task completion score. Can be -1, 0, 1
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Task Completed
  #     Score 0: Task Not Completed
  #     Score -1: User Goal Undefined
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the task completion score.
  class TaskCompletionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of the evaluation or expectation.
  module Outcome
    # Evaluation outcome is not specified.
    OUTCOME_UNSPECIFIED = 0

    # Evaluation/Expectation passed. In the case of an evaluation, this means
    # that all expectations were met.
    PASS = 1

    # Evaluation/Expectation failed. In the case of an evaluation, this means
    # that at least one expectation was not met.
    FAIL = 2

    # Evaluation/Expectation was skipped.
    SKIPPED = 3
  end

  # The state of the evaluation result execution.
  module ExecutionState
    # Evaluation result execution state is not specified.
    EXECUTION_STATE_UNSPECIFIED = 0

    # Evaluation result execution is running.
    RUNNING = 1

    # Evaluation result execution has completed.
    COMPLETED = 2

    # Evaluation result execution failed due to an internal error.
    ERROR = 3
  end
end

#error ⇒ ::Google::Rpc::Status (readonly)

Deprecated.

This field is deprecated and may be removed in the next major version update.

Returns Output only. Deprecated: Use error_info instead. Errors encountered during execution.

Returns:

  • (::Google::Rpc::Status) —

    Output only. Deprecated: Use error_info instead. Errors encountered during execution.



549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
 
# File 'proto_docs/google/cloud/ces/v1beta/evaluation.rb', line 549

class EvaluationResult
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # Specifies the expectation and the result of that expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
  #     Output only. The result of the tool call expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_tool_response`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_tool_response
  #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
  #     Output only. The result of the tool response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_response`, `observed_tool_call`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The result of the agent response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`, `observed_tool_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_transfer
  #   @return [::Google::Cloud::Ces::V1beta::AgentTransfer]
  #     Output only. The result of the agent transfer expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_transfer`, `observed_tool_call`, `observed_tool_response`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::GoldenExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the expectation.
  # @!attribute [r] semantic_similarity_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
  #     Output only. The result of the semantic similarity check.
  # @!attribute [r] tool_invocation_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome::ToolInvocationResult]
  #     Output only. The result of the tool invocation check.
  class GoldenExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of the tool invocation check.
    # @!attribute [r] parameter_correctness_score
    #   @return [::Float]
    #     Output only. The tool invocation parameter correctness score. This
    #     indicates the percent of parameters from the expected tool call that
    #     were also present in the actual tool call.
    # @!attribute [r] outcome
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
    #     Output only. The outcome of the tool invocation check. This is
    #     determined by comparing the parameter_correctness_score to the
    #     threshold. If the score is equal to or above the threshold, the outcome
    #     will be PASS. Otherwise, the outcome will be FAIL.
    # @!attribute [r] explanation
    #   @return [::String]
    #     Output only. A free text explanation for the tool invocation result.
    class ToolInvocationResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The result of a single evaluation expectation.
  # @!attribute [r] evaluation_expectation
  #   @return [::String]
  #     Output only. The evaluation expectation.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluation_expectation}`
  # @!attribute [r] prompt
  #   @return [::String]
  #     Output only. The prompt that was used for the evaluation.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the evaluation expectation.
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the result.
  class EvaluationExpectationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a golden evaluation.
  # @!attribute [r] turn_replay_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenResult::TurnReplayResult>]
  #     Output only. The result of running each turn of the golden conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class GoldenResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of running a single turn of the golden conversation.
    # @!attribute [r] conversation
    #   @return [::String]
    #     Output only. The conversation that was generated for this turn.
    # @!attribute [r] expectation_outcome
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome>]
    #     Output only. The outcome of each expectation.
    # @!attribute [r] hallucination_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult]
    #     Output only. The result of the hallucination check.
    # @!attribute [r] tool_invocation_score
    #   @deprecated This field is deprecated and may be removed in the next major version update.
    #   @return [::Float]
    #     Output only. Deprecated. Use OverallToolInvocationResult instead.
    # @!attribute [r] tool_ordered_invocation_score
    #   @return [::Float]
    #     Output only. The overall tool ordered invocation score for this turn.
    #     This indicates the overall percent of tools from the expected turn that
    #     were actually invoked in the expected order.
    # @!attribute [r] turn_latency
    #   @return [::Google::Protobuf::Duration]
    #     Output only. Duration of the turn.
    # @!attribute [r] tool_call_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
    #     Output only. The latency of each tool call in the turn.
    # @!attribute [r] semantic_similarity_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
    #     Output only. The result of the semantic similarity check.
    # @!attribute [r] overall_tool_invocation_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::OverallToolInvocationResult]
    #     Output only. The result of the overall tool invocation check.
    # @!attribute [r] error_info
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationErrorInfo]
    #     Output only. Information about the error that occurred during this
    #     turn.
    # @!attribute [r] span_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
    #     Output only. The latency of spans in the turn.
    class TurnReplayResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of the evaluation against the rubric.
  # @!attribute [r] rubric
  #   @return [::String]
  #     Output only. The rubric that was used to evaluate the conversation.
  # @!attribute [r] score
  #   @return [::Float]
  #     Output only. The score of the conversation against the rubric.
  # @!attribute [r] score_explanation
  #   @return [::String]
  #     Output only. The rater's response to the rubric.
  class ScenarioRubricOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of a scenario expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome::ObservedToolCall]
  #     Output only. The observed tool call.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The observed agent response.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::ScenarioExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the ScenarioExpectation.
  class ScenarioExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The observed tool call and response.
    # @!attribute [r] tool_call
    #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
    #     Output only. The observed tool call.
    # @!attribute [r] tool_response
    #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
    #     Output only. The observed tool response.
    class ObservedToolCall
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of a scenario evaluation.
  # @!attribute [r] conversation
  #   @return [::String]
  #     Output only. The conversation that was generated in the scenario.
  # @!attribute [r] task
  #   @return [::String]
  #     Output only. The task that was used when running the scenario for this
  #     result.
  # @!attribute [r] user_facts
  #   @return [::Array<::Google::Cloud::Ces::V1beta::Evaluation::Scenario::UserFact>]
  #     Output only. The user facts that were used by the scenario for this
  #     result.
  # @!attribute [r] expectation_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome>]
  #     Output only. The outcome of each expectation.
  # @!attribute [r] rubric_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioRubricOutcome>]
  #     Output only. The outcome of the rubric.
  # @!attribute [r] hallucination_result
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult>]
  #     Output only. The result of the hallucination check.
  #     There will be one hallucination result for each turn in the conversation.
  # @!attribute [r] task_completion_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::TaskCompletionResult]
  #     Output only. The result of the task completion check.
  # @!attribute [r] tool_call_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
  #     Output only. The latency of each tool call execution in the conversation.
  # @!attribute [r] user_goal_satisfaction_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::UserGoalSatisfactionResult]
  #     Output only. The result of the user goal satisfaction check.
  # @!attribute [r] all_expectations_satisfied
  #   @return [::Boolean]
  #     Output only. Whether all expectations were satisfied for this turn.
  # @!attribute [r] task_completed
  #   @return [::Boolean]
  #     Output only. Whether the task was completed for this turn. This is a
  #     composite of all expectations satisfied, no hallucinations, and user goal
  #     satisfaction.
  # @!attribute [r] span_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
  #     Output only. The latency of spans in the conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class ScenarioResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The latency of a span execution.
  # @!attribute [r] resource
  #   @return [::String]
  #     Output only. The resource name of the guardrail or tool spans.
  #
  #     Note: The following fields are mutually exclusive: `resource`, `toolset`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] toolset
  #   @return [::Google::Cloud::Ces::V1beta::ToolsetTool]
  #     Output only. The toolset tool identifier.
  #
  #     Note: The following fields are mutually exclusive: `toolset`, `resource`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] model
  #   @return [::String]
  #     Output only. The name of the LLM span.
  #
  #     Note: The following fields are mutually exclusive: `model`, `resource`, `toolset`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] callback
  #   @return [::String]
  #     Output only. The name of the user callback span.
  #
  #     Note: The following fields are mutually exclusive: `callback`, `resource`, `toolset`, `model`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] type
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency::Type]
  #     Output only. The type of span.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the span. Applicable to tool and
  #     guardrail spans.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of span.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of span.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of span.
  class SpanLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The type of span.
    # Additional values may be added in the future.
    module Type
      # Default value. This value is unused.
      TYPE_UNSPECIFIED = 0

      # Tool call span.
      TOOL = 1

      # User callback span.
      USER_CALLBACK = 2

      # Guardrail span.
      GUARDRAIL = 3

      # LLM span.
      LLM = 4
    end
  end

  # The latency of a tool call execution.
  # @!attribute [r] tool
  #   @return [::String]
  #     Output only. The name of the tool that got executed.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/tools/{tool}`.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the tool.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of the tool call execution.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of the tool call execution.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of the tool call execution.
  class ToolCallLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the hallucination check for a single turn.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The hallucination score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Justified
  #     Score 0: Not Justified
  #     Score -1: No Claim To Assess
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the hallucination score.
  class HallucinationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a user goal satisfaction check for a conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The user task satisfaction score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: User Task Satisfied
  #     Score 0: User Task Not Satisfied
  #     Score -1: User Task Unspecified
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the user task satisfaction score.
  class UserGoalSatisfactionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the semantic similarity check.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The semantic similarity score. Can be 0, 1, 2, 3, or 4.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 4: Fully Consistent
  #     Score 3: Mostly Consistent
  #     Score 2: Partially Consistent (Minor Omissions)
  #     Score 1: Largely Inconsistent (Major Omissions)
  #     Score 0: Completely Inconsistent / Contradictory
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the semantic similarity score.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the semantic similarity check. This is
  #     determined by comparing the score to the
  #     semantic_similarity_success_threshold. If the score is equal to or above
  #     the threshold, the outcome will be PASS. Otherwise, the outcome will be
  #     FAIL.
  class SemanticSimilarityResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the overall tool invocation check.
  # @!attribute [rw] tool_invocation_score
  #   @return [::Float]
  #     The overall tool invocation score for this turn. This indicates the
  #     overall percent of tools from the expected turn that were actually
  #     invoked.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the tool invocation check. This is determined
  #     by comparing the tool_invocation_score to the
  #     overall_tool_invocation_correctness_threshold. If the
  #     score is equal to or above the threshold, the outcome will be PASS.
  #     Otherwise, the outcome will be FAIL.
  class OverallToolInvocationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the task completion check for the conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The task completion score. Can be -1, 0, 1
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Task Completed
  #     Score 0: Task Not Completed
  #     Score -1: User Goal Undefined
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the task completion score.
  class TaskCompletionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of the evaluation or expectation.
  module Outcome
    # Evaluation outcome is not specified.
    OUTCOME_UNSPECIFIED = 0

    # Evaluation/Expectation passed. In the case of an evaluation, this means
    # that all expectations were met.
    PASS = 1

    # Evaluation/Expectation failed. In the case of an evaluation, this means
    # that at least one expectation was not met.
    FAIL = 2

    # Evaluation/Expectation was skipped.
    SKIPPED = 3
  end

  # The state of the evaluation result execution.
  module ExecutionState
    # Evaluation result execution state is not specified.
    EXECUTION_STATE_UNSPECIFIED = 0

    # Evaluation result execution is running.
    RUNNING = 1

    # Evaluation result execution has completed.
    COMPLETED = 2

    # Evaluation result execution failed due to an internal error.
    ERROR = 3
  end
end

#error_info ⇒ ::Google::Cloud::Ces::V1beta::EvaluationErrorInfo (readonly)

Returns Output only. Error information for the evaluation result.

Returns:



549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
 
# File 'proto_docs/google/cloud/ces/v1beta/evaluation.rb', line 549

class EvaluationResult
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # Specifies the expectation and the result of that expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
  #     Output only. The result of the tool call expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_tool_response`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_tool_response
  #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
  #     Output only. The result of the tool response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_response`, `observed_tool_call`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The result of the agent response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`, `observed_tool_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_transfer
  #   @return [::Google::Cloud::Ces::V1beta::AgentTransfer]
  #     Output only. The result of the agent transfer expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_transfer`, `observed_tool_call`, `observed_tool_response`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::GoldenExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the expectation.
  # @!attribute [r] semantic_similarity_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
  #     Output only. The result of the semantic similarity check.
  # @!attribute [r] tool_invocation_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome::ToolInvocationResult]
  #     Output only. The result of the tool invocation check.
  class GoldenExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of the tool invocation check.
    # @!attribute [r] parameter_correctness_score
    #   @return [::Float]
    #     Output only. The tool invocation parameter correctness score. This
    #     indicates the percent of parameters from the expected tool call that
    #     were also present in the actual tool call.
    # @!attribute [r] outcome
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
    #     Output only. The outcome of the tool invocation check. This is
    #     determined by comparing the parameter_correctness_score to the
    #     threshold. If the score is equal to or above the threshold, the outcome
    #     will be PASS. Otherwise, the outcome will be FAIL.
    # @!attribute [r] explanation
    #   @return [::String]
    #     Output only. A free text explanation for the tool invocation result.
    class ToolInvocationResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The result of a single evaluation expectation.
  # @!attribute [r] evaluation_expectation
  #   @return [::String]
  #     Output only. The evaluation expectation.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluation_expectation}`
  # @!attribute [r] prompt
  #   @return [::String]
  #     Output only. The prompt that was used for the evaluation.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the evaluation expectation.
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the result.
  class EvaluationExpectationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a golden evaluation.
  # @!attribute [r] turn_replay_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenResult::TurnReplayResult>]
  #     Output only. The result of running each turn of the golden conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class GoldenResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of running a single turn of the golden conversation.
    # @!attribute [r] conversation
    #   @return [::String]
    #     Output only. The conversation that was generated for this turn.
    # @!attribute [r] expectation_outcome
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome>]
    #     Output only. The outcome of each expectation.
    # @!attribute [r] hallucination_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult]
    #     Output only. The result of the hallucination check.
    # @!attribute [r] tool_invocation_score
    #   @deprecated This field is deprecated and may be removed in the next major version update.
    #   @return [::Float]
    #     Output only. Deprecated. Use OverallToolInvocationResult instead.
    # @!attribute [r] tool_ordered_invocation_score
    #   @return [::Float]
    #     Output only. The overall tool ordered invocation score for this turn.
    #     This indicates the overall percent of tools from the expected turn that
    #     were actually invoked in the expected order.
    # @!attribute [r] turn_latency
    #   @return [::Google::Protobuf::Duration]
    #     Output only. Duration of the turn.
    # @!attribute [r] tool_call_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
    #     Output only. The latency of each tool call in the turn.
    # @!attribute [r] semantic_similarity_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
    #     Output only. The result of the semantic similarity check.
    # @!attribute [r] overall_tool_invocation_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::OverallToolInvocationResult]
    #     Output only. The result of the overall tool invocation check.
    # @!attribute [r] error_info
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationErrorInfo]
    #     Output only. Information about the error that occurred during this
    #     turn.
    # @!attribute [r] span_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
    #     Output only. The latency of spans in the turn.
    class TurnReplayResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of the evaluation against the rubric.
  # @!attribute [r] rubric
  #   @return [::String]
  #     Output only. The rubric that was used to evaluate the conversation.
  # @!attribute [r] score
  #   @return [::Float]
  #     Output only. The score of the conversation against the rubric.
  # @!attribute [r] score_explanation
  #   @return [::String]
  #     Output only. The rater's response to the rubric.
  class ScenarioRubricOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of a scenario expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome::ObservedToolCall]
  #     Output only. The observed tool call.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The observed agent response.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::ScenarioExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the ScenarioExpectation.
  class ScenarioExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The observed tool call and response.
    # @!attribute [r] tool_call
    #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
    #     Output only. The observed tool call.
    # @!attribute [r] tool_response
    #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
    #     Output only. The observed tool response.
    class ObservedToolCall
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of a scenario evaluation.
  # @!attribute [r] conversation
  #   @return [::String]
  #     Output only. The conversation that was generated in the scenario.
  # @!attribute [r] task
  #   @return [::String]
  #     Output only. The task that was used when running the scenario for this
  #     result.
  # @!attribute [r] user_facts
  #   @return [::Array<::Google::Cloud::Ces::V1beta::Evaluation::Scenario::UserFact>]
  #     Output only. The user facts that were used by the scenario for this
  #     result.
  # @!attribute [r] expectation_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome>]
  #     Output only. The outcome of each expectation.
  # @!attribute [r] rubric_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioRubricOutcome>]
  #     Output only. The outcome of the rubric.
  # @!attribute [r] hallucination_result
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult>]
  #     Output only. The result of the hallucination check.
  #     There will be one hallucination result for each turn in the conversation.
  # @!attribute [r] task_completion_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::TaskCompletionResult]
  #     Output only. The result of the task completion check.
  # @!attribute [r] tool_call_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
  #     Output only. The latency of each tool call execution in the conversation.
  # @!attribute [r] user_goal_satisfaction_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::UserGoalSatisfactionResult]
  #     Output only. The result of the user goal satisfaction check.
  # @!attribute [r] all_expectations_satisfied
  #   @return [::Boolean]
  #     Output only. Whether all expectations were satisfied for this turn.
  # @!attribute [r] task_completed
  #   @return [::Boolean]
  #     Output only. Whether the task was completed for this turn. This is a
  #     composite of all expectations satisfied, no hallucinations, and user goal
  #     satisfaction.
  # @!attribute [r] span_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
  #     Output only. The latency of spans in the conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class ScenarioResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The latency of a span execution.
  # @!attribute [r] resource
  #   @return [::String]
  #     Output only. The resource name of the guardrail or tool spans.
  #
  #     Note: The following fields are mutually exclusive: `resource`, `toolset`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] toolset
  #   @return [::Google::Cloud::Ces::V1beta::ToolsetTool]
  #     Output only. The toolset tool identifier.
  #
  #     Note: The following fields are mutually exclusive: `toolset`, `resource`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] model
  #   @return [::String]
  #     Output only. The name of the LLM span.
  #
  #     Note: The following fields are mutually exclusive: `model`, `resource`, `toolset`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] callback
  #   @return [::String]
  #     Output only. The name of the user callback span.
  #
  #     Note: The following fields are mutually exclusive: `callback`, `resource`, `toolset`, `model`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] type
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency::Type]
  #     Output only. The type of span.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the span. Applicable to tool and
  #     guardrail spans.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of span.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of span.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of span.
  class SpanLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The type of span.
    # Additional values may be added in the future.
    module Type
      # Default value. This value is unused.
      TYPE_UNSPECIFIED = 0

      # Tool call span.
      TOOL = 1

      # User callback span.
      USER_CALLBACK = 2

      # Guardrail span.
      GUARDRAIL = 3

      # LLM span.
      LLM = 4
    end
  end

  # The latency of a tool call execution.
  # @!attribute [r] tool
  #   @return [::String]
  #     Output only. The name of the tool that got executed.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/tools/{tool}`.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the tool.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of the tool call execution.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of the tool call execution.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of the tool call execution.
  class ToolCallLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the hallucination check for a single turn.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The hallucination score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Justified
  #     Score 0: Not Justified
  #     Score -1: No Claim To Assess
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the hallucination score.
  class HallucinationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a user goal satisfaction check for a conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The user task satisfaction score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: User Task Satisfied
  #     Score 0: User Task Not Satisfied
  #     Score -1: User Task Unspecified
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the user task satisfaction score.
  class UserGoalSatisfactionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the semantic similarity check.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The semantic similarity score. Can be 0, 1, 2, 3, or 4.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 4: Fully Consistent
  #     Score 3: Mostly Consistent
  #     Score 2: Partially Consistent (Minor Omissions)
  #     Score 1: Largely Inconsistent (Major Omissions)
  #     Score 0: Completely Inconsistent / Contradictory
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the semantic similarity score.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the semantic similarity check. This is
  #     determined by comparing the score to the
  #     semantic_similarity_success_threshold. If the score is equal to or above
  #     the threshold, the outcome will be PASS. Otherwise, the outcome will be
  #     FAIL.
  class SemanticSimilarityResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the overall tool invocation check.
  # @!attribute [rw] tool_invocation_score
  #   @return [::Float]
  #     The overall tool invocation score for this turn. This indicates the
  #     overall percent of tools from the expected turn that were actually
  #     invoked.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the tool invocation check. This is determined
  #     by comparing the tool_invocation_score to the
  #     overall_tool_invocation_correctness_threshold. If the
  #     score is equal to or above the threshold, the outcome will be PASS.
  #     Otherwise, the outcome will be FAIL.
  class OverallToolInvocationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the task completion check for the conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The task completion score. Can be -1, 0, 1
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Task Completed
  #     Score 0: Task Not Completed
  #     Score -1: User Goal Undefined
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the task completion score.
  class TaskCompletionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of the evaluation or expectation.
  module Outcome
    # Evaluation outcome is not specified.
    OUTCOME_UNSPECIFIED = 0

    # Evaluation/Expectation passed. In the case of an evaluation, this means
    # that all expectations were met.
    PASS = 1

    # Evaluation/Expectation failed. In the case of an evaluation, this means
    # that at least one expectation was not met.
    FAIL = 2

    # Evaluation/Expectation was skipped.
    SKIPPED = 3
  end

  # The state of the evaluation result execution.
  module ExecutionState
    # Evaluation result execution state is not specified.
    EXECUTION_STATE_UNSPECIFIED = 0

    # Evaluation result execution is running.
    RUNNING = 1

    # Evaluation result execution has completed.
    COMPLETED = 2

    # Evaluation result execution failed due to an internal error.
    ERROR = 3
  end
end

#evaluation_metrics_thresholds ⇒ ::Google::Cloud::Ces::V1beta::EvaluationMetricsThresholds (readonly)

Returns Output only. The evaluation thresholds for the result.

Returns:



549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
 
# File 'proto_docs/google/cloud/ces/v1beta/evaluation.rb', line 549

class EvaluationResult
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # Specifies the expectation and the result of that expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
  #     Output only. The result of the tool call expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_tool_response`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_tool_response
  #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
  #     Output only. The result of the tool response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_response`, `observed_tool_call`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The result of the agent response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`, `observed_tool_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_transfer
  #   @return [::Google::Cloud::Ces::V1beta::AgentTransfer]
  #     Output only. The result of the agent transfer expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_transfer`, `observed_tool_call`, `observed_tool_response`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::GoldenExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the expectation.
  # @!attribute [r] semantic_similarity_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
  #     Output only. The result of the semantic similarity check.
  # @!attribute [r] tool_invocation_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome::ToolInvocationResult]
  #     Output only. The result of the tool invocation check.
  class GoldenExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of the tool invocation check.
    # @!attribute [r] parameter_correctness_score
    #   @return [::Float]
    #     Output only. The tool invocation parameter correctness score. This
    #     indicates the percent of parameters from the expected tool call that
    #     were also present in the actual tool call.
    # @!attribute [r] outcome
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
    #     Output only. The outcome of the tool invocation check. This is
    #     determined by comparing the parameter_correctness_score to the
    #     threshold. If the score is equal to or above the threshold, the outcome
    #     will be PASS. Otherwise, the outcome will be FAIL.
    # @!attribute [r] explanation
    #   @return [::String]
    #     Output only. A free text explanation for the tool invocation result.
    class ToolInvocationResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The result of a single evaluation expectation.
  # @!attribute [r] evaluation_expectation
  #   @return [::String]
  #     Output only. The evaluation expectation.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluation_expectation}`
  # @!attribute [r] prompt
  #   @return [::String]
  #     Output only. The prompt that was used for the evaluation.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the evaluation expectation.
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the result.
  class EvaluationExpectationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a golden evaluation.
  # @!attribute [r] turn_replay_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenResult::TurnReplayResult>]
  #     Output only. The result of running each turn of the golden conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class GoldenResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of running a single turn of the golden conversation.
    # @!attribute [r] conversation
    #   @return [::String]
    #     Output only. The conversation that was generated for this turn.
    # @!attribute [r] expectation_outcome
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome>]
    #     Output only. The outcome of each expectation.
    # @!attribute [r] hallucination_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult]
    #     Output only. The result of the hallucination check.
    # @!attribute [r] tool_invocation_score
    #   @deprecated This field is deprecated and may be removed in the next major version update.
    #   @return [::Float]
    #     Output only. Deprecated. Use OverallToolInvocationResult instead.
    # @!attribute [r] tool_ordered_invocation_score
    #   @return [::Float]
    #     Output only. The overall tool ordered invocation score for this turn.
    #     This indicates the overall percent of tools from the expected turn that
    #     were actually invoked in the expected order.
    # @!attribute [r] turn_latency
    #   @return [::Google::Protobuf::Duration]
    #     Output only. Duration of the turn.
    # @!attribute [r] tool_call_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
    #     Output only. The latency of each tool call in the turn.
    # @!attribute [r] semantic_similarity_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
    #     Output only. The result of the semantic similarity check.
    # @!attribute [r] overall_tool_invocation_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::OverallToolInvocationResult]
    #     Output only. The result of the overall tool invocation check.
    # @!attribute [r] error_info
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationErrorInfo]
    #     Output only. Information about the error that occurred during this
    #     turn.
    # @!attribute [r] span_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
    #     Output only. The latency of spans in the turn.
    class TurnReplayResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of the evaluation against the rubric.
  # @!attribute [r] rubric
  #   @return [::String]
  #     Output only. The rubric that was used to evaluate the conversation.
  # @!attribute [r] score
  #   @return [::Float]
  #     Output only. The score of the conversation against the rubric.
  # @!attribute [r] score_explanation
  #   @return [::String]
  #     Output only. The rater's response to the rubric.
  class ScenarioRubricOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of a scenario expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome::ObservedToolCall]
  #     Output only. The observed tool call.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The observed agent response.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::ScenarioExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the ScenarioExpectation.
  class ScenarioExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The observed tool call and response.
    # @!attribute [r] tool_call
    #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
    #     Output only. The observed tool call.
    # @!attribute [r] tool_response
    #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
    #     Output only. The observed tool response.
    class ObservedToolCall
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of a scenario evaluation.
  # @!attribute [r] conversation
  #   @return [::String]
  #     Output only. The conversation that was generated in the scenario.
  # @!attribute [r] task
  #   @return [::String]
  #     Output only. The task that was used when running the scenario for this
  #     result.
  # @!attribute [r] user_facts
  #   @return [::Array<::Google::Cloud::Ces::V1beta::Evaluation::Scenario::UserFact>]
  #     Output only. The user facts that were used by the scenario for this
  #     result.
  # @!attribute [r] expectation_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome>]
  #     Output only. The outcome of each expectation.
  # @!attribute [r] rubric_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioRubricOutcome>]
  #     Output only. The outcome of the rubric.
  # @!attribute [r] hallucination_result
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult>]
  #     Output only. The result of the hallucination check.
  #     There will be one hallucination result for each turn in the conversation.
  # @!attribute [r] task_completion_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::TaskCompletionResult]
  #     Output only. The result of the task completion check.
  # @!attribute [r] tool_call_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
  #     Output only. The latency of each tool call execution in the conversation.
  # @!attribute [r] user_goal_satisfaction_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::UserGoalSatisfactionResult]
  #     Output only. The result of the user goal satisfaction check.
  # @!attribute [r] all_expectations_satisfied
  #   @return [::Boolean]
  #     Output only. Whether all expectations were satisfied for this turn.
  # @!attribute [r] task_completed
  #   @return [::Boolean]
  #     Output only. Whether the task was completed for this turn. This is a
  #     composite of all expectations satisfied, no hallucinations, and user goal
  #     satisfaction.
  # @!attribute [r] span_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
  #     Output only. The latency of spans in the conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class ScenarioResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The latency of a span execution.
  # @!attribute [r] resource
  #   @return [::String]
  #     Output only. The resource name of the guardrail or tool spans.
  #
  #     Note: The following fields are mutually exclusive: `resource`, `toolset`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] toolset
  #   @return [::Google::Cloud::Ces::V1beta::ToolsetTool]
  #     Output only. The toolset tool identifier.
  #
  #     Note: The following fields are mutually exclusive: `toolset`, `resource`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] model
  #   @return [::String]
  #     Output only. The name of the LLM span.
  #
  #     Note: The following fields are mutually exclusive: `model`, `resource`, `toolset`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] callback
  #   @return [::String]
  #     Output only. The name of the user callback span.
  #
  #     Note: The following fields are mutually exclusive: `callback`, `resource`, `toolset`, `model`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] type
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency::Type]
  #     Output only. The type of span.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the span. Applicable to tool and
  #     guardrail spans.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of span.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of span.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of span.
  class SpanLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The type of span.
    # Additional values may be added in the future.
    module Type
      # Default value. This value is unused.
      TYPE_UNSPECIFIED = 0

      # Tool call span.
      TOOL = 1

      # User callback span.
      USER_CALLBACK = 2

      # Guardrail span.
      GUARDRAIL = 3

      # LLM span.
      LLM = 4
    end
  end

  # The latency of a tool call execution.
  # @!attribute [r] tool
  #   @return [::String]
  #     Output only. The name of the tool that got executed.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/tools/{tool}`.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the tool.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of the tool call execution.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of the tool call execution.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of the tool call execution.
  class ToolCallLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the hallucination check for a single turn.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The hallucination score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Justified
  #     Score 0: Not Justified
  #     Score -1: No Claim To Assess
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the hallucination score.
  class HallucinationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a user goal satisfaction check for a conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The user task satisfaction score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: User Task Satisfied
  #     Score 0: User Task Not Satisfied
  #     Score -1: User Task Unspecified
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the user task satisfaction score.
  class UserGoalSatisfactionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the semantic similarity check.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The semantic similarity score. Can be 0, 1, 2, 3, or 4.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 4: Fully Consistent
  #     Score 3: Mostly Consistent
  #     Score 2: Partially Consistent (Minor Omissions)
  #     Score 1: Largely Inconsistent (Major Omissions)
  #     Score 0: Completely Inconsistent / Contradictory
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the semantic similarity score.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the semantic similarity check. This is
  #     determined by comparing the score to the
  #     semantic_similarity_success_threshold. If the score is equal to or above
  #     the threshold, the outcome will be PASS. Otherwise, the outcome will be
  #     FAIL.
  class SemanticSimilarityResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the overall tool invocation check.
  # @!attribute [rw] tool_invocation_score
  #   @return [::Float]
  #     The overall tool invocation score for this turn. This indicates the
  #     overall percent of tools from the expected turn that were actually
  #     invoked.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the tool invocation check. This is determined
  #     by comparing the tool_invocation_score to the
  #     overall_tool_invocation_correctness_threshold. If the
  #     score is equal to or above the threshold, the outcome will be PASS.
  #     Otherwise, the outcome will be FAIL.
  class OverallToolInvocationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the task completion check for the conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The task completion score. Can be -1, 0, 1
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Task Completed
  #     Score 0: Task Not Completed
  #     Score -1: User Goal Undefined
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the task completion score.
  class TaskCompletionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of the evaluation or expectation.
  module Outcome
    # Evaluation outcome is not specified.
    OUTCOME_UNSPECIFIED = 0

    # Evaluation/Expectation passed. In the case of an evaluation, this means
    # that all expectations were met.
    PASS = 1

    # Evaluation/Expectation failed. In the case of an evaluation, this means
    # that at least one expectation was not met.
    FAIL = 2

    # Evaluation/Expectation was skipped.
    SKIPPED = 3
  end

  # The state of the evaluation result execution.
  module ExecutionState
    # Evaluation result execution state is not specified.
    EXECUTION_STATE_UNSPECIFIED = 0

    # Evaluation result execution is running.
    RUNNING = 1

    # Evaluation result execution has completed.
    COMPLETED = 2

    # Evaluation result execution failed due to an internal error.
    ERROR = 3
  end
end

#evaluation_run ⇒ ::String (readonly)

Returns Output only. The evaluation run that produced this result. Format: projects/{project}/locations/{location}/apps/{app}/evaluationRuns/{evaluationRun}.

Returns:

  • (::String) —

    Output only. The evaluation run that produced this result. Format: projects/{project}/locations/{location}/apps/{app}/evaluationRuns/{evaluationRun}



549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
 
# File 'proto_docs/google/cloud/ces/v1beta/evaluation.rb', line 549

class EvaluationResult
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # Specifies the expectation and the result of that expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
  #     Output only. The result of the tool call expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_tool_response`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_tool_response
  #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
  #     Output only. The result of the tool response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_response`, `observed_tool_call`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The result of the agent response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`, `observed_tool_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_transfer
  #   @return [::Google::Cloud::Ces::V1beta::AgentTransfer]
  #     Output only. The result of the agent transfer expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_transfer`, `observed_tool_call`, `observed_tool_response`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::GoldenExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the expectation.
  # @!attribute [r] semantic_similarity_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
  #     Output only. The result of the semantic similarity check.
  # @!attribute [r] tool_invocation_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome::ToolInvocationResult]
  #     Output only. The result of the tool invocation check.
  class GoldenExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of the tool invocation check.
    # @!attribute [r] parameter_correctness_score
    #   @return [::Float]
    #     Output only. The tool invocation parameter correctness score. This
    #     indicates the percent of parameters from the expected tool call that
    #     were also present in the actual tool call.
    # @!attribute [r] outcome
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
    #     Output only. The outcome of the tool invocation check. This is
    #     determined by comparing the parameter_correctness_score to the
    #     threshold. If the score is equal to or above the threshold, the outcome
    #     will be PASS. Otherwise, the outcome will be FAIL.
    # @!attribute [r] explanation
    #   @return [::String]
    #     Output only. A free text explanation for the tool invocation result.
    class ToolInvocationResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The result of a single evaluation expectation.
  # @!attribute [r] evaluation_expectation
  #   @return [::String]
  #     Output only. The evaluation expectation.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluation_expectation}`
  # @!attribute [r] prompt
  #   @return [::String]
  #     Output only. The prompt that was used for the evaluation.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the evaluation expectation.
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the result.
  class EvaluationExpectationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a golden evaluation.
  # @!attribute [r] turn_replay_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenResult::TurnReplayResult>]
  #     Output only. The result of running each turn of the golden conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class GoldenResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of running a single turn of the golden conversation.
    # @!attribute [r] conversation
    #   @return [::String]
    #     Output only. The conversation that was generated for this turn.
    # @!attribute [r] expectation_outcome
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome>]
    #     Output only. The outcome of each expectation.
    # @!attribute [r] hallucination_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult]
    #     Output only. The result of the hallucination check.
    # @!attribute [r] tool_invocation_score
    #   @deprecated This field is deprecated and may be removed in the next major version update.
    #   @return [::Float]
    #     Output only. Deprecated. Use OverallToolInvocationResult instead.
    # @!attribute [r] tool_ordered_invocation_score
    #   @return [::Float]
    #     Output only. The overall tool ordered invocation score for this turn.
    #     This indicates the overall percent of tools from the expected turn that
    #     were actually invoked in the expected order.
    # @!attribute [r] turn_latency
    #   @return [::Google::Protobuf::Duration]
    #     Output only. Duration of the turn.
    # @!attribute [r] tool_call_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
    #     Output only. The latency of each tool call in the turn.
    # @!attribute [r] semantic_similarity_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
    #     Output only. The result of the semantic similarity check.
    # @!attribute [r] overall_tool_invocation_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::OverallToolInvocationResult]
    #     Output only. The result of the overall tool invocation check.
    # @!attribute [r] error_info
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationErrorInfo]
    #     Output only. Information about the error that occurred during this
    #     turn.
    # @!attribute [r] span_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
    #     Output only. The latency of spans in the turn.
    class TurnReplayResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of the evaluation against the rubric.
  # @!attribute [r] rubric
  #   @return [::String]
  #     Output only. The rubric that was used to evaluate the conversation.
  # @!attribute [r] score
  #   @return [::Float]
  #     Output only. The score of the conversation against the rubric.
  # @!attribute [r] score_explanation
  #   @return [::String]
  #     Output only. The rater's response to the rubric.
  class ScenarioRubricOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of a scenario expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome::ObservedToolCall]
  #     Output only. The observed tool call.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The observed agent response.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::ScenarioExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the ScenarioExpectation.
  class ScenarioExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The observed tool call and response.
    # @!attribute [r] tool_call
    #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
    #     Output only. The observed tool call.
    # @!attribute [r] tool_response
    #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
    #     Output only. The observed tool response.
    class ObservedToolCall
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of a scenario evaluation.
  # @!attribute [r] conversation
  #   @return [::String]
  #     Output only. The conversation that was generated in the scenario.
  # @!attribute [r] task
  #   @return [::String]
  #     Output only. The task that was used when running the scenario for this
  #     result.
  # @!attribute [r] user_facts
  #   @return [::Array<::Google::Cloud::Ces::V1beta::Evaluation::Scenario::UserFact>]
  #     Output only. The user facts that were used by the scenario for this
  #     result.
  # @!attribute [r] expectation_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome>]
  #     Output only. The outcome of each expectation.
  # @!attribute [r] rubric_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioRubricOutcome>]
  #     Output only. The outcome of the rubric.
  # @!attribute [r] hallucination_result
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult>]
  #     Output only. The result of the hallucination check.
  #     There will be one hallucination result for each turn in the conversation.
  # @!attribute [r] task_completion_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::TaskCompletionResult]
  #     Output only. The result of the task completion check.
  # @!attribute [r] tool_call_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
  #     Output only. The latency of each tool call execution in the conversation.
  # @!attribute [r] user_goal_satisfaction_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::UserGoalSatisfactionResult]
  #     Output only. The result of the user goal satisfaction check.
  # @!attribute [r] all_expectations_satisfied
  #   @return [::Boolean]
  #     Output only. Whether all expectations were satisfied for this turn.
  # @!attribute [r] task_completed
  #   @return [::Boolean]
  #     Output only. Whether the task was completed for this turn. This is a
  #     composite of all expectations satisfied, no hallucinations, and user goal
  #     satisfaction.
  # @!attribute [r] span_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
  #     Output only. The latency of spans in the conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class ScenarioResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The latency of a span execution.
  # @!attribute [r] resource
  #   @return [::String]
  #     Output only. The resource name of the guardrail or tool spans.
  #
  #     Note: The following fields are mutually exclusive: `resource`, `toolset`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] toolset
  #   @return [::Google::Cloud::Ces::V1beta::ToolsetTool]
  #     Output only. The toolset tool identifier.
  #
  #     Note: The following fields are mutually exclusive: `toolset`, `resource`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] model
  #   @return [::String]
  #     Output only. The name of the LLM span.
  #
  #     Note: The following fields are mutually exclusive: `model`, `resource`, `toolset`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] callback
  #   @return [::String]
  #     Output only. The name of the user callback span.
  #
  #     Note: The following fields are mutually exclusive: `callback`, `resource`, `toolset`, `model`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] type
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency::Type]
  #     Output only. The type of span.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the span. Applicable to tool and
  #     guardrail spans.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of span.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of span.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of span.
  class SpanLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The type of span.
    # Additional values may be added in the future.
    module Type
      # Default value. This value is unused.
      TYPE_UNSPECIFIED = 0

      # Tool call span.
      TOOL = 1

      # User callback span.
      USER_CALLBACK = 2

      # Guardrail span.
      GUARDRAIL = 3

      # LLM span.
      LLM = 4
    end
  end

  # The latency of a tool call execution.
  # @!attribute [r] tool
  #   @return [::String]
  #     Output only. The name of the tool that got executed.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/tools/{tool}`.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the tool.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of the tool call execution.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of the tool call execution.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of the tool call execution.
  class ToolCallLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the hallucination check for a single turn.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The hallucination score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Justified
  #     Score 0: Not Justified
  #     Score -1: No Claim To Assess
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the hallucination score.
  class HallucinationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a user goal satisfaction check for a conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The user task satisfaction score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: User Task Satisfied
  #     Score 0: User Task Not Satisfied
  #     Score -1: User Task Unspecified
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the user task satisfaction score.
  class UserGoalSatisfactionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the semantic similarity check.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The semantic similarity score. Can be 0, 1, 2, 3, or 4.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 4: Fully Consistent
  #     Score 3: Mostly Consistent
  #     Score 2: Partially Consistent (Minor Omissions)
  #     Score 1: Largely Inconsistent (Major Omissions)
  #     Score 0: Completely Inconsistent / Contradictory
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the semantic similarity score.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the semantic similarity check. This is
  #     determined by comparing the score to the
  #     semantic_similarity_success_threshold. If the score is equal to or above
  #     the threshold, the outcome will be PASS. Otherwise, the outcome will be
  #     FAIL.
  class SemanticSimilarityResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the overall tool invocation check.
  # @!attribute [rw] tool_invocation_score
  #   @return [::Float]
  #     The overall tool invocation score for this turn. This indicates the
  #     overall percent of tools from the expected turn that were actually
  #     invoked.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the tool invocation check. This is determined
  #     by comparing the tool_invocation_score to the
  #     overall_tool_invocation_correctness_threshold. If the
  #     score is equal to or above the threshold, the outcome will be PASS.
  #     Otherwise, the outcome will be FAIL.
  class OverallToolInvocationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the task completion check for the conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The task completion score. Can be -1, 0, 1
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Task Completed
  #     Score 0: Task Not Completed
  #     Score -1: User Goal Undefined
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the task completion score.
  class TaskCompletionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of the evaluation or expectation.
  module Outcome
    # Evaluation outcome is not specified.
    OUTCOME_UNSPECIFIED = 0

    # Evaluation/Expectation passed. In the case of an evaluation, this means
    # that all expectations were met.
    PASS = 1

    # Evaluation/Expectation failed. In the case of an evaluation, this means
    # that at least one expectation was not met.
    FAIL = 2

    # Evaluation/Expectation was skipped.
    SKIPPED = 3
  end

  # The state of the evaluation result execution.
  module ExecutionState
    # Evaluation result execution state is not specified.
    EXECUTION_STATE_UNSPECIFIED = 0

    # Evaluation result execution is running.
    RUNNING = 1

    # Evaluation result execution has completed.
    COMPLETED = 2

    # Evaluation result execution failed due to an internal error.
    ERROR = 3
  end
end

#evaluation_status ⇒ ::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome (readonly)

Returns Output only. The outcome of the evaluation. Only populated if execution_state is COMPLETE.

Returns:



549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
 
# File 'proto_docs/google/cloud/ces/v1beta/evaluation.rb', line 549

class EvaluationResult
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # Specifies the expectation and the result of that expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
  #     Output only. The result of the tool call expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_tool_response`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_tool_response
  #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
  #     Output only. The result of the tool response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_response`, `observed_tool_call`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The result of the agent response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`, `observed_tool_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_transfer
  #   @return [::Google::Cloud::Ces::V1beta::AgentTransfer]
  #     Output only. The result of the agent transfer expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_transfer`, `observed_tool_call`, `observed_tool_response`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::GoldenExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the expectation.
  # @!attribute [r] semantic_similarity_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
  #     Output only. The result of the semantic similarity check.
  # @!attribute [r] tool_invocation_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome::ToolInvocationResult]
  #     Output only. The result of the tool invocation check.
  class GoldenExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of the tool invocation check.
    # @!attribute [r] parameter_correctness_score
    #   @return [::Float]
    #     Output only. The tool invocation parameter correctness score. This
    #     indicates the percent of parameters from the expected tool call that
    #     were also present in the actual tool call.
    # @!attribute [r] outcome
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
    #     Output only. The outcome of the tool invocation check. This is
    #     determined by comparing the parameter_correctness_score to the
    #     threshold. If the score is equal to or above the threshold, the outcome
    #     will be PASS. Otherwise, the outcome will be FAIL.
    # @!attribute [r] explanation
    #   @return [::String]
    #     Output only. A free text explanation for the tool invocation result.
    class ToolInvocationResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The result of a single evaluation expectation.
  # @!attribute [r] evaluation_expectation
  #   @return [::String]
  #     Output only. The evaluation expectation.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluation_expectation}`
  # @!attribute [r] prompt
  #   @return [::String]
  #     Output only. The prompt that was used for the evaluation.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the evaluation expectation.
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the result.
  class EvaluationExpectationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a golden evaluation.
  # @!attribute [r] turn_replay_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenResult::TurnReplayResult>]
  #     Output only. The result of running each turn of the golden conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class GoldenResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of running a single turn of the golden conversation.
    # @!attribute [r] conversation
    #   @return [::String]
    #     Output only. The conversation that was generated for this turn.
    # @!attribute [r] expectation_outcome
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome>]
    #     Output only. The outcome of each expectation.
    # @!attribute [r] hallucination_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult]
    #     Output only. The result of the hallucination check.
    # @!attribute [r] tool_invocation_score
    #   @deprecated This field is deprecated and may be removed in the next major version update.
    #   @return [::Float]
    #     Output only. Deprecated. Use OverallToolInvocationResult instead.
    # @!attribute [r] tool_ordered_invocation_score
    #   @return [::Float]
    #     Output only. The overall tool ordered invocation score for this turn.
    #     This indicates the overall percent of tools from the expected turn that
    #     were actually invoked in the expected order.
    # @!attribute [r] turn_latency
    #   @return [::Google::Protobuf::Duration]
    #     Output only. Duration of the turn.
    # @!attribute [r] tool_call_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
    #     Output only. The latency of each tool call in the turn.
    # @!attribute [r] semantic_similarity_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
    #     Output only. The result of the semantic similarity check.
    # @!attribute [r] overall_tool_invocation_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::OverallToolInvocationResult]
    #     Output only. The result of the overall tool invocation check.
    # @!attribute [r] error_info
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationErrorInfo]
    #     Output only. Information about the error that occurred during this
    #     turn.
    # @!attribute [r] span_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
    #     Output only. The latency of spans in the turn.
    class TurnReplayResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of the evaluation against the rubric.
  # @!attribute [r] rubric
  #   @return [::String]
  #     Output only. The rubric that was used to evaluate the conversation.
  # @!attribute [r] score
  #   @return [::Float]
  #     Output only. The score of the conversation against the rubric.
  # @!attribute [r] score_explanation
  #   @return [::String]
  #     Output only. The rater's response to the rubric.
  class ScenarioRubricOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of a scenario expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome::ObservedToolCall]
  #     Output only. The observed tool call.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The observed agent response.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::ScenarioExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the ScenarioExpectation.
  class ScenarioExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The observed tool call and response.
    # @!attribute [r] tool_call
    #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
    #     Output only. The observed tool call.
    # @!attribute [r] tool_response
    #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
    #     Output only. The observed tool response.
    class ObservedToolCall
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of a scenario evaluation.
  # @!attribute [r] conversation
  #   @return [::String]
  #     Output only. The conversation that was generated in the scenario.
  # @!attribute [r] task
  #   @return [::String]
  #     Output only. The task that was used when running the scenario for this
  #     result.
  # @!attribute [r] user_facts
  #   @return [::Array<::Google::Cloud::Ces::V1beta::Evaluation::Scenario::UserFact>]
  #     Output only. The user facts that were used by the scenario for this
  #     result.
  # @!attribute [r] expectation_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome>]
  #     Output only. The outcome of each expectation.
  # @!attribute [r] rubric_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioRubricOutcome>]
  #     Output only. The outcome of the rubric.
  # @!attribute [r] hallucination_result
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult>]
  #     Output only. The result of the hallucination check.
  #     There will be one hallucination result for each turn in the conversation.
  # @!attribute [r] task_completion_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::TaskCompletionResult]
  #     Output only. The result of the task completion check.
  # @!attribute [r] tool_call_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
  #     Output only. The latency of each tool call execution in the conversation.
  # @!attribute [r] user_goal_satisfaction_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::UserGoalSatisfactionResult]
  #     Output only. The result of the user goal satisfaction check.
  # @!attribute [r] all_expectations_satisfied
  #   @return [::Boolean]
  #     Output only. Whether all expectations were satisfied for this turn.
  # @!attribute [r] task_completed
  #   @return [::Boolean]
  #     Output only. Whether the task was completed for this turn. This is a
  #     composite of all expectations satisfied, no hallucinations, and user goal
  #     satisfaction.
  # @!attribute [r] span_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
  #     Output only. The latency of spans in the conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class ScenarioResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The latency of a span execution.
  # @!attribute [r] resource
  #   @return [::String]
  #     Output only. The resource name of the guardrail or tool spans.
  #
  #     Note: The following fields are mutually exclusive: `resource`, `toolset`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] toolset
  #   @return [::Google::Cloud::Ces::V1beta::ToolsetTool]
  #     Output only. The toolset tool identifier.
  #
  #     Note: The following fields are mutually exclusive: `toolset`, `resource`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] model
  #   @return [::String]
  #     Output only. The name of the LLM span.
  #
  #     Note: The following fields are mutually exclusive: `model`, `resource`, `toolset`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] callback
  #   @return [::String]
  #     Output only. The name of the user callback span.
  #
  #     Note: The following fields are mutually exclusive: `callback`, `resource`, `toolset`, `model`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] type
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency::Type]
  #     Output only. The type of span.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the span. Applicable to tool and
  #     guardrail spans.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of span.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of span.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of span.
  class SpanLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The type of span.
    # Additional values may be added in the future.
    module Type
      # Default value. This value is unused.
      TYPE_UNSPECIFIED = 0

      # Tool call span.
      TOOL = 1

      # User callback span.
      USER_CALLBACK = 2

      # Guardrail span.
      GUARDRAIL = 3

      # LLM span.
      LLM = 4
    end
  end

  # The latency of a tool call execution.
  # @!attribute [r] tool
  #   @return [::String]
  #     Output only. The name of the tool that got executed.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/tools/{tool}`.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the tool.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of the tool call execution.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of the tool call execution.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of the tool call execution.
  class ToolCallLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the hallucination check for a single turn.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The hallucination score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Justified
  #     Score 0: Not Justified
  #     Score -1: No Claim To Assess
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the hallucination score.
  class HallucinationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a user goal satisfaction check for a conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The user task satisfaction score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: User Task Satisfied
  #     Score 0: User Task Not Satisfied
  #     Score -1: User Task Unspecified
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the user task satisfaction score.
  class UserGoalSatisfactionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the semantic similarity check.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The semantic similarity score. Can be 0, 1, 2, 3, or 4.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 4: Fully Consistent
  #     Score 3: Mostly Consistent
  #     Score 2: Partially Consistent (Minor Omissions)
  #     Score 1: Largely Inconsistent (Major Omissions)
  #     Score 0: Completely Inconsistent / Contradictory
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the semantic similarity score.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the semantic similarity check. This is
  #     determined by comparing the score to the
  #     semantic_similarity_success_threshold. If the score is equal to or above
  #     the threshold, the outcome will be PASS. Otherwise, the outcome will be
  #     FAIL.
  class SemanticSimilarityResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the overall tool invocation check.
  # @!attribute [rw] tool_invocation_score
  #   @return [::Float]
  #     The overall tool invocation score for this turn. This indicates the
  #     overall percent of tools from the expected turn that were actually
  #     invoked.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the tool invocation check. This is determined
  #     by comparing the tool_invocation_score to the
  #     overall_tool_invocation_correctness_threshold. If the
  #     score is equal to or above the threshold, the outcome will be PASS.
  #     Otherwise, the outcome will be FAIL.
  class OverallToolInvocationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the task completion check for the conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The task completion score. Can be -1, 0, 1
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Task Completed
  #     Score 0: Task Not Completed
  #     Score -1: User Goal Undefined
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the task completion score.
  class TaskCompletionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of the evaluation or expectation.
  module Outcome
    # Evaluation outcome is not specified.
    OUTCOME_UNSPECIFIED = 0

    # Evaluation/Expectation passed. In the case of an evaluation, this means
    # that all expectations were met.
    PASS = 1

    # Evaluation/Expectation failed. In the case of an evaluation, this means
    # that at least one expectation was not met.
    FAIL = 2

    # Evaluation/Expectation was skipped.
    SKIPPED = 3
  end

  # The state of the evaluation result execution.
  module ExecutionState
    # Evaluation result execution state is not specified.
    EXECUTION_STATE_UNSPECIFIED = 0

    # Evaluation result execution is running.
    RUNNING = 1

    # Evaluation result execution has completed.
    COMPLETED = 2

    # Evaluation result execution failed due to an internal error.
    ERROR = 3
  end
end

#execution_state ⇒ ::Google::Cloud::Ces::V1beta::EvaluationResult::ExecutionState (readonly)

Returns Output only. The state of the evaluation result execution.

Returns:



549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
 
# File 'proto_docs/google/cloud/ces/v1beta/evaluation.rb', line 549

class EvaluationResult
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # Specifies the expectation and the result of that expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
  #     Output only. The result of the tool call expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_tool_response`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_tool_response
  #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
  #     Output only. The result of the tool response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_response`, `observed_tool_call`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The result of the agent response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`, `observed_tool_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_transfer
  #   @return [::Google::Cloud::Ces::V1beta::AgentTransfer]
  #     Output only. The result of the agent transfer expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_transfer`, `observed_tool_call`, `observed_tool_response`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::GoldenExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the expectation.
  # @!attribute [r] semantic_similarity_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
  #     Output only. The result of the semantic similarity check.
  # @!attribute [r] tool_invocation_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome::ToolInvocationResult]
  #     Output only. The result of the tool invocation check.
  class GoldenExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of the tool invocation check.
    # @!attribute [r] parameter_correctness_score
    #   @return [::Float]
    #     Output only. The tool invocation parameter correctness score. This
    #     indicates the percent of parameters from the expected tool call that
    #     were also present in the actual tool call.
    # @!attribute [r] outcome
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
    #     Output only. The outcome of the tool invocation check. This is
    #     determined by comparing the parameter_correctness_score to the
    #     threshold. If the score is equal to or above the threshold, the outcome
    #     will be PASS. Otherwise, the outcome will be FAIL.
    # @!attribute [r] explanation
    #   @return [::String]
    #     Output only. A free text explanation for the tool invocation result.
    class ToolInvocationResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The result of a single evaluation expectation.
  # @!attribute [r] evaluation_expectation
  #   @return [::String]
  #     Output only. The evaluation expectation.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluation_expectation}`
  # @!attribute [r] prompt
  #   @return [::String]
  #     Output only. The prompt that was used for the evaluation.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the evaluation expectation.
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the result.
  class EvaluationExpectationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a golden evaluation.
  # @!attribute [r] turn_replay_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenResult::TurnReplayResult>]
  #     Output only. The result of running each turn of the golden conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class GoldenResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of running a single turn of the golden conversation.
    # @!attribute [r] conversation
    #   @return [::String]
    #     Output only. The conversation that was generated for this turn.
    # @!attribute [r] expectation_outcome
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome>]
    #     Output only. The outcome of each expectation.
    # @!attribute [r] hallucination_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult]
    #     Output only. The result of the hallucination check.
    # @!attribute [r] tool_invocation_score
    #   @deprecated This field is deprecated and may be removed in the next major version update.
    #   @return [::Float]
    #     Output only. Deprecated. Use OverallToolInvocationResult instead.
    # @!attribute [r] tool_ordered_invocation_score
    #   @return [::Float]
    #     Output only. The overall tool ordered invocation score for this turn.
    #     This indicates the overall percent of tools from the expected turn that
    #     were actually invoked in the expected order.
    # @!attribute [r] turn_latency
    #   @return [::Google::Protobuf::Duration]
    #     Output only. Duration of the turn.
    # @!attribute [r] tool_call_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
    #     Output only. The latency of each tool call in the turn.
    # @!attribute [r] semantic_similarity_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
    #     Output only. The result of the semantic similarity check.
    # @!attribute [r] overall_tool_invocation_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::OverallToolInvocationResult]
    #     Output only. The result of the overall tool invocation check.
    # @!attribute [r] error_info
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationErrorInfo]
    #     Output only. Information about the error that occurred during this
    #     turn.
    # @!attribute [r] span_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
    #     Output only. The latency of spans in the turn.
    class TurnReplayResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of the evaluation against the rubric.
  # @!attribute [r] rubric
  #   @return [::String]
  #     Output only. The rubric that was used to evaluate the conversation.
  # @!attribute [r] score
  #   @return [::Float]
  #     Output only. The score of the conversation against the rubric.
  # @!attribute [r] score_explanation
  #   @return [::String]
  #     Output only. The rater's response to the rubric.
  class ScenarioRubricOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of a scenario expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome::ObservedToolCall]
  #     Output only. The observed tool call.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The observed agent response.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::ScenarioExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the ScenarioExpectation.
  class ScenarioExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The observed tool call and response.
    # @!attribute [r] tool_call
    #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
    #     Output only. The observed tool call.
    # @!attribute [r] tool_response
    #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
    #     Output only. The observed tool response.
    class ObservedToolCall
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of a scenario evaluation.
  # @!attribute [r] conversation
  #   @return [::String]
  #     Output only. The conversation that was generated in the scenario.
  # @!attribute [r] task
  #   @return [::String]
  #     Output only. The task that was used when running the scenario for this
  #     result.
  # @!attribute [r] user_facts
  #   @return [::Array<::Google::Cloud::Ces::V1beta::Evaluation::Scenario::UserFact>]
  #     Output only. The user facts that were used by the scenario for this
  #     result.
  # @!attribute [r] expectation_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome>]
  #     Output only. The outcome of each expectation.
  # @!attribute [r] rubric_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioRubricOutcome>]
  #     Output only. The outcome of the rubric.
  # @!attribute [r] hallucination_result
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult>]
  #     Output only. The result of the hallucination check.
  #     There will be one hallucination result for each turn in the conversation.
  # @!attribute [r] task_completion_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::TaskCompletionResult]
  #     Output only. The result of the task completion check.
  # @!attribute [r] tool_call_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
  #     Output only. The latency of each tool call execution in the conversation.
  # @!attribute [r] user_goal_satisfaction_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::UserGoalSatisfactionResult]
  #     Output only. The result of the user goal satisfaction check.
  # @!attribute [r] all_expectations_satisfied
  #   @return [::Boolean]
  #     Output only. Whether all expectations were satisfied for this turn.
  # @!attribute [r] task_completed
  #   @return [::Boolean]
  #     Output only. Whether the task was completed for this turn. This is a
  #     composite of all expectations satisfied, no hallucinations, and user goal
  #     satisfaction.
  # @!attribute [r] span_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
  #     Output only. The latency of spans in the conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class ScenarioResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The latency of a span execution.
  # @!attribute [r] resource
  #   @return [::String]
  #     Output only. The resource name of the guardrail or tool spans.
  #
  #     Note: The following fields are mutually exclusive: `resource`, `toolset`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] toolset
  #   @return [::Google::Cloud::Ces::V1beta::ToolsetTool]
  #     Output only. The toolset tool identifier.
  #
  #     Note: The following fields are mutually exclusive: `toolset`, `resource`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] model
  #   @return [::String]
  #     Output only. The name of the LLM span.
  #
  #     Note: The following fields are mutually exclusive: `model`, `resource`, `toolset`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] callback
  #   @return [::String]
  #     Output only. The name of the user callback span.
  #
  #     Note: The following fields are mutually exclusive: `callback`, `resource`, `toolset`, `model`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] type
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency::Type]
  #     Output only. The type of span.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the span. Applicable to tool and
  #     guardrail spans.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of span.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of span.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of span.
  class SpanLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The type of span.
    # Additional values may be added in the future.
    module Type
      # Default value. This value is unused.
      TYPE_UNSPECIFIED = 0

      # Tool call span.
      TOOL = 1

      # User callback span.
      USER_CALLBACK = 2

      # Guardrail span.
      GUARDRAIL = 3

      # LLM span.
      LLM = 4
    end
  end

  # The latency of a tool call execution.
  # @!attribute [r] tool
  #   @return [::String]
  #     Output only. The name of the tool that got executed.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/tools/{tool}`.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the tool.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of the tool call execution.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of the tool call execution.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of the tool call execution.
  class ToolCallLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the hallucination check for a single turn.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The hallucination score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Justified
  #     Score 0: Not Justified
  #     Score -1: No Claim To Assess
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the hallucination score.
  class HallucinationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a user goal satisfaction check for a conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The user task satisfaction score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: User Task Satisfied
  #     Score 0: User Task Not Satisfied
  #     Score -1: User Task Unspecified
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the user task satisfaction score.
  class UserGoalSatisfactionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the semantic similarity check.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The semantic similarity score. Can be 0, 1, 2, 3, or 4.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 4: Fully Consistent
  #     Score 3: Mostly Consistent
  #     Score 2: Partially Consistent (Minor Omissions)
  #     Score 1: Largely Inconsistent (Major Omissions)
  #     Score 0: Completely Inconsistent / Contradictory
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the semantic similarity score.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the semantic similarity check. This is
  #     determined by comparing the score to the
  #     semantic_similarity_success_threshold. If the score is equal to or above
  #     the threshold, the outcome will be PASS. Otherwise, the outcome will be
  #     FAIL.
  class SemanticSimilarityResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the overall tool invocation check.
  # @!attribute [rw] tool_invocation_score
  #   @return [::Float]
  #     The overall tool invocation score for this turn. This indicates the
  #     overall percent of tools from the expected turn that were actually
  #     invoked.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the tool invocation check. This is determined
  #     by comparing the tool_invocation_score to the
  #     overall_tool_invocation_correctness_threshold. If the
  #     score is equal to or above the threshold, the outcome will be PASS.
  #     Otherwise, the outcome will be FAIL.
  class OverallToolInvocationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the task completion check for the conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The task completion score. Can be -1, 0, 1
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Task Completed
  #     Score 0: Task Not Completed
  #     Score -1: User Goal Undefined
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the task completion score.
  class TaskCompletionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of the evaluation or expectation.
  module Outcome
    # Evaluation outcome is not specified.
    OUTCOME_UNSPECIFIED = 0

    # Evaluation/Expectation passed. In the case of an evaluation, this means
    # that all expectations were met.
    PASS = 1

    # Evaluation/Expectation failed. In the case of an evaluation, this means
    # that at least one expectation was not met.
    FAIL = 2

    # Evaluation/Expectation was skipped.
    SKIPPED = 3
  end

  # The state of the evaluation result execution.
  module ExecutionState
    # Evaluation result execution state is not specified.
    EXECUTION_STATE_UNSPECIFIED = 0

    # Evaluation result execution is running.
    RUNNING = 1

    # Evaluation result execution has completed.
    COMPLETED = 2

    # Evaluation result execution failed due to an internal error.
    ERROR = 3
  end
end

#golden_result ⇒ ::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenResult (readonly)

Returns Output only. The outcome of a golden evaluation.

Note: The following fields are mutually exclusive: golden_result, scenario_result. If a field in that set is populated, all other fields in the set will automatically be cleared.

Returns:



549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
 
# File 'proto_docs/google/cloud/ces/v1beta/evaluation.rb', line 549

class EvaluationResult
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # Specifies the expectation and the result of that expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
  #     Output only. The result of the tool call expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_tool_response`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_tool_response
  #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
  #     Output only. The result of the tool response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_response`, `observed_tool_call`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The result of the agent response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`, `observed_tool_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_transfer
  #   @return [::Google::Cloud::Ces::V1beta::AgentTransfer]
  #     Output only. The result of the agent transfer expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_transfer`, `observed_tool_call`, `observed_tool_response`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::GoldenExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the expectation.
  # @!attribute [r] semantic_similarity_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
  #     Output only. The result of the semantic similarity check.
  # @!attribute [r] tool_invocation_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome::ToolInvocationResult]
  #     Output only. The result of the tool invocation check.
  class GoldenExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of the tool invocation check.
    # @!attribute [r] parameter_correctness_score
    #   @return [::Float]
    #     Output only. The tool invocation parameter correctness score. This
    #     indicates the percent of parameters from the expected tool call that
    #     were also present in the actual tool call.
    # @!attribute [r] outcome
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
    #     Output only. The outcome of the tool invocation check. This is
    #     determined by comparing the parameter_correctness_score to the
    #     threshold. If the score is equal to or above the threshold, the outcome
    #     will be PASS. Otherwise, the outcome will be FAIL.
    # @!attribute [r] explanation
    #   @return [::String]
    #     Output only. A free text explanation for the tool invocation result.
    class ToolInvocationResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The result of a single evaluation expectation.
  # @!attribute [r] evaluation_expectation
  #   @return [::String]
  #     Output only. The evaluation expectation.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluation_expectation}`
  # @!attribute [r] prompt
  #   @return [::String]
  #     Output only. The prompt that was used for the evaluation.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the evaluation expectation.
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the result.
  class EvaluationExpectationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a golden evaluation.
  # @!attribute [r] turn_replay_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenResult::TurnReplayResult>]
  #     Output only. The result of running each turn of the golden conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class GoldenResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of running a single turn of the golden conversation.
    # @!attribute [r] conversation
    #   @return [::String]
    #     Output only. The conversation that was generated for this turn.
    # @!attribute [r] expectation_outcome
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome>]
    #     Output only. The outcome of each expectation.
    # @!attribute [r] hallucination_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult]
    #     Output only. The result of the hallucination check.
    # @!attribute [r] tool_invocation_score
    #   @deprecated This field is deprecated and may be removed in the next major version update.
    #   @return [::Float]
    #     Output only. Deprecated. Use OverallToolInvocationResult instead.
    # @!attribute [r] tool_ordered_invocation_score
    #   @return [::Float]
    #     Output only. The overall tool ordered invocation score for this turn.
    #     This indicates the overall percent of tools from the expected turn that
    #     were actually invoked in the expected order.
    # @!attribute [r] turn_latency
    #   @return [::Google::Protobuf::Duration]
    #     Output only. Duration of the turn.
    # @!attribute [r] tool_call_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
    #     Output only. The latency of each tool call in the turn.
    # @!attribute [r] semantic_similarity_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
    #     Output only. The result of the semantic similarity check.
    # @!attribute [r] overall_tool_invocation_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::OverallToolInvocationResult]
    #     Output only. The result of the overall tool invocation check.
    # @!attribute [r] error_info
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationErrorInfo]
    #     Output only. Information about the error that occurred during this
    #     turn.
    # @!attribute [r] span_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
    #     Output only. The latency of spans in the turn.
    class TurnReplayResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of the evaluation against the rubric.
  # @!attribute [r] rubric
  #   @return [::String]
  #     Output only. The rubric that was used to evaluate the conversation.
  # @!attribute [r] score
  #   @return [::Float]
  #     Output only. The score of the conversation against the rubric.
  # @!attribute [r] score_explanation
  #   @return [::String]
  #     Output only. The rater's response to the rubric.
  class ScenarioRubricOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of a scenario expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome::ObservedToolCall]
  #     Output only. The observed tool call.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The observed agent response.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::ScenarioExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the ScenarioExpectation.
  class ScenarioExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The observed tool call and response.
    # @!attribute [r] tool_call
    #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
    #     Output only. The observed tool call.
    # @!attribute [r] tool_response
    #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
    #     Output only. The observed tool response.
    class ObservedToolCall
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of a scenario evaluation.
  # @!attribute [r] conversation
  #   @return [::String]
  #     Output only. The conversation that was generated in the scenario.
  # @!attribute [r] task
  #   @return [::String]
  #     Output only. The task that was used when running the scenario for this
  #     result.
  # @!attribute [r] user_facts
  #   @return [::Array<::Google::Cloud::Ces::V1beta::Evaluation::Scenario::UserFact>]
  #     Output only. The user facts that were used by the scenario for this
  #     result.
  # @!attribute [r] expectation_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome>]
  #     Output only. The outcome of each expectation.
  # @!attribute [r] rubric_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioRubricOutcome>]
  #     Output only. The outcome of the rubric.
  # @!attribute [r] hallucination_result
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult>]
  #     Output only. The result of the hallucination check.
  #     There will be one hallucination result for each turn in the conversation.
  # @!attribute [r] task_completion_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::TaskCompletionResult]
  #     Output only. The result of the task completion check.
  # @!attribute [r] tool_call_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
  #     Output only. The latency of each tool call execution in the conversation.
  # @!attribute [r] user_goal_satisfaction_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::UserGoalSatisfactionResult]
  #     Output only. The result of the user goal satisfaction check.
  # @!attribute [r] all_expectations_satisfied
  #   @return [::Boolean]
  #     Output only. Whether all expectations were satisfied for this turn.
  # @!attribute [r] task_completed
  #   @return [::Boolean]
  #     Output only. Whether the task was completed for this turn. This is a
  #     composite of all expectations satisfied, no hallucinations, and user goal
  #     satisfaction.
  # @!attribute [r] span_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
  #     Output only. The latency of spans in the conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class ScenarioResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The latency of a span execution.
  # @!attribute [r] resource
  #   @return [::String]
  #     Output only. The resource name of the guardrail or tool spans.
  #
  #     Note: The following fields are mutually exclusive: `resource`, `toolset`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] toolset
  #   @return [::Google::Cloud::Ces::V1beta::ToolsetTool]
  #     Output only. The toolset tool identifier.
  #
  #     Note: The following fields are mutually exclusive: `toolset`, `resource`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] model
  #   @return [::String]
  #     Output only. The name of the LLM span.
  #
  #     Note: The following fields are mutually exclusive: `model`, `resource`, `toolset`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] callback
  #   @return [::String]
  #     Output only. The name of the user callback span.
  #
  #     Note: The following fields are mutually exclusive: `callback`, `resource`, `toolset`, `model`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] type
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency::Type]
  #     Output only. The type of span.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the span. Applicable to tool and
  #     guardrail spans.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of span.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of span.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of span.
  class SpanLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The type of span.
    # Additional values may be added in the future.
    module Type
      # Default value. This value is unused.
      TYPE_UNSPECIFIED = 0

      # Tool call span.
      TOOL = 1

      # User callback span.
      USER_CALLBACK = 2

      # Guardrail span.
      GUARDRAIL = 3

      # LLM span.
      LLM = 4
    end
  end

  # The latency of a tool call execution.
  # @!attribute [r] tool
  #   @return [::String]
  #     Output only. The name of the tool that got executed.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/tools/{tool}`.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the tool.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of the tool call execution.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of the tool call execution.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of the tool call execution.
  class ToolCallLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the hallucination check for a single turn.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The hallucination score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Justified
  #     Score 0: Not Justified
  #     Score -1: No Claim To Assess
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the hallucination score.
  class HallucinationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a user goal satisfaction check for a conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The user task satisfaction score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: User Task Satisfied
  #     Score 0: User Task Not Satisfied
  #     Score -1: User Task Unspecified
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the user task satisfaction score.
  class UserGoalSatisfactionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the semantic similarity check.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The semantic similarity score. Can be 0, 1, 2, 3, or 4.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 4: Fully Consistent
  #     Score 3: Mostly Consistent
  #     Score 2: Partially Consistent (Minor Omissions)
  #     Score 1: Largely Inconsistent (Major Omissions)
  #     Score 0: Completely Inconsistent / Contradictory
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the semantic similarity score.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the semantic similarity check. This is
  #     determined by comparing the score to the
  #     semantic_similarity_success_threshold. If the score is equal to or above
  #     the threshold, the outcome will be PASS. Otherwise, the outcome will be
  #     FAIL.
  class SemanticSimilarityResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the overall tool invocation check.
  # @!attribute [rw] tool_invocation_score
  #   @return [::Float]
  #     The overall tool invocation score for this turn. This indicates the
  #     overall percent of tools from the expected turn that were actually
  #     invoked.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the tool invocation check. This is determined
  #     by comparing the tool_invocation_score to the
  #     overall_tool_invocation_correctness_threshold. If the
  #     score is equal to or above the threshold, the outcome will be PASS.
  #     Otherwise, the outcome will be FAIL.
  class OverallToolInvocationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the task completion check for the conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The task completion score. Can be -1, 0, 1
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Task Completed
  #     Score 0: Task Not Completed
  #     Score -1: User Goal Undefined
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the task completion score.
  class TaskCompletionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of the evaluation or expectation.
  module Outcome
    # Evaluation outcome is not specified.
    OUTCOME_UNSPECIFIED = 0

    # Evaluation/Expectation passed. In the case of an evaluation, this means
    # that all expectations were met.
    PASS = 1

    # Evaluation/Expectation failed. In the case of an evaluation, this means
    # that at least one expectation was not met.
    FAIL = 2

    # Evaluation/Expectation was skipped.
    SKIPPED = 3
  end

  # The state of the evaluation result execution.
  module ExecutionState
    # Evaluation result execution state is not specified.
    EXECUTION_STATE_UNSPECIFIED = 0

    # Evaluation result execution is running.
    RUNNING = 1

    # Evaluation result execution has completed.
    COMPLETED = 2

    # Evaluation result execution failed due to an internal error.
    ERROR = 3
  end
end

#golden_run_method ⇒ ::Google::Cloud::Ces::V1beta::GoldenRunMethod (readonly)

Returns Output only. The method used to run the golden evaluation.

Returns:



549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
 
# File 'proto_docs/google/cloud/ces/v1beta/evaluation.rb', line 549

class EvaluationResult
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # Specifies the expectation and the result of that expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
  #     Output only. The result of the tool call expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_tool_response`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_tool_response
  #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
  #     Output only. The result of the tool response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_response`, `observed_tool_call`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The result of the agent response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`, `observed_tool_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_transfer
  #   @return [::Google::Cloud::Ces::V1beta::AgentTransfer]
  #     Output only. The result of the agent transfer expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_transfer`, `observed_tool_call`, `observed_tool_response`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::GoldenExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the expectation.
  # @!attribute [r] semantic_similarity_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
  #     Output only. The result of the semantic similarity check.
  # @!attribute [r] tool_invocation_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome::ToolInvocationResult]
  #     Output only. The result of the tool invocation check.
  class GoldenExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of the tool invocation check.
    # @!attribute [r] parameter_correctness_score
    #   @return [::Float]
    #     Output only. The tool invocation parameter correctness score. This
    #     indicates the percent of parameters from the expected tool call that
    #     were also present in the actual tool call.
    # @!attribute [r] outcome
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
    #     Output only. The outcome of the tool invocation check. This is
    #     determined by comparing the parameter_correctness_score to the
    #     threshold. If the score is equal to or above the threshold, the outcome
    #     will be PASS. Otherwise, the outcome will be FAIL.
    # @!attribute [r] explanation
    #   @return [::String]
    #     Output only. A free text explanation for the tool invocation result.
    class ToolInvocationResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The result of a single evaluation expectation.
  # @!attribute [r] evaluation_expectation
  #   @return [::String]
  #     Output only. The evaluation expectation.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluation_expectation}`
  # @!attribute [r] prompt
  #   @return [::String]
  #     Output only. The prompt that was used for the evaluation.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the evaluation expectation.
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the result.
  class EvaluationExpectationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a golden evaluation.
  # @!attribute [r] turn_replay_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenResult::TurnReplayResult>]
  #     Output only. The result of running each turn of the golden conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class GoldenResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of running a single turn of the golden conversation.
    # @!attribute [r] conversation
    #   @return [::String]
    #     Output only. The conversation that was generated for this turn.
    # @!attribute [r] expectation_outcome
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome>]
    #     Output only. The outcome of each expectation.
    # @!attribute [r] hallucination_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult]
    #     Output only. The result of the hallucination check.
    # @!attribute [r] tool_invocation_score
    #   @deprecated This field is deprecated and may be removed in the next major version update.
    #   @return [::Float]
    #     Output only. Deprecated. Use OverallToolInvocationResult instead.
    # @!attribute [r] tool_ordered_invocation_score
    #   @return [::Float]
    #     Output only. The overall tool ordered invocation score for this turn.
    #     This indicates the overall percent of tools from the expected turn that
    #     were actually invoked in the expected order.
    # @!attribute [r] turn_latency
    #   @return [::Google::Protobuf::Duration]
    #     Output only. Duration of the turn.
    # @!attribute [r] tool_call_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
    #     Output only. The latency of each tool call in the turn.
    # @!attribute [r] semantic_similarity_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
    #     Output only. The result of the semantic similarity check.
    # @!attribute [r] overall_tool_invocation_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::OverallToolInvocationResult]
    #     Output only. The result of the overall tool invocation check.
    # @!attribute [r] error_info
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationErrorInfo]
    #     Output only. Information about the error that occurred during this
    #     turn.
    # @!attribute [r] span_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
    #     Output only. The latency of spans in the turn.
    class TurnReplayResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of the evaluation against the rubric.
  # @!attribute [r] rubric
  #   @return [::String]
  #     Output only. The rubric that was used to evaluate the conversation.
  # @!attribute [r] score
  #   @return [::Float]
  #     Output only. The score of the conversation against the rubric.
  # @!attribute [r] score_explanation
  #   @return [::String]
  #     Output only. The rater's response to the rubric.
  class ScenarioRubricOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of a scenario expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome::ObservedToolCall]
  #     Output only. The observed tool call.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The observed agent response.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::ScenarioExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the ScenarioExpectation.
  class ScenarioExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The observed tool call and response.
    # @!attribute [r] tool_call
    #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
    #     Output only. The observed tool call.
    # @!attribute [r] tool_response
    #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
    #     Output only. The observed tool response.
    class ObservedToolCall
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of a scenario evaluation.
  # @!attribute [r] conversation
  #   @return [::String]
  #     Output only. The conversation that was generated in the scenario.
  # @!attribute [r] task
  #   @return [::String]
  #     Output only. The task that was used when running the scenario for this
  #     result.
  # @!attribute [r] user_facts
  #   @return [::Array<::Google::Cloud::Ces::V1beta::Evaluation::Scenario::UserFact>]
  #     Output only. The user facts that were used by the scenario for this
  #     result.
  # @!attribute [r] expectation_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome>]
  #     Output only. The outcome of each expectation.
  # @!attribute [r] rubric_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioRubricOutcome>]
  #     Output only. The outcome of the rubric.
  # @!attribute [r] hallucination_result
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult>]
  #     Output only. The result of the hallucination check.
  #     There will be one hallucination result for each turn in the conversation.
  # @!attribute [r] task_completion_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::TaskCompletionResult]
  #     Output only. The result of the task completion check.
  # @!attribute [r] tool_call_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
  #     Output only. The latency of each tool call execution in the conversation.
  # @!attribute [r] user_goal_satisfaction_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::UserGoalSatisfactionResult]
  #     Output only. The result of the user goal satisfaction check.
  # @!attribute [r] all_expectations_satisfied
  #   @return [::Boolean]
  #     Output only. Whether all expectations were satisfied for this turn.
  # @!attribute [r] task_completed
  #   @return [::Boolean]
  #     Output only. Whether the task was completed for this turn. This is a
  #     composite of all expectations satisfied, no hallucinations, and user goal
  #     satisfaction.
  # @!attribute [r] span_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
  #     Output only. The latency of spans in the conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class ScenarioResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The latency of a span execution.
  # @!attribute [r] resource
  #   @return [::String]
  #     Output only. The resource name of the guardrail or tool spans.
  #
  #     Note: The following fields are mutually exclusive: `resource`, `toolset`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] toolset
  #   @return [::Google::Cloud::Ces::V1beta::ToolsetTool]
  #     Output only. The toolset tool identifier.
  #
  #     Note: The following fields are mutually exclusive: `toolset`, `resource`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] model
  #   @return [::String]
  #     Output only. The name of the LLM span.
  #
  #     Note: The following fields are mutually exclusive: `model`, `resource`, `toolset`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] callback
  #   @return [::String]
  #     Output only. The name of the user callback span.
  #
  #     Note: The following fields are mutually exclusive: `callback`, `resource`, `toolset`, `model`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] type
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency::Type]
  #     Output only. The type of span.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the span. Applicable to tool and
  #     guardrail spans.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of span.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of span.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of span.
  class SpanLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The type of span.
    # Additional values may be added in the future.
    module Type
      # Default value. This value is unused.
      TYPE_UNSPECIFIED = 0

      # Tool call span.
      TOOL = 1

      # User callback span.
      USER_CALLBACK = 2

      # Guardrail span.
      GUARDRAIL = 3

      # LLM span.
      LLM = 4
    end
  end

  # The latency of a tool call execution.
  # @!attribute [r] tool
  #   @return [::String]
  #     Output only. The name of the tool that got executed.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/tools/{tool}`.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the tool.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of the tool call execution.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of the tool call execution.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of the tool call execution.
  class ToolCallLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the hallucination check for a single turn.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The hallucination score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Justified
  #     Score 0: Not Justified
  #     Score -1: No Claim To Assess
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the hallucination score.
  class HallucinationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a user goal satisfaction check for a conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The user task satisfaction score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: User Task Satisfied
  #     Score 0: User Task Not Satisfied
  #     Score -1: User Task Unspecified
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the user task satisfaction score.
  class UserGoalSatisfactionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the semantic similarity check.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The semantic similarity score. Can be 0, 1, 2, 3, or 4.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 4: Fully Consistent
  #     Score 3: Mostly Consistent
  #     Score 2: Partially Consistent (Minor Omissions)
  #     Score 1: Largely Inconsistent (Major Omissions)
  #     Score 0: Completely Inconsistent / Contradictory
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the semantic similarity score.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the semantic similarity check. This is
  #     determined by comparing the score to the
  #     semantic_similarity_success_threshold. If the score is equal to or above
  #     the threshold, the outcome will be PASS. Otherwise, the outcome will be
  #     FAIL.
  class SemanticSimilarityResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the overall tool invocation check.
  # @!attribute [rw] tool_invocation_score
  #   @return [::Float]
  #     The overall tool invocation score for this turn. This indicates the
  #     overall percent of tools from the expected turn that were actually
  #     invoked.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the tool invocation check. This is determined
  #     by comparing the tool_invocation_score to the
  #     overall_tool_invocation_correctness_threshold. If the
  #     score is equal to or above the threshold, the outcome will be PASS.
  #     Otherwise, the outcome will be FAIL.
  class OverallToolInvocationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the task completion check for the conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The task completion score. Can be -1, 0, 1
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Task Completed
  #     Score 0: Task Not Completed
  #     Score -1: User Goal Undefined
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the task completion score.
  class TaskCompletionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of the evaluation or expectation.
  module Outcome
    # Evaluation outcome is not specified.
    OUTCOME_UNSPECIFIED = 0

    # Evaluation/Expectation passed. In the case of an evaluation, this means
    # that all expectations were met.
    PASS = 1

    # Evaluation/Expectation failed. In the case of an evaluation, this means
    # that at least one expectation was not met.
    FAIL = 2

    # Evaluation/Expectation was skipped.
    SKIPPED = 3
  end

  # The state of the evaluation result execution.
  module ExecutionState
    # Evaluation result execution state is not specified.
    EXECUTION_STATE_UNSPECIFIED = 0

    # Evaluation result execution is running.
    RUNNING = 1

    # Evaluation result execution has completed.
    COMPLETED = 2

    # Evaluation result execution failed due to an internal error.
    ERROR = 3
  end
end

#initiated_by ⇒ ::String (readonly)

Returns Output only. The user who initiated the evaluation run that resulted in this result.

Returns:

  • (::String) —

    Output only. The user who initiated the evaluation run that resulted in this result.



549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
 
# File 'proto_docs/google/cloud/ces/v1beta/evaluation.rb', line 549

class EvaluationResult
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # Specifies the expectation and the result of that expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
  #     Output only. The result of the tool call expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_tool_response`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_tool_response
  #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
  #     Output only. The result of the tool response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_response`, `observed_tool_call`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The result of the agent response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`, `observed_tool_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_transfer
  #   @return [::Google::Cloud::Ces::V1beta::AgentTransfer]
  #     Output only. The result of the agent transfer expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_transfer`, `observed_tool_call`, `observed_tool_response`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::GoldenExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the expectation.
  # @!attribute [r] semantic_similarity_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
  #     Output only. The result of the semantic similarity check.
  # @!attribute [r] tool_invocation_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome::ToolInvocationResult]
  #     Output only. The result of the tool invocation check.
  class GoldenExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of the tool invocation check.
    # @!attribute [r] parameter_correctness_score
    #   @return [::Float]
    #     Output only. The tool invocation parameter correctness score. This
    #     indicates the percent of parameters from the expected tool call that
    #     were also present in the actual tool call.
    # @!attribute [r] outcome
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
    #     Output only. The outcome of the tool invocation check. This is
    #     determined by comparing the parameter_correctness_score to the
    #     threshold. If the score is equal to or above the threshold, the outcome
    #     will be PASS. Otherwise, the outcome will be FAIL.
    # @!attribute [r] explanation
    #   @return [::String]
    #     Output only. A free text explanation for the tool invocation result.
    class ToolInvocationResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The result of a single evaluation expectation.
  # @!attribute [r] evaluation_expectation
  #   @return [::String]
  #     Output only. The evaluation expectation.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluation_expectation}`
  # @!attribute [r] prompt
  #   @return [::String]
  #     Output only. The prompt that was used for the evaluation.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the evaluation expectation.
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the result.
  class EvaluationExpectationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a golden evaluation.
  # @!attribute [r] turn_replay_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenResult::TurnReplayResult>]
  #     Output only. The result of running each turn of the golden conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class GoldenResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of running a single turn of the golden conversation.
    # @!attribute [r] conversation
    #   @return [::String]
    #     Output only. The conversation that was generated for this turn.
    # @!attribute [r] expectation_outcome
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome>]
    #     Output only. The outcome of each expectation.
    # @!attribute [r] hallucination_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult]
    #     Output only. The result of the hallucination check.
    # @!attribute [r] tool_invocation_score
    #   @deprecated This field is deprecated and may be removed in the next major version update.
    #   @return [::Float]
    #     Output only. Deprecated. Use OverallToolInvocationResult instead.
    # @!attribute [r] tool_ordered_invocation_score
    #   @return [::Float]
    #     Output only. The overall tool ordered invocation score for this turn.
    #     This indicates the overall percent of tools from the expected turn that
    #     were actually invoked in the expected order.
    # @!attribute [r] turn_latency
    #   @return [::Google::Protobuf::Duration]
    #     Output only. Duration of the turn.
    # @!attribute [r] tool_call_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
    #     Output only. The latency of each tool call in the turn.
    # @!attribute [r] semantic_similarity_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
    #     Output only. The result of the semantic similarity check.
    # @!attribute [r] overall_tool_invocation_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::OverallToolInvocationResult]
    #     Output only. The result of the overall tool invocation check.
    # @!attribute [r] error_info
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationErrorInfo]
    #     Output only. Information about the error that occurred during this
    #     turn.
    # @!attribute [r] span_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
    #     Output only. The latency of spans in the turn.
    class TurnReplayResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of the evaluation against the rubric.
  # @!attribute [r] rubric
  #   @return [::String]
  #     Output only. The rubric that was used to evaluate the conversation.
  # @!attribute [r] score
  #   @return [::Float]
  #     Output only. The score of the conversation against the rubric.
  # @!attribute [r] score_explanation
  #   @return [::String]
  #     Output only. The rater's response to the rubric.
  class ScenarioRubricOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of a scenario expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome::ObservedToolCall]
  #     Output only. The observed tool call.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The observed agent response.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::ScenarioExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the ScenarioExpectation.
  class ScenarioExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The observed tool call and response.
    # @!attribute [r] tool_call
    #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
    #     Output only. The observed tool call.
    # @!attribute [r] tool_response
    #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
    #     Output only. The observed tool response.
    class ObservedToolCall
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of a scenario evaluation.
  # @!attribute [r] conversation
  #   @return [::String]
  #     Output only. The conversation that was generated in the scenario.
  # @!attribute [r] task
  #   @return [::String]
  #     Output only. The task that was used when running the scenario for this
  #     result.
  # @!attribute [r] user_facts
  #   @return [::Array<::Google::Cloud::Ces::V1beta::Evaluation::Scenario::UserFact>]
  #     Output only. The user facts that were used by the scenario for this
  #     result.
  # @!attribute [r] expectation_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome>]
  #     Output only. The outcome of each expectation.
  # @!attribute [r] rubric_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioRubricOutcome>]
  #     Output only. The outcome of the rubric.
  # @!attribute [r] hallucination_result
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult>]
  #     Output only. The result of the hallucination check.
  #     There will be one hallucination result for each turn in the conversation.
  # @!attribute [r] task_completion_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::TaskCompletionResult]
  #     Output only. The result of the task completion check.
  # @!attribute [r] tool_call_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
  #     Output only. The latency of each tool call execution in the conversation.
  # @!attribute [r] user_goal_satisfaction_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::UserGoalSatisfactionResult]
  #     Output only. The result of the user goal satisfaction check.
  # @!attribute [r] all_expectations_satisfied
  #   @return [::Boolean]
  #     Output only. Whether all expectations were satisfied for this turn.
  # @!attribute [r] task_completed
  #   @return [::Boolean]
  #     Output only. Whether the task was completed for this turn. This is a
  #     composite of all expectations satisfied, no hallucinations, and user goal
  #     satisfaction.
  # @!attribute [r] span_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
  #     Output only. The latency of spans in the conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class ScenarioResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The latency of a span execution.
  # @!attribute [r] resource
  #   @return [::String]
  #     Output only. The resource name of the guardrail or tool spans.
  #
  #     Note: The following fields are mutually exclusive: `resource`, `toolset`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] toolset
  #   @return [::Google::Cloud::Ces::V1beta::ToolsetTool]
  #     Output only. The toolset tool identifier.
  #
  #     Note: The following fields are mutually exclusive: `toolset`, `resource`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] model
  #   @return [::String]
  #     Output only. The name of the LLM span.
  #
  #     Note: The following fields are mutually exclusive: `model`, `resource`, `toolset`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] callback
  #   @return [::String]
  #     Output only. The name of the user callback span.
  #
  #     Note: The following fields are mutually exclusive: `callback`, `resource`, `toolset`, `model`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] type
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency::Type]
  #     Output only. The type of span.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the span. Applicable to tool and
  #     guardrail spans.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of span.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of span.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of span.
  class SpanLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The type of span.
    # Additional values may be added in the future.
    module Type
      # Default value. This value is unused.
      TYPE_UNSPECIFIED = 0

      # Tool call span.
      TOOL = 1

      # User callback span.
      USER_CALLBACK = 2

      # Guardrail span.
      GUARDRAIL = 3

      # LLM span.
      LLM = 4
    end
  end

  # The latency of a tool call execution.
  # @!attribute [r] tool
  #   @return [::String]
  #     Output only. The name of the tool that got executed.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/tools/{tool}`.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the tool.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of the tool call execution.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of the tool call execution.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of the tool call execution.
  class ToolCallLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the hallucination check for a single turn.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The hallucination score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Justified
  #     Score 0: Not Justified
  #     Score -1: No Claim To Assess
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the hallucination score.
  class HallucinationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a user goal satisfaction check for a conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The user task satisfaction score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: User Task Satisfied
  #     Score 0: User Task Not Satisfied
  #     Score -1: User Task Unspecified
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the user task satisfaction score.
  class UserGoalSatisfactionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the semantic similarity check.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The semantic similarity score. Can be 0, 1, 2, 3, or 4.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 4: Fully Consistent
  #     Score 3: Mostly Consistent
  #     Score 2: Partially Consistent (Minor Omissions)
  #     Score 1: Largely Inconsistent (Major Omissions)
  #     Score 0: Completely Inconsistent / Contradictory
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the semantic similarity score.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the semantic similarity check. This is
  #     determined by comparing the score to the
  #     semantic_similarity_success_threshold. If the score is equal to or above
  #     the threshold, the outcome will be PASS. Otherwise, the outcome will be
  #     FAIL.
  class SemanticSimilarityResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the overall tool invocation check.
  # @!attribute [rw] tool_invocation_score
  #   @return [::Float]
  #     The overall tool invocation score for this turn. This indicates the
  #     overall percent of tools from the expected turn that were actually
  #     invoked.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the tool invocation check. This is determined
  #     by comparing the tool_invocation_score to the
  #     overall_tool_invocation_correctness_threshold. If the
  #     score is equal to or above the threshold, the outcome will be PASS.
  #     Otherwise, the outcome will be FAIL.
  class OverallToolInvocationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the task completion check for the conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The task completion score. Can be -1, 0, 1
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Task Completed
  #     Score 0: Task Not Completed
  #     Score -1: User Goal Undefined
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the task completion score.
  class TaskCompletionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of the evaluation or expectation.
  module Outcome
    # Evaluation outcome is not specified.
    OUTCOME_UNSPECIFIED = 0

    # Evaluation/Expectation passed. In the case of an evaluation, this means
    # that all expectations were met.
    PASS = 1

    # Evaluation/Expectation failed. In the case of an evaluation, this means
    # that at least one expectation was not met.
    FAIL = 2

    # Evaluation/Expectation was skipped.
    SKIPPED = 3
  end

  # The state of the evaluation result execution.
  module ExecutionState
    # Evaluation result execution state is not specified.
    EXECUTION_STATE_UNSPECIFIED = 0

    # Evaluation result execution is running.
    RUNNING = 1

    # Evaluation result execution has completed.
    COMPLETED = 2

    # Evaluation result execution failed due to an internal error.
    ERROR = 3
  end
end

#name ⇒ ::String

Returns Identifier. The unique identifier of the evaluation result. Format: projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation}/results/{result}.

Returns:

  • (::String) —

    Identifier. The unique identifier of the evaluation result. Format: projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation}/results/{result}



549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
 
# File 'proto_docs/google/cloud/ces/v1beta/evaluation.rb', line 549

class EvaluationResult
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # Specifies the expectation and the result of that expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
  #     Output only. The result of the tool call expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_tool_response`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_tool_response
  #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
  #     Output only. The result of the tool response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_response`, `observed_tool_call`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The result of the agent response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`, `observed_tool_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_transfer
  #   @return [::Google::Cloud::Ces::V1beta::AgentTransfer]
  #     Output only. The result of the agent transfer expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_transfer`, `observed_tool_call`, `observed_tool_response`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::GoldenExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the expectation.
  # @!attribute [r] semantic_similarity_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
  #     Output only. The result of the semantic similarity check.
  # @!attribute [r] tool_invocation_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome::ToolInvocationResult]
  #     Output only. The result of the tool invocation check.
  class GoldenExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of the tool invocation check.
    # @!attribute [r] parameter_correctness_score
    #   @return [::Float]
    #     Output only. The tool invocation parameter correctness score. This
    #     indicates the percent of parameters from the expected tool call that
    #     were also present in the actual tool call.
    # @!attribute [r] outcome
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
    #     Output only. The outcome of the tool invocation check. This is
    #     determined by comparing the parameter_correctness_score to the
    #     threshold. If the score is equal to or above the threshold, the outcome
    #     will be PASS. Otherwise, the outcome will be FAIL.
    # @!attribute [r] explanation
    #   @return [::String]
    #     Output only. A free text explanation for the tool invocation result.
    class ToolInvocationResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The result of a single evaluation expectation.
  # @!attribute [r] evaluation_expectation
  #   @return [::String]
  #     Output only. The evaluation expectation.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluation_expectation}`
  # @!attribute [r] prompt
  #   @return [::String]
  #     Output only. The prompt that was used for the evaluation.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the evaluation expectation.
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the result.
  class EvaluationExpectationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a golden evaluation.
  # @!attribute [r] turn_replay_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenResult::TurnReplayResult>]
  #     Output only. The result of running each turn of the golden conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class GoldenResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of running a single turn of the golden conversation.
    # @!attribute [r] conversation
    #   @return [::String]
    #     Output only. The conversation that was generated for this turn.
    # @!attribute [r] expectation_outcome
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome>]
    #     Output only. The outcome of each expectation.
    # @!attribute [r] hallucination_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult]
    #     Output only. The result of the hallucination check.
    # @!attribute [r] tool_invocation_score
    #   @deprecated This field is deprecated and may be removed in the next major version update.
    #   @return [::Float]
    #     Output only. Deprecated. Use OverallToolInvocationResult instead.
    # @!attribute [r] tool_ordered_invocation_score
    #   @return [::Float]
    #     Output only. The overall tool ordered invocation score for this turn.
    #     This indicates the overall percent of tools from the expected turn that
    #     were actually invoked in the expected order.
    # @!attribute [r] turn_latency
    #   @return [::Google::Protobuf::Duration]
    #     Output only. Duration of the turn.
    # @!attribute [r] tool_call_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
    #     Output only. The latency of each tool call in the turn.
    # @!attribute [r] semantic_similarity_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
    #     Output only. The result of the semantic similarity check.
    # @!attribute [r] overall_tool_invocation_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::OverallToolInvocationResult]
    #     Output only. The result of the overall tool invocation check.
    # @!attribute [r] error_info
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationErrorInfo]
    #     Output only. Information about the error that occurred during this
    #     turn.
    # @!attribute [r] span_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
    #     Output only. The latency of spans in the turn.
    class TurnReplayResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of the evaluation against the rubric.
  # @!attribute [r] rubric
  #   @return [::String]
  #     Output only. The rubric that was used to evaluate the conversation.
  # @!attribute [r] score
  #   @return [::Float]
  #     Output only. The score of the conversation against the rubric.
  # @!attribute [r] score_explanation
  #   @return [::String]
  #     Output only. The rater's response to the rubric.
  class ScenarioRubricOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of a scenario expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome::ObservedToolCall]
  #     Output only. The observed tool call.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The observed agent response.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::ScenarioExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the ScenarioExpectation.
  class ScenarioExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The observed tool call and response.
    # @!attribute [r] tool_call
    #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
    #     Output only. The observed tool call.
    # @!attribute [r] tool_response
    #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
    #     Output only. The observed tool response.
    class ObservedToolCall
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of a scenario evaluation.
  # @!attribute [r] conversation
  #   @return [::String]
  #     Output only. The conversation that was generated in the scenario.
  # @!attribute [r] task
  #   @return [::String]
  #     Output only. The task that was used when running the scenario for this
  #     result.
  # @!attribute [r] user_facts
  #   @return [::Array<::Google::Cloud::Ces::V1beta::Evaluation::Scenario::UserFact>]
  #     Output only. The user facts that were used by the scenario for this
  #     result.
  # @!attribute [r] expectation_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome>]
  #     Output only. The outcome of each expectation.
  # @!attribute [r] rubric_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioRubricOutcome>]
  #     Output only. The outcome of the rubric.
  # @!attribute [r] hallucination_result
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult>]
  #     Output only. The result of the hallucination check.
  #     There will be one hallucination result for each turn in the conversation.
  # @!attribute [r] task_completion_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::TaskCompletionResult]
  #     Output only. The result of the task completion check.
  # @!attribute [r] tool_call_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
  #     Output only. The latency of each tool call execution in the conversation.
  # @!attribute [r] user_goal_satisfaction_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::UserGoalSatisfactionResult]
  #     Output only. The result of the user goal satisfaction check.
  # @!attribute [r] all_expectations_satisfied
  #   @return [::Boolean]
  #     Output only. Whether all expectations were satisfied for this turn.
  # @!attribute [r] task_completed
  #   @return [::Boolean]
  #     Output only. Whether the task was completed for this turn. This is a
  #     composite of all expectations satisfied, no hallucinations, and user goal
  #     satisfaction.
  # @!attribute [r] span_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
  #     Output only. The latency of spans in the conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class ScenarioResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The latency of a span execution.
  # @!attribute [r] resource
  #   @return [::String]
  #     Output only. The resource name of the guardrail or tool spans.
  #
  #     Note: The following fields are mutually exclusive: `resource`, `toolset`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] toolset
  #   @return [::Google::Cloud::Ces::V1beta::ToolsetTool]
  #     Output only. The toolset tool identifier.
  #
  #     Note: The following fields are mutually exclusive: `toolset`, `resource`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] model
  #   @return [::String]
  #     Output only. The name of the LLM span.
  #
  #     Note: The following fields are mutually exclusive: `model`, `resource`, `toolset`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] callback
  #   @return [::String]
  #     Output only. The name of the user callback span.
  #
  #     Note: The following fields are mutually exclusive: `callback`, `resource`, `toolset`, `model`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] type
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency::Type]
  #     Output only. The type of span.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the span. Applicable to tool and
  #     guardrail spans.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of span.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of span.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of span.
  class SpanLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The type of span.
    # Additional values may be added in the future.
    module Type
      # Default value. This value is unused.
      TYPE_UNSPECIFIED = 0

      # Tool call span.
      TOOL = 1

      # User callback span.
      USER_CALLBACK = 2

      # Guardrail span.
      GUARDRAIL = 3

      # LLM span.
      LLM = 4
    end
  end

  # The latency of a tool call execution.
  # @!attribute [r] tool
  #   @return [::String]
  #     Output only. The name of the tool that got executed.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/tools/{tool}`.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the tool.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of the tool call execution.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of the tool call execution.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of the tool call execution.
  class ToolCallLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the hallucination check for a single turn.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The hallucination score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Justified
  #     Score 0: Not Justified
  #     Score -1: No Claim To Assess
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the hallucination score.
  class HallucinationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a user goal satisfaction check for a conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The user task satisfaction score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: User Task Satisfied
  #     Score 0: User Task Not Satisfied
  #     Score -1: User Task Unspecified
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the user task satisfaction score.
  class UserGoalSatisfactionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the semantic similarity check.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The semantic similarity score. Can be 0, 1, 2, 3, or 4.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 4: Fully Consistent
  #     Score 3: Mostly Consistent
  #     Score 2: Partially Consistent (Minor Omissions)
  #     Score 1: Largely Inconsistent (Major Omissions)
  #     Score 0: Completely Inconsistent / Contradictory
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the semantic similarity score.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the semantic similarity check. This is
  #     determined by comparing the score to the
  #     semantic_similarity_success_threshold. If the score is equal to or above
  #     the threshold, the outcome will be PASS. Otherwise, the outcome will be
  #     FAIL.
  class SemanticSimilarityResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the overall tool invocation check.
  # @!attribute [rw] tool_invocation_score
  #   @return [::Float]
  #     The overall tool invocation score for this turn. This indicates the
  #     overall percent of tools from the expected turn that were actually
  #     invoked.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the tool invocation check. This is determined
  #     by comparing the tool_invocation_score to the
  #     overall_tool_invocation_correctness_threshold. If the
  #     score is equal to or above the threshold, the outcome will be PASS.
  #     Otherwise, the outcome will be FAIL.
  class OverallToolInvocationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the task completion check for the conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The task completion score. Can be -1, 0, 1
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Task Completed
  #     Score 0: Task Not Completed
  #     Score -1: User Goal Undefined
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the task completion score.
  class TaskCompletionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of the evaluation or expectation.
  module Outcome
    # Evaluation outcome is not specified.
    OUTCOME_UNSPECIFIED = 0

    # Evaluation/Expectation passed. In the case of an evaluation, this means
    # that all expectations were met.
    PASS = 1

    # Evaluation/Expectation failed. In the case of an evaluation, this means
    # that at least one expectation was not met.
    FAIL = 2

    # Evaluation/Expectation was skipped.
    SKIPPED = 3
  end

  # The state of the evaluation result execution.
  module ExecutionState
    # Evaluation result execution state is not specified.
    EXECUTION_STATE_UNSPECIFIED = 0

    # Evaluation result execution is running.
    RUNNING = 1

    # Evaluation result execution has completed.
    COMPLETED = 2

    # Evaluation result execution failed due to an internal error.
    ERROR = 3
  end
end

#persona ⇒ ::Google::Cloud::Ces::V1beta::EvaluationPersona (readonly)

Returns Output only. The persona used to generate the conversation for the evaluation result.

Returns:



549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
 
# File 'proto_docs/google/cloud/ces/v1beta/evaluation.rb', line 549

class EvaluationResult
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # Specifies the expectation and the result of that expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
  #     Output only. The result of the tool call expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_tool_response`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_tool_response
  #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
  #     Output only. The result of the tool response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_response`, `observed_tool_call`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The result of the agent response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`, `observed_tool_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_transfer
  #   @return [::Google::Cloud::Ces::V1beta::AgentTransfer]
  #     Output only. The result of the agent transfer expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_transfer`, `observed_tool_call`, `observed_tool_response`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::GoldenExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the expectation.
  # @!attribute [r] semantic_similarity_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
  #     Output only. The result of the semantic similarity check.
  # @!attribute [r] tool_invocation_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome::ToolInvocationResult]
  #     Output only. The result of the tool invocation check.
  class GoldenExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of the tool invocation check.
    # @!attribute [r] parameter_correctness_score
    #   @return [::Float]
    #     Output only. The tool invocation parameter correctness score. This
    #     indicates the percent of parameters from the expected tool call that
    #     were also present in the actual tool call.
    # @!attribute [r] outcome
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
    #     Output only. The outcome of the tool invocation check. This is
    #     determined by comparing the parameter_correctness_score to the
    #     threshold. If the score is equal to or above the threshold, the outcome
    #     will be PASS. Otherwise, the outcome will be FAIL.
    # @!attribute [r] explanation
    #   @return [::String]
    #     Output only. A free text explanation for the tool invocation result.
    class ToolInvocationResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The result of a single evaluation expectation.
  # @!attribute [r] evaluation_expectation
  #   @return [::String]
  #     Output only. The evaluation expectation.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluation_expectation}`
  # @!attribute [r] prompt
  #   @return [::String]
  #     Output only. The prompt that was used for the evaluation.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the evaluation expectation.
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the result.
  class EvaluationExpectationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a golden evaluation.
  # @!attribute [r] turn_replay_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenResult::TurnReplayResult>]
  #     Output only. The result of running each turn of the golden conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class GoldenResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of running a single turn of the golden conversation.
    # @!attribute [r] conversation
    #   @return [::String]
    #     Output only. The conversation that was generated for this turn.
    # @!attribute [r] expectation_outcome
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome>]
    #     Output only. The outcome of each expectation.
    # @!attribute [r] hallucination_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult]
    #     Output only. The result of the hallucination check.
    # @!attribute [r] tool_invocation_score
    #   @deprecated This field is deprecated and may be removed in the next major version update.
    #   @return [::Float]
    #     Output only. Deprecated. Use OverallToolInvocationResult instead.
    # @!attribute [r] tool_ordered_invocation_score
    #   @return [::Float]
    #     Output only. The overall tool ordered invocation score for this turn.
    #     This indicates the overall percent of tools from the expected turn that
    #     were actually invoked in the expected order.
    # @!attribute [r] turn_latency
    #   @return [::Google::Protobuf::Duration]
    #     Output only. Duration of the turn.
    # @!attribute [r] tool_call_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
    #     Output only. The latency of each tool call in the turn.
    # @!attribute [r] semantic_similarity_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
    #     Output only. The result of the semantic similarity check.
    # @!attribute [r] overall_tool_invocation_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::OverallToolInvocationResult]
    #     Output only. The result of the overall tool invocation check.
    # @!attribute [r] error_info
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationErrorInfo]
    #     Output only. Information about the error that occurred during this
    #     turn.
    # @!attribute [r] span_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
    #     Output only. The latency of spans in the turn.
    class TurnReplayResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of the evaluation against the rubric.
  # @!attribute [r] rubric
  #   @return [::String]
  #     Output only. The rubric that was used to evaluate the conversation.
  # @!attribute [r] score
  #   @return [::Float]
  #     Output only. The score of the conversation against the rubric.
  # @!attribute [r] score_explanation
  #   @return [::String]
  #     Output only. The rater's response to the rubric.
  class ScenarioRubricOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of a scenario expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome::ObservedToolCall]
  #     Output only. The observed tool call.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The observed agent response.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::ScenarioExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the ScenarioExpectation.
  class ScenarioExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The observed tool call and response.
    # @!attribute [r] tool_call
    #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
    #     Output only. The observed tool call.
    # @!attribute [r] tool_response
    #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
    #     Output only. The observed tool response.
    class ObservedToolCall
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of a scenario evaluation.
  # @!attribute [r] conversation
  #   @return [::String]
  #     Output only. The conversation that was generated in the scenario.
  # @!attribute [r] task
  #   @return [::String]
  #     Output only. The task that was used when running the scenario for this
  #     result.
  # @!attribute [r] user_facts
  #   @return [::Array<::Google::Cloud::Ces::V1beta::Evaluation::Scenario::UserFact>]
  #     Output only. The user facts that were used by the scenario for this
  #     result.
  # @!attribute [r] expectation_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome>]
  #     Output only. The outcome of each expectation.
  # @!attribute [r] rubric_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioRubricOutcome>]
  #     Output only. The outcome of the rubric.
  # @!attribute [r] hallucination_result
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult>]
  #     Output only. The result of the hallucination check.
  #     There will be one hallucination result for each turn in the conversation.
  # @!attribute [r] task_completion_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::TaskCompletionResult]
  #     Output only. The result of the task completion check.
  # @!attribute [r] tool_call_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
  #     Output only. The latency of each tool call execution in the conversation.
  # @!attribute [r] user_goal_satisfaction_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::UserGoalSatisfactionResult]
  #     Output only. The result of the user goal satisfaction check.
  # @!attribute [r] all_expectations_satisfied
  #   @return [::Boolean]
  #     Output only. Whether all expectations were satisfied for this turn.
  # @!attribute [r] task_completed
  #   @return [::Boolean]
  #     Output only. Whether the task was completed for this turn. This is a
  #     composite of all expectations satisfied, no hallucinations, and user goal
  #     satisfaction.
  # @!attribute [r] span_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
  #     Output only. The latency of spans in the conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class ScenarioResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The latency of a span execution.
  # @!attribute [r] resource
  #   @return [::String]
  #     Output only. The resource name of the guardrail or tool spans.
  #
  #     Note: The following fields are mutually exclusive: `resource`, `toolset`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] toolset
  #   @return [::Google::Cloud::Ces::V1beta::ToolsetTool]
  #     Output only. The toolset tool identifier.
  #
  #     Note: The following fields are mutually exclusive: `toolset`, `resource`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] model
  #   @return [::String]
  #     Output only. The name of the LLM span.
  #
  #     Note: The following fields are mutually exclusive: `model`, `resource`, `toolset`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] callback
  #   @return [::String]
  #     Output only. The name of the user callback span.
  #
  #     Note: The following fields are mutually exclusive: `callback`, `resource`, `toolset`, `model`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] type
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency::Type]
  #     Output only. The type of span.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the span. Applicable to tool and
  #     guardrail spans.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of span.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of span.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of span.
  class SpanLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The type of span.
    # Additional values may be added in the future.
    module Type
      # Default value. This value is unused.
      TYPE_UNSPECIFIED = 0

      # Tool call span.
      TOOL = 1

      # User callback span.
      USER_CALLBACK = 2

      # Guardrail span.
      GUARDRAIL = 3

      # LLM span.
      LLM = 4
    end
  end

  # The latency of a tool call execution.
  # @!attribute [r] tool
  #   @return [::String]
  #     Output only. The name of the tool that got executed.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/tools/{tool}`.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the tool.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of the tool call execution.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of the tool call execution.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of the tool call execution.
  class ToolCallLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the hallucination check for a single turn.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The hallucination score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Justified
  #     Score 0: Not Justified
  #     Score -1: No Claim To Assess
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the hallucination score.
  class HallucinationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a user goal satisfaction check for a conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The user task satisfaction score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: User Task Satisfied
  #     Score 0: User Task Not Satisfied
  #     Score -1: User Task Unspecified
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the user task satisfaction score.
  class UserGoalSatisfactionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the semantic similarity check.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The semantic similarity score. Can be 0, 1, 2, 3, or 4.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 4: Fully Consistent
  #     Score 3: Mostly Consistent
  #     Score 2: Partially Consistent (Minor Omissions)
  #     Score 1: Largely Inconsistent (Major Omissions)
  #     Score 0: Completely Inconsistent / Contradictory
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the semantic similarity score.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the semantic similarity check. This is
  #     determined by comparing the score to the
  #     semantic_similarity_success_threshold. If the score is equal to or above
  #     the threshold, the outcome will be PASS. Otherwise, the outcome will be
  #     FAIL.
  class SemanticSimilarityResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the overall tool invocation check.
  # @!attribute [rw] tool_invocation_score
  #   @return [::Float]
  #     The overall tool invocation score for this turn. This indicates the
  #     overall percent of tools from the expected turn that were actually
  #     invoked.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the tool invocation check. This is determined
  #     by comparing the tool_invocation_score to the
  #     overall_tool_invocation_correctness_threshold. If the
  #     score is equal to or above the threshold, the outcome will be PASS.
  #     Otherwise, the outcome will be FAIL.
  class OverallToolInvocationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the task completion check for the conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The task completion score. Can be -1, 0, 1
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Task Completed
  #     Score 0: Task Not Completed
  #     Score -1: User Goal Undefined
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the task completion score.
  class TaskCompletionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of the evaluation or expectation.
  module Outcome
    # Evaluation outcome is not specified.
    OUTCOME_UNSPECIFIED = 0

    # Evaluation/Expectation passed. In the case of an evaluation, this means
    # that all expectations were met.
    PASS = 1

    # Evaluation/Expectation failed. In the case of an evaluation, this means
    # that at least one expectation was not met.
    FAIL = 2

    # Evaluation/Expectation was skipped.
    SKIPPED = 3
  end

  # The state of the evaluation result execution.
  module ExecutionState
    # Evaluation result execution state is not specified.
    EXECUTION_STATE_UNSPECIFIED = 0

    # Evaluation result execution is running.
    RUNNING = 1

    # Evaluation result execution has completed.
    COMPLETED = 2

    # Evaluation result execution failed due to an internal error.
    ERROR = 3
  end
end

#scenario_result ⇒ ::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioResult (readonly)

Returns Output only. The outcome of a scenario evaluation.

Note: The following fields are mutually exclusive: scenario_result, golden_result. If a field in that set is populated, all other fields in the set will automatically be cleared.

Returns:



549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
 
# File 'proto_docs/google/cloud/ces/v1beta/evaluation.rb', line 549

class EvaluationResult
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # Specifies the expectation and the result of that expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
  #     Output only. The result of the tool call expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_tool_response`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_tool_response
  #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
  #     Output only. The result of the tool response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_response`, `observed_tool_call`, `observed_agent_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The result of the agent response expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`, `observed_tool_response`, `observed_agent_transfer`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_transfer
  #   @return [::Google::Cloud::Ces::V1beta::AgentTransfer]
  #     Output only. The result of the agent transfer expectation.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_transfer`, `observed_tool_call`, `observed_tool_response`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::GoldenExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the expectation.
  # @!attribute [r] semantic_similarity_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
  #     Output only. The result of the semantic similarity check.
  # @!attribute [r] tool_invocation_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome::ToolInvocationResult]
  #     Output only. The result of the tool invocation check.
  class GoldenExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of the tool invocation check.
    # @!attribute [r] parameter_correctness_score
    #   @return [::Float]
    #     Output only. The tool invocation parameter correctness score. This
    #     indicates the percent of parameters from the expected tool call that
    #     were also present in the actual tool call.
    # @!attribute [r] outcome
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
    #     Output only. The outcome of the tool invocation check. This is
    #     determined by comparing the parameter_correctness_score to the
    #     threshold. If the score is equal to or above the threshold, the outcome
    #     will be PASS. Otherwise, the outcome will be FAIL.
    # @!attribute [r] explanation
    #   @return [::String]
    #     Output only. A free text explanation for the tool invocation result.
    class ToolInvocationResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The result of a single evaluation expectation.
  # @!attribute [r] evaluation_expectation
  #   @return [::String]
  #     Output only. The evaluation expectation.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluation_expectation}`
  # @!attribute [r] prompt
  #   @return [::String]
  #     Output only. The prompt that was used for the evaluation.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the evaluation expectation.
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the result.
  class EvaluationExpectationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a golden evaluation.
  # @!attribute [r] turn_replay_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenResult::TurnReplayResult>]
  #     Output only. The result of running each turn of the golden conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class GoldenResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The result of running a single turn of the golden conversation.
    # @!attribute [r] conversation
    #   @return [::String]
    #     Output only. The conversation that was generated for this turn.
    # @!attribute [r] expectation_outcome
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::GoldenExpectationOutcome>]
    #     Output only. The outcome of each expectation.
    # @!attribute [r] hallucination_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult]
    #     Output only. The result of the hallucination check.
    # @!attribute [r] tool_invocation_score
    #   @deprecated This field is deprecated and may be removed in the next major version update.
    #   @return [::Float]
    #     Output only. Deprecated. Use OverallToolInvocationResult instead.
    # @!attribute [r] tool_ordered_invocation_score
    #   @return [::Float]
    #     Output only. The overall tool ordered invocation score for this turn.
    #     This indicates the overall percent of tools from the expected turn that
    #     were actually invoked in the expected order.
    # @!attribute [r] turn_latency
    #   @return [::Google::Protobuf::Duration]
    #     Output only. Duration of the turn.
    # @!attribute [r] tool_call_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
    #     Output only. The latency of each tool call in the turn.
    # @!attribute [r] semantic_similarity_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SemanticSimilarityResult]
    #     Output only. The result of the semantic similarity check.
    # @!attribute [r] overall_tool_invocation_result
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::OverallToolInvocationResult]
    #     Output only. The result of the overall tool invocation check.
    # @!attribute [r] error_info
    #   @return [::Google::Cloud::Ces::V1beta::EvaluationErrorInfo]
    #     Output only. Information about the error that occurred during this
    #     turn.
    # @!attribute [r] span_latencies
    #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
    #     Output only. The latency of spans in the turn.
    class TurnReplayResult
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of the evaluation against the rubric.
  # @!attribute [r] rubric
  #   @return [::String]
  #     Output only. The rubric that was used to evaluate the conversation.
  # @!attribute [r] score
  #   @return [::Float]
  #     Output only. The score of the conversation against the rubric.
  # @!attribute [r] score_explanation
  #   @return [::String]
  #     Output only. The rater's response to the rubric.
  class ScenarioRubricOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of a scenario expectation.
  # @!attribute [r] observed_tool_call
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome::ObservedToolCall]
  #     Output only. The observed tool call.
  #
  #     Note: The following fields are mutually exclusive: `observed_tool_call`, `observed_agent_response`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] observed_agent_response
  #   @return [::Google::Cloud::Ces::V1beta::Message]
  #     Output only. The observed agent response.
  #
  #     Note: The following fields are mutually exclusive: `observed_agent_response`, `observed_tool_call`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] expectation
  #   @return [::Google::Cloud::Ces::V1beta::Evaluation::ScenarioExpectation]
  #     Output only. The expectation that was evaluated.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the ScenarioExpectation.
  class ScenarioExpectationOutcome
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The observed tool call and response.
    # @!attribute [r] tool_call
    #   @return [::Google::Cloud::Ces::V1beta::ToolCall]
    #     Output only. The observed tool call.
    # @!attribute [r] tool_response
    #   @return [::Google::Cloud::Ces::V1beta::ToolResponse]
    #     Output only. The observed tool response.
    class ObservedToolCall
      include ::Google::Protobuf::MessageExts
      extend ::Google::Protobuf::MessageExts::ClassMethods
    end
  end

  # The outcome of a scenario evaluation.
  # @!attribute [r] conversation
  #   @return [::String]
  #     Output only. The conversation that was generated in the scenario.
  # @!attribute [r] task
  #   @return [::String]
  #     Output only. The task that was used when running the scenario for this
  #     result.
  # @!attribute [r] user_facts
  #   @return [::Array<::Google::Cloud::Ces::V1beta::Evaluation::Scenario::UserFact>]
  #     Output only. The user facts that were used by the scenario for this
  #     result.
  # @!attribute [r] expectation_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioExpectationOutcome>]
  #     Output only. The outcome of each expectation.
  # @!attribute [r] rubric_outcomes
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ScenarioRubricOutcome>]
  #     Output only. The outcome of the rubric.
  # @!attribute [r] hallucination_result
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::HallucinationResult>]
  #     Output only. The result of the hallucination check.
  #     There will be one hallucination result for each turn in the conversation.
  # @!attribute [r] task_completion_result
  #   @deprecated This field is deprecated and may be removed in the next major version update.
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::TaskCompletionResult]
  #     Output only. The result of the task completion check.
  # @!attribute [r] tool_call_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::ToolCallLatency>]
  #     Output only. The latency of each tool call execution in the conversation.
  # @!attribute [r] user_goal_satisfaction_result
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::UserGoalSatisfactionResult]
  #     Output only. The result of the user goal satisfaction check.
  # @!attribute [r] all_expectations_satisfied
  #   @return [::Boolean]
  #     Output only. Whether all expectations were satisfied for this turn.
  # @!attribute [r] task_completed
  #   @return [::Boolean]
  #     Output only. Whether the task was completed for this turn. This is a
  #     composite of all expectations satisfied, no hallucinations, and user goal
  #     satisfaction.
  # @!attribute [r] span_latencies
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency>]
  #     Output only. The latency of spans in the conversation.
  # @!attribute [r] evaluation_expectation_results
  #   @return [::Array<::Google::Cloud::Ces::V1beta::EvaluationResult::EvaluationExpectationResult>]
  #     Output only. The results of the evaluation expectations.
  class ScenarioResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The latency of a span execution.
  # @!attribute [r] resource
  #   @return [::String]
  #     Output only. The resource name of the guardrail or tool spans.
  #
  #     Note: The following fields are mutually exclusive: `resource`, `toolset`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] toolset
  #   @return [::Google::Cloud::Ces::V1beta::ToolsetTool]
  #     Output only. The toolset tool identifier.
  #
  #     Note: The following fields are mutually exclusive: `toolset`, `resource`, `model`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] model
  #   @return [::String]
  #     Output only. The name of the LLM span.
  #
  #     Note: The following fields are mutually exclusive: `model`, `resource`, `toolset`, `callback`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] callback
  #   @return [::String]
  #     Output only. The name of the user callback span.
  #
  #     Note: The following fields are mutually exclusive: `callback`, `resource`, `toolset`, `model`. If a field in that set is populated, all other fields in the set will automatically be cleared.
  # @!attribute [r] type
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::SpanLatency::Type]
  #     Output only. The type of span.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the span. Applicable to tool and
  #     guardrail spans.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of span.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of span.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of span.
  class SpanLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods

    # The type of span.
    # Additional values may be added in the future.
    module Type
      # Default value. This value is unused.
      TYPE_UNSPECIFIED = 0

      # Tool call span.
      TOOL = 1

      # User callback span.
      USER_CALLBACK = 2

      # Guardrail span.
      GUARDRAIL = 3

      # LLM span.
      LLM = 4
    end
  end

  # The latency of a tool call execution.
  # @!attribute [r] tool
  #   @return [::String]
  #     Output only. The name of the tool that got executed.
  #     Format:
  #     `projects/{project}/locations/{location}/apps/{app}/tools/{tool}`.
  # @!attribute [r] display_name
  #   @return [::String]
  #     Output only. The display name of the tool.
  # @!attribute [r] start_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The start time of the tool call execution.
  # @!attribute [r] end_time
  #   @return [::Google::Protobuf::Timestamp]
  #     Output only. The end time of the tool call execution.
  # @!attribute [r] execution_latency
  #   @return [::Google::Protobuf::Duration]
  #     Output only. The latency of the tool call execution.
  class ToolCallLatency
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the hallucination check for a single turn.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The hallucination score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Justified
  #     Score 0: Not Justified
  #     Score -1: No Claim To Assess
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the hallucination score.
  class HallucinationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of a user goal satisfaction check for a conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The user task satisfaction score. Can be -1, 0, 1.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: User Task Satisfied
  #     Score 0: User Task Not Satisfied
  #     Score -1: User Task Unspecified
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the user task satisfaction score.
  class UserGoalSatisfactionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the semantic similarity check.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The semantic similarity score. Can be 0, 1, 2, 3, or 4.
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 4: Fully Consistent
  #     Score 3: Mostly Consistent
  #     Score 2: Partially Consistent (Minor Omissions)
  #     Score 1: Largely Inconsistent (Major Omissions)
  #     Score 0: Completely Inconsistent / Contradictory
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the semantic similarity score.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the semantic similarity check. This is
  #     determined by comparing the score to the
  #     semantic_similarity_success_threshold. If the score is equal to or above
  #     the threshold, the outcome will be PASS. Otherwise, the outcome will be
  #     FAIL.
  class SemanticSimilarityResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the overall tool invocation check.
  # @!attribute [rw] tool_invocation_score
  #   @return [::Float]
  #     The overall tool invocation score for this turn. This indicates the
  #     overall percent of tools from the expected turn that were actually
  #     invoked.
  # @!attribute [r] outcome
  #   @return [::Google::Cloud::Ces::V1beta::EvaluationResult::Outcome]
  #     Output only. The outcome of the tool invocation check. This is determined
  #     by comparing the tool_invocation_score to the
  #     overall_tool_invocation_correctness_threshold. If the
  #     score is equal to or above the threshold, the outcome will be PASS.
  #     Otherwise, the outcome will be FAIL.
  class OverallToolInvocationResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The result of the task completion check for the conversation.
  # @!attribute [r] score
  #   @return [::Integer]
  #     Output only. The task completion score. Can be -1, 0, 1
  # @!attribute [r] label
  #   @return [::String]
  #     Output only. The label associated with each score.
  #     Score 1: Task Completed
  #     Score 0: Task Not Completed
  #     Score -1: User Goal Undefined
  # @!attribute [r] explanation
  #   @return [::String]
  #     Output only. The explanation for the task completion score.
  class TaskCompletionResult
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end

  # The outcome of the evaluation or expectation.
  module Outcome
    # Evaluation outcome is not specified.
    OUTCOME_UNSPECIFIED = 0

    # Evaluation/Expectation passed. In the case of an evaluation, this means
    # that all expectations were met.
    PASS = 1

    # Evaluation/Expectation failed. In the case of an evaluation, this means
    # that at least one expectation was not met.
    FAIL = 2

    # Evaluation/Expectation was skipped.
    SKIPPED = 3
  end

  # The state of the evaluation result execution.
  module ExecutionState
    # Evaluation result execution state is not specified.
    EXECUTION_STATE_UNSPECIFIED = 0

    # Evaluation result execution is running.
    RUNNING = 1

    # Evaluation result execution has completed.
    COMPLETED = 2

    # Evaluation result execution failed due to an internal error.
    ERROR = 3
  end
end