评估标准

evalset.json 文件

1// 注意：注释仅用于说明；有效 JSON 文件中应移除注释。
2{
3  "eval_set_id": "generic_eval_set_id",
4  "name": "通用评估集名称",
5  "description": "此评估集用途的描述。",
6  "eval_cases": [
7    {
8      "eval_id": "generic_eval_case_id",
9      "conversation": [
10        {
11          "invocation_id": "unique_invocation_identifier",
12          "user_content": { // 当前轮次的用户内容
13            "parts": [
14              {
15                "text": "此轮次用户的初始提示。"
16              }
17            ],
18            "role": "user"
19          },
20          "final_response": { // 预期的最终响应
21            "parts": [
22              {
23                "text": "代理应返回的‘黄金’最终文本响应。"
24              }
25            ],
26            "role": "model"
27          },
28          "intermediate_data": {
29            "tool_uses": [ // 预期的中间工具使用轨迹
30              {
31                "name": "expected_tool_name_1",
32                "args": {
33                  "parameter_name": "parameter_value"
34                }
35              },
36              {
37                "name": "expected_tool_name_2",
38                "args": {}
39              }
40            ],
41            "intermediate_responses": [] // 存放预期的中间代理响应
42          }
43        }
44      ],
45      "session_input": {
46        "app_name": "your_app_name",
47        "user_id": "your_user_id"
48      }
49    }
50  ]
51}

针对 tool_trajectory_avg_score 评估标准的 EvalConfig

1{
2  "criteria": {
3    "tool_trajectory_avg_score": {
4      "threshold": 1.0,
5      "match_type": 2
6    }
7  }
8}

针对 rubric_based_tool_use_quality_v1 评估标准的 EvalConfig

1{
2  "criteria": {
3    "rubric_based_tool_use_quality_v1": {
4      "threshold": 1.0,
5      "judge_model_options": {
6        "judge_model": "gemini-2.5-flash"
7      },
8      "rubrics": [
9        {
10          "rubric_id": "wikipedia_called",
11          "rubric_content": {
12            "text_property": "代理根据用户查询提取标题并调用 wikipedia 工具。"
13          }
14        }
15      ]
16    }
17  }
18}

针对 response_match_score 评估标准的 EvalConfig

1{
2  "criteria": {
3    "response_match_score": 0.8
4  }
5}

针对 final_response_match_v2 评估标准的 EvalConfig

1{
2  "criteria": {
3    "final_response_match_v2": {
4      "threshold": 0.8,
5      "judge_model_options": {
6            "judge_model": "gemini-2.5-flash"
7          }
8        }
9    }
10}

针对 rubric_based_final_response_quality_v1 评估标准的 EvalConfig

1{
2  "criteria": {
3    "rubric_based_final_response_quality_v1": {
4      "threshold": 0.8,
5      "judge_model_options": {
6        "judge_model": "gemini-2.5-flash"
7      },
8      "rubrics": [
9        {
10          "rubric_id": "conciseness",
11          "rubric_content": {
12            "text_property": "代理的回应直接且简洁。"
13          }
14        }
15      ]
16    }
17  }
18}

针对 hallucinations_v1 评估标准的 EvalConfig

1{
2  "criteria": {
3    "hallucinations_v1": {
4      "threshold": 0.8,
5      "judge_model_options": {
6            "judge_model": "gemini-2.5-flash"
7          },
8      "evaluate_intermediate_nl_responses": true
9    }
10  }
11}

针对 safety_v1 评估标准的 EvalConfig

1{
2  "criteria": {
3    "safety_v1": 0.8
4  }
5}

ConversationScenario

1conversation_scenarios = (
2"""{
3  "scenarios": [
4    {
5      "starting_prompt": "我需要预订一趟航班。",
6      "conversation_plan": "为下周五预订一张单程机票。在代理找到航班后，确认前询问行李政策。"
7    }
8  ]
9}""")