hello-agents/code/chapter12/06_gaia_best_practices.py

"""
第十二章示例6：GAIA评估最佳实践

对应文档：12.3.9 GAIA评估最佳实践

这个示例展示了GAIA评估的最佳实践，包括：
1. 分级评估
2. 小样本快速测试
3. 结果解读
"""

import os
from hello_agents import SimpleAgent, HelloAgentsLLM
from hello_agents.tools import GAIAEvaluationTool

# GAIA官方系统提示词
GAIA_SYSTEM_PROMPT = """You are a general AI assistant. I will ask you a question. Report your thoughts, and finish your answer with the following template: FINAL ANSWER: [YOUR FINAL ANSWER].
YOUR FINAL ANSWER should be a number OR as few words as possible OR a comma separated list of numbers and/or strings.
If you are asked for a number, don't use comma to write your number neither use units such as $ or percent sign unless specified otherwise.
If you are asked for a string, don't use articles, neither abbreviations (e.g. for cities), and write the digits in plain text unless specified otherwise.
If you are asked for a comma separated list, apply the above rules depending of whether the element to be put in the list is a number or a string."""

# 创建智能体
llm = HelloAgentsLLM()
agent = SimpleAgent(
    name="TestAgent",
    llm=llm,
    system_prompt=GAIA_SYSTEM_PROMPT
)

# 创建评估工具
gaia_tool = GAIAEvaluationTool()

# ============================================================
# 最佳实践1：分级评估
# ============================================================
print("="*60)
print("最佳实践1：分级评估")
print("="*60)

# 第一步：评估Level 1（简单任务）
print("\n第一步：评估Level 1（简单任务）")
results_l1 = gaia_tool.run(agent, level=1, max_samples=10)
print(f"Level 1精确匹配率: {results_l1['exact_match_rate']:.2%}")

# 第二步：如果Level 1表现良好，评估Level 2
if results_l1['exact_match_rate'] > 0.6:
    print("\n第二步：评估Level 2（中等任务）")
    results_l2 = gaia_tool.run(agent, level=2, max_samples=10)
    print(f"Level 2精确匹配率: {results_l2['exact_match_rate']:.2%}")
    
    # 第三步：如果Level 2表现良好，评估Level 3
    if results_l2['exact_match_rate'] > 0.4:
        print("\n第三步：评估Level 3（困难任务）")
        results_l3 = gaia_tool.run(agent, level=3, max_samples=10)
        print(f"Level 3精确匹配率: {results_l3['exact_match_rate']:.2%}")
    else:
        print("\n⚠️ Level 2表现不佳，建议先优化后再评估Level 3")
else:
    print("\n⚠️ Level 1表现不佳，建议先优化后再评估更高级别")

# ============================================================
# 最佳实践2：小样本快速测试
# ============================================================
print("\n" + "="*60)
print("最佳实践2：小样本快速测试")
print("="*60)

# 快速测试（每个级别2个样本）
for level in [1, 2, 3]:
    print(f"\n快速测试 Level {level}:")
    results = gaia_tool.run(agent, level=level, max_samples=2)
    print(f"  精确匹配率: {results['exact_match_rate']:.2%}")

# ============================================================
# 最佳实践3：结果解读
# ============================================================
print("\n" + "="*60)
print("最佳实践3：结果解读")
print("="*60)

def interpret_results(level, exact_match_rate):
    """解读评估结果"""
    print(f"\nLevel {level} 结果解读:")
    print(f"精确匹配率: {exact_match_rate:.2%}")
    
    if level == 1:
        if exact_match_rate >= 0.6:
            print("✅ 优秀 - 基础能力扎实")
        elif exact_match_rate >= 0.4:
            print("⚠️ 良好 - 基础能力可用")
        else:
            print("❌ 较差 - 需要改进")
            print("建议:")
            print("  - 检查系统提示词是否包含GAIA官方格式要求")
            print("  - 检查答案提取逻辑是否正确")
            print("  - 检查LLM模型是否足够强大")
    
    elif level == 2:
        if exact_match_rate >= 0.4:
            print("✅ 优秀 - 中等任务能力强")
        elif exact_match_rate >= 0.2:
            print("⚠️ 良好 - 中等任务能力可用")
        else:
            print("❌ 较差 - 需要改进")
            print("建议:")
            print("  - 增强多步推理能力")
            print("  - 增加工具使用能力")
            print("  - 优化推理链的构建")
    
    elif level == 3:
        if exact_match_rate >= 0.2:
            print("✅ 优秀 - 复杂任务能力强")
        elif exact_match_rate >= 0.1:
            print("⚠️ 良好 - 复杂任务能力可用")
        else:
            print("❌ 较差 - 需要改进")
            print("建议:")
            print("  - 增强复杂推理能力")
            print("  - 增加长上下文处理能力")
            print("  - 优化工具链的组合使用")

# 解读结果
if 'results_l1' in locals():
    interpret_results(1, results_l1['exact_match_rate'])
if 'results_l2' in locals():
    interpret_results(2, results_l2['exact_match_rate'])
if 'results_l3' in locals():
    interpret_results(3, results_l3['exact_match_rate'])

# ============================================================
# 难度递进分析
# ============================================================
print("\n" + "="*60)
print("难度递进分析")
print("="*60)

if 'results_l1' in locals() and 'results_l2' in locals():
    if results_l1['exact_match_rate'] > results_l2['exact_match_rate']:
        print("✅ 正常递进：Level 1 > Level 2")
    else:
        print("⚠️ 异常情况：Level 2 >= Level 1（可能是数据集偏差或智能体特性）")

if 'results_l2' in locals() and 'results_l3' in locals():
    if results_l2['exact_match_rate'] > results_l3['exact_match_rate']:
        print("✅ 正常递进：Level 2 > Level 3")
    else:
        print("⚠️ 异常情况：Level 3 >= Level 2（可能是数据集偏差或智能体特性）")