💻 Exercise: Exercise: Classify a failure

📝 Instructions

Write classify_failure(trace). If the run succeeded ('success' truthy) return None. Otherwise classify in this order: 'max_steps_hit' if stop_reason == 'max_steps' or steps >= max_steps; 'tool_error' if tool_errors > 0 or stop_reason == 'error'; 'no_final_answer' if final_answer is missing or empty; else 'wrong_answer'. Standard library only.

🧪 Initial Code / Tests 1 Files

📄 evaluate.py

from unittest import TestCase
from exercise import classify_failure


class Evaluate(TestCase):
    def test_success_returns_none(self):
        self.assertIsNone(classify_failure({'success': True}))

    def test_max_steps_hit(self):
        self.assertEqual(classify_failure(
            {'success': False, 'stop_reason': 'max_steps', 'steps': 8, 'max_steps': 8}),
            'max_steps_hit')

    def test_tool_error(self):
        self.assertEqual(classify_failure(
            {'success': False, 'stop_reason': 'error', 'tool_errors': 1}), 'tool_error')

    def test_wrong_answer_when_answer_present(self):
        self.assertEqual(classify_failure(
            {'success': False, 'final_answer': 'nope', 'steps': 2, 'max_steps': 8}),
            'wrong_answer')

✅ Solutions 1 Files

📄 exercise.py

def classify_failure(trace):
    if trace.get('success'):
        return None
    steps = trace.get('steps', 0)
    max_steps = trace.get('max_steps', float('inf'))
    if trace.get('stop_reason') == 'max_steps' or steps >= max_steps:
        return 'max_steps_hit'
    if trace.get('tool_errors', 0) > 0 or trace.get('stop_reason') == 'error':
        return 'tool_error'
    if not trace.get('final_answer'):
        return 'no_final_answer'
    return 'wrong_answer'