drharness/tests/test_expected_errors.sh

#!/bin/bash
# test_expected_errors.sh - Tests for std::expected error handling in Harness::chat()

# Test 1: Client retry exhaustion produces final error
# After 20 failed client calls, should return "max retries exceeded" error
test_client_retry_exhaustion() {
    local test_name="client_retry_exhaustion"
    
    # Create mock responses file with 19 invalid JSON responses
    # The debug client reads from stdin and the retry loop attempts 19 calls
    # before returning "max retries exceeded"
    > mock_responses.txt
    for i in $(seq 1 19); do
        echo "invalid json" >> mock_responses.txt
    done
    
    # Run drharness - should fail with non-zero exit code due to max retries exceeded
    OUTPUT=$("${DRHARNESS}" -F chat_retry.json -n -m "test" -d < "mock_responses.txt" 2>&1) || true
    
    # Check that the error message contains "max retries exceeded"
    if ! echo "$OUTPUT" | grep -q "max retries exceeded"; then
        echo "FAIL: $test_name - expected 'max retries exceeded' in output"
        echo "Output was: $OUTPUT"
        return 1
    fi
    
    # Ensure we didn't hit stdin exhaustion - all errors should be parse errors
    if echo "$OUTPUT" | grep -q "failed to read from stdin"; then
        echo "FAIL: $test_name - unexpected 'failed to read from stdin' (not enough mock responses)"
        echo "Output was: $OUTPUT"
        return 1
    fi
    
    echo "PASS: $test_name - max retries exceeded error detected"
}

# Test 2: Max tool iterations produces error
# When tool calls keep being requested, should eventually hit max iterations limit
test_max_tool_iterations() {
    local test_name="max_tool_iterations"
    
    # Create mock responses that keep requesting tool calls
    # The harness has max_tool_iterations=1000 by default in main.cpp
    # But we can test with a smaller limit by checking the error message
    
    # For this test, we need to create a scenario where tool iterations are exhausted
    # We'll use the edit tool which will fail, but we keep getting tool_call responses
    
    # Create a file that the read_file tool can read
    echo "test content" > test_file.txt
    
    # Generate 1001 tool call responses (exceeds default max of 1000)
    > mock_responses.txt
    resp_line=$(tool_call read_file '{"path":"test_file.txt"}')
    for i in $(seq 1 1001); do
        echo "$resp_line" >> mock_responses.txt
    done
    be_done "final response" >> mock_responses.txt
    
    # Run drharness - should fail with non-zero exit code due to max tool iterations exceeded
    OUTPUT=$("${DRHARNESS}" -F chat_iter.json -n -m "test" -t read_file -d < "mock_responses.txt" 2>&1) || true
    
    # Check that the error message contains "max tool iterations exceeded"
    if echo "$OUTPUT" | grep -q "max tool iterations exceeded"; then
        echo "PASS: $test_name - max tool iterations exceeded error detected"
    else
        echo "FAIL: $test_name - expected 'max tool iterations exceeded' in output"
        echo "Output was: $OUTPUT"
        return 1
    fi
}

# Test 3: Tool invocation failure produces error string as tool result
# When a tool fails, the error should be returned to the LLM as a tool result
test_tool_invocation_failure() {
    local test_name="tool_invocation_failure"
    
    # Create a tool call with invalid JSON arguments
    # This should produce an "Error: invalid JSON in tool arguments" message
    > mock_responses.txt
    tool_call_raw read_file 'invalid json' >> mock_responses.txt
    be_done "Done" >> mock_responses.txt
    
    OUTPUT=$("${DRHARNESS}" -F chat_invocation.json -n -m "test" -t read_file -d < "mock_responses.txt")
    
    if [ "${OUTPUT}" != "Done" ]; then
        echo "FAIL: $test_name - expected 'Done' as final output"
        echo "Output was: ${OUTPUT}"
        return 1
    fi
    
    # Check that the tool result contains the error message
    TOOL_RESULT=$(dr_result chat_invocation.json 2)
    if echo "$TOOL_RESULT" | grep -q "Error: invalid JSON"; then
        echo "PASS: $test_name - tool invocation error returned as tool result"
    else
        echo "FAIL: $test_name - expected error in tool result"
        echo "Tool result was: $TOOL_RESULT"
        return 1
    fi
}

# Test 4: Missing required field in tool parameters produces error string
# When a tool call is missing a required field, the error should be returned to the LLM
test_tool_missing_required_field() {
    local test_name="tool_missing_required_field"
    
    # Create a tool call with missing required field (filePath for edit tool)
    > mock_responses.txt
    tool_call edit '{"oldString":"foo","newString":"bar"}' >> mock_responses.txt
    be_done "Done" >> mock_responses.txt
    
    OUTPUT=$("${DRHARNESS}" -F chat_missing.json -n -m "test" -t edit -d < "mock_responses.txt")
    
    if [ "${OUTPUT}" != "Done" ]; then
        echo "FAIL: $test_name - expected 'Done' as final output"
        echo "Output was: ${OUTPUT}"
        return 1
    fi
    
    # Check that the tool result contains the error message about missing field
    TOOL_RESULT=$(dr_result chat_missing.json 2)
    if echo "$TOOL_RESULT" | grep -q "missing required field"; then
        echo "PASS: $test_name - missing required field error returned as tool result"
    else
        echo "FAIL: $test_name - expected 'missing required field' in tool result"
        echo "Tool result was: $TOOL_RESULT"
        return 1
    fi
}

# Run all tests
test_client_retry_exhaustion
test_max_tool_iterations
test_tool_invocation_failure
test_tool_missing_required_field

echo "All expected error tests passed"