drharness/tests/test_encoding.sh

#!/bin/bash
# test_encoding.sh - Tests for UTF-8 encoding validation and conversion
# All tests exercise encoding through the read_file tool

# Test 1: Valid ASCII passes through unchanged
printf 'Hello' > test_ascii.txt
tool_call read_file '{"path":"test_ascii.txt"}' >> mock1.txt
be_done 'done' >> mock1.txt
"${DRHARNESS}" -F chat1.json -n -m "test" -t read_file -d < "mock1.txt" >/dev/null 2>&1
RESULT=$(dr_result chat1.json 2)
if [ "${RESULT}" != "Hello" ]; then
    echo "FAIL: ASCII passthrough - expected 'Hello', got '${RESULT}'"
    exit 1
fi

# Test 2: Valid UTF-8 2-byte sequence (é = c3a9) passes through unchanged
printf '\xc3\xa9' > test_utf8_2byte.txt
tool_call read_file '{"path":"test_utf8_2byte.txt"}' >> mock2.txt
be_done 'done' >> mock2.txt
"${DRHARNESS}" -F chat2.json -n -m "test" -t read_file -d < "mock2.txt" >/dev/null 2>&1
RESULT=$(dr_result chat2.json 2)
RESULT_HEX=$(od -An -tx1 -v <<< "${RESULT}" | tr -d ' \n' | sed 's/0a$//')
if [ "${RESULT_HEX}" != "c3a9" ]; then
    echo "FAIL: valid UTF-8 2-byte - expected 'c3a9', got '${RESULT_HEX}'"
    exit 1
fi

# Test 3: Valid UTF-8 3-byte sequence (中 = e4b8ad) passes through unchanged
printf '\xe4\xb8\xad' > test_utf8_3byte.txt
tool_call read_file '{"path":"test_utf8_3byte.txt"}' >> mock3.txt
be_done 'done' >> mock3.txt
"${DRHARNESS}" -F chat3.json -n -m "test" -t read_file -d < "mock3.txt" >/dev/null 2>&1
RESULT=$(dr_result chat3.json 2)
RESULT_HEX=$(od -An -tx1 -v <<< "${RESULT}" | tr -d ' \n' | sed 's/0a$//')
if [ "${RESULT_HEX}" != "e4b8ad" ]; then
    echo "FAIL: valid UTF-8 3-byte - expected 'e4b8ad', got '${RESULT_HEX}'"
    exit 1
fi

# Test 4: Valid UTF-8 4-byte sequence (😀 = f09f9880) passes through unchanged
printf '\xf0\x9f\x98\x80' > test_utf8_4byte.txt
tool_call read_file '{"path":"test_utf8_4byte.txt"}' >> mock4.txt
be_done 'done' >> mock4.txt
"${DRHARNESS}" -F chat4.json -n -m "test" -t read_file -d < "mock4.txt" >/dev/null 2>&1
RESULT=$(dr_result chat4.json 2)
RESULT_HEX=$(od -An -tx1 -v <<< "${RESULT}" | tr -d ' \n' | sed 's/0a$//')
if [ "${RESULT_HEX}" != "f09f9880" ]; then
    echo "FAIL: valid UTF-8 4-byte - expected 'f09f9880', got '${RESULT_HEX}'"
    exit 1
fi

# Test 5: Latin-1 byte 0xe9 (é in Latin-1) should be converted to UTF-8 c3a9
printf '\xe9' > test_latin1_e9.txt
tool_call read_file '{"path":"test_latin1_e9.txt"}' >> mock5.txt
be_done 'done' >> mock5.txt
"${DRHARNESS}" -F chat5.json -n -m "test" -t read_file -d < "mock5.txt" >/dev/null 2>&1
RESULT=$(dr_result chat5.json 2)
RESULT_HEX=$(od -An -tx1 -v <<< "${RESULT}" | tr -d ' \n' | sed 's/0a$//')
if [ "${RESULT_HEX}" != "c3a9" ]; then
    echo "FAIL: Latin-1 e9 to UTF-8 - expected 'c3a9', got '${RESULT_HEX}'"
    exit 1
fi

# Test 6: Multiple Latin-1 bytes (café in Latin-1 = 636166e9)
printf '\x63\x61\x66\xe9' > test_latin1_cafe.txt
tool_call read_file '{"path":"test_latin1_cafe.txt"}' >> mock6.txt
be_done 'done' >> mock6.txt
"${DRHARNESS}" -F chat6.json -n -m "test" -t read_file -d < "mock6.txt" >/dev/null 2>&1
RESULT=$(dr_result chat6.json 2)
RESULT_HEX=$(od -An -tx1 -v <<< "${RESULT}" | tr -d ' \n' | sed 's/0a$//')
if [ "${RESULT_HEX}" != "636166c3a9" ]; then
    echo "FAIL: Latin-1 café to UTF-8 - expected '636166c3a9', got '${RESULT_HEX}'"
    exit 1
fi

# Test 7: Byte 0x80 (isolated continuation byte) should be converted to c280
printf '\x80' > test_cont_byte.txt
tool_call read_file '{"path":"test_cont_byte.txt"}' >> mock7.txt
be_done 'done' >> mock7.txt
"${DRHARNESS}" -F chat7.json -n -m "test" -t read_file -d < "mock7.txt" >/dev/null 2>&1
RESULT=$(dr_result chat7.json 2)
RESULT_HEX=$(od -An -tx1 -v <<< "${RESULT}" | tr -d ' \n' | sed 's/0a$//')
if [ "${RESULT_HEX}" != "c280" ]; then
    echo "FAIL: continuation byte 80 - expected 'c280', got '${RESULT_HEX}'"
    exit 1
fi

# Test 8: Mixed valid UTF-8 and invalid bytes
# "A" + invalid 0xe9 + "B" = 41e942 -> expected: 41c3a942
printf '\x41\xe9\x42' > test_mixed.txt
tool_call read_file '{"path":"test_mixed.txt"}' >> mock8.txt
be_done 'done' >> mock8.txt
"${DRHARNESS}" -F chat8.json -n -m "test" -t read_file -d < "mock8.txt" >/dev/null 2>&1
RESULT=$(dr_result chat8.json 2)
RESULT_HEX=$(od -An -tx1 -v <<< "${RESULT}" | tr -d ' \n' | sed 's/0a$//')
if [ "${RESULT_HEX}" != "41c3a942" ]; then
    echo "FAIL: mixed valid/invalid - expected '41c3a942', got '${RESULT_HEX}'"
    exit 1
fi

# Test 9: Empty file returns "(empty file)"
printf '' > test_empty.txt
tool_call read_file '{"path":"test_empty.txt"}' >> mock9.txt
be_done 'done' >> mock9.txt
"${DRHARNESS}" -F chat9.json -n -m "test" -t read_file -d < "mock9.txt" >/dev/null 2>&1
RESULT=$(dr_result chat9.json 2)
if [ "${RESULT}" != "(empty file)" ]; then
    echo "FAIL: empty file - expected '(empty file)', got '${RESULT}'"
    exit 1
fi

# Test 10: File with newline and non-UTF8 content
printf 'line1\n\xe9\nline3\n' > test_multiline.txt
tool_call read_file '{"path":"test_multiline.txt"}' >> mock10.txt
be_done 'done' >> mock10.txt
"${DRHARNESS}" -F chat10.json -n -m "test" -t read_file -d < "mock10.txt" >/dev/null 2>&1
RESULT=$(dr_result chat10.json 2)
# Check that the file content was properly converted
if ! echo "${RESULT}" | grep -q "line1"; then
    echo "FAIL: multiline - missing 'line1' in result"
    exit 1
fi
# Verify the é is properly UTF-8 encoded
RESULT_HEX=$(od -An -tx1 -v <<< "${RESULT}" | tr -d ' \n' | sed 's/0a$//')
if ! echo "${RESULT_HEX}" | grep -q "c3a9"; then
    echo "FAIL: multiline - expected UTF-8 é (c3a9) in result, got '${RESULT_HEX}'"
    exit 1
fi

echo "PASS: encoding tests"