darthrubik/drharness
1
0
Fork 0
Code Issues 13 Pull Requests 1 Actions Packages Projects Releases 3 Wiki Activity

Refactoring tests a little

Browse Source
This commit is contained in:
Jacob Gamble
2026-05-09 11:56:21 -05:00
parent 4cbdcabae8
commit 2889a21eb6
9 changed files with 65 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/run_test.sh
+15 -4
View File
@@ -11,14 +11,12 @@ function check_tool_call() {
entry="$2"
expected_out="$3"
if [ "$(cat $chat_file | jq -r '.history['$entry'].role')" != "tool" ]; then
if [ "$(cat "$chat_file" | jq --arg e "$entry" -r '.history[$e | tonumber].role')" != "tool" ]; then
echo "Expected $entry chat to be a tool call result"
exit 1
fi
cat chat.json | jq -r '.history['$entry'].content' > tool_result
diff -u $expected_out tool_result
diff -u "$expected_out" <(cat "$chat_file" | jq --arg e "$entry" -r '.history[$e | tonumber].content')
}
function sanitize_json() {
echo "$1" | jq -R | sed -r 's/^.//g' | sed -r 's/.$//g'
@@ -40,6 +38,19 @@ export -f tool_call
export -f sanitize_json
export -f be_done
function history_length() {
local chat_file="$1"
jq '.history | length' "$chat_file"
}
export -f history_length
function dr_result() {
local chat_file="$1"
local entry="$2"
jq -r --arg e "$entry" '.history[$e | tonumber].content' "$chat_file" 2>/dev/null || echo ""
}
export -f dr_result
# Resolve paths relative to this script
SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
PROJECT_ROOT=$(cd "${SCRIPT_DIR}/.." && pwd)
tests/test_load_chat.sh
+9 -11
View File
@@ -11,18 +11,16 @@ be_done 'Step 1 done' >> mock_responses1.txt
"${DRHARNESS}" -F load_chat.json -n -m "Start chat step 1" -t read_file -d < "mock_responses1.txt" >/dev/null 2>&1
# Verify step 1 saved correctly
ITEMS_BEFORE=$(jq '.history | length' load_chat.json)
if [ "$ITEMS_BEFORE" -lt 4 ]; then
HIST_LEN=$(history_length load_chat.json)
if [ "$HIST_LEN" -lt 4 ]; then
echo "FAIL: load_chat - initial chat didn't save properly"
exit 1
fi
# Verify the file content was read correctly
FIRST_RESULT=$(jq -r '.history[2].content' load_chat.json)
if [ "$FIRST_RESULT" != "Initial content" ]; then
RESULT=$(dr_result load_chat.json 2)
if [ "$RESULT" != "Initial content" ]; then
echo "FAIL: load_chat - step 1 file content mismatch"
echo "Expected: Initial content"
echo "Got: $FIRST_RESULT"
exit 1
fi
@@ -32,17 +30,17 @@ be_done 'Step 2 done' >> mock_responses2.txt
"${DRHARNESS}" -F load_chat.json -m "Continue chat step 2" -t read_file -d < "mock_responses2.txt" >/dev/null 2>&1
ITEMS_AFTER=$(jq '.history | length' load_chat.json)
if [ "$ITEMS_AFTER" -le "$ITEMS_BEFORE" ]; then
HIST_LEN_AFTER=$(history_length load_chat.json)
if [ "$HIST_LEN_AFTER" -le "$HIST_LEN" ]; then
echo "FAIL: load_chat - history was not appended"
exit 1
fi
# Verify the old tool result is still there
OLD_RESULT=$(jq -r '.history[2].content' load_chat.json)
if [ "$OLD_RESULT" != "Initial content" ]; then
RESULT2=$(dr_result load_chat.json 2)
if [ "$RESULT2" != "Initial content" ]; then
echo "FAIL: load_chat - history was not preserved"
echo "Expected initial content, got: $OLD_RESULT"
echo "Expected initial content, got: $RESULT2"
exit 1
fi
tests/test_n_flag.sh
+2 -6
View File
@@ -5,12 +5,8 @@
echo "existing data" > non_empty.json
# Attempt to start a new conversation with -n pointing to a non-empty file
# Use || true to prevent run_test.sh's set -e from exiting the script
"${DRHARNESS}" -F non_empty.json -n -m "Should fail" -d < /dev/null 2> stderr_output || true
# Check that the error message is present in stderr
if ! grep -q "Error: Cannot start a new conversation with an existing, non-empty file" stderr_output; then
echo "FAIL: -n flag - unexpected error message"
if "${DRHARNESS}" -F non_empty.json -n -m "Should fail" -d < /dev/null 2> stderr_output ; then
echo "Didn't fail on new on non empty file"
cat stderr_output
exit 1
fi
tests/test_tool_edit.sh
+5 -12
View File
@@ -45,17 +45,10 @@ be_done 'Finished' >> mock_responses2.txt
OUTPUT=$("${DRHARNESS}" -F chat2.json -n -m "Try invalid edit" -t edit -d < "mock_responses2.txt" 2>/dev/null)
# The harness returns the error from the tool
if ! echo "${OUTPUT}" | grep -q "Edit complete\|Finished"; then
# Actually, it should just say Finished because the tool returned an error
# But we want to check the tool result in the json
true
fi
if ! grep -q '"Error: oldString not found in content"' chat2.json; then
echo "FAIL: edit tool - did not record error from tool"
cat chat2.json
exit 1
fi
# Check the tool result in the json using helper
cat > expected_edit_error.txt << 'EOF'
Error: oldString not found in content
EOF
check_tool_call chat2.json 2 expected_edit_error.txt
echo "PASS: tool_edit"
tests/test_tool_grep.sh
+11 -12
View File
@@ -17,17 +17,16 @@ if [ "${OUTPUT}" != "Grep complete" ]; then
exit 1
fi
TOOL_RESULT=$(jq -r '.history[2].content' chat.json)
if ! echo "${TOOL_RESULT}" | grep -q "Hello World"; then
result=$(dr_result chat.json 2)
if ! echo "$result" | grep -q "Hello World"; then
echo "FAIL: grep_search tool - expected 'Hello World' in output"
echo "Got: ${TOOL_RESULT}"
echo "Got: $result"
exit 1
fi
if ! echo "${TOOL_RESULT}" | grep -q "Hello drharness"; then
if ! echo "$result" | grep -q "Hello drharness"; then
echo "FAIL: grep_search tool - expected 'Hello drharness' in output"
echo "Got: ${TOOL_RESULT}"
echo "Got: $result"
exit 1
fi
@@ -37,11 +36,11 @@ tool_call grep_search '{"path":"grep_test.txt","pattern":"goodbye","case_insensi
be_done 'Case insensitive done' >> mock_responses2.txt
"${DRHARNESS}" -F chat.json -n -m "Search case insensitive" -t grep_search -d < "mock_responses2.txt" >/dev/null 2>&1
TOOL_RESULT=$(jq -r '.history[2].content' chat.json)
result=$(dr_result chat.json 2)
if ! echo "${TOOL_RESULT}" | grep -q "Goodbye World"; then
if ! echo "$result" | grep -q "Goodbye World"; then
echo "FAIL: grep_search tool - expected case insensitive match 'Goodbye World'"
echo "Got: ${TOOL_RESULT}"
echo "Got: $result"
exit 1
fi
@@ -51,11 +50,11 @@ tool_call grep_search '{"path":"grep_test.txt","pattern":"[invalid"}' >> mock_re
be_done 'Regex error done' >> mock_responses3.txt
"${DRHARNESS}" -F chat.json -n -m "Invalid regex" -t grep_search -d < "mock_responses3.txt" >/dev/null 2>&1
TOOL_RESULT=$(jq -r '.history[2].content' chat.json)
result=$(dr_result chat.json 2)
if ! echo "${TOOL_RESULT}" | grep -q "Error: invalid regex pattern"; then
if ! echo "$result" | grep -q "Error: invalid regex pattern"; then
echo "FAIL: grep_search tool - expected regex error"
echo "Got: ${TOOL_RESULT}"
echo "Got: $result"
exit 1
fi
tests/test_tool_list_files.sh
+5 -5
View File
@@ -18,9 +18,9 @@ if [ "${OUTPUT}" != "List complete" ]; then
fi
# Verify tool result in chat.json contains directory listing
TOOL_RESULT=$(jq -r '.history[2].content' chat.json)
# Check that file1.txt and subdir/ are present
# Using dr_result helper to get the content
# Note: check_tool_call expects a file, so we use it with a dummy file or direct dr_result check
TOOL_RESULT=$(dr_result chat.json 2)
if ! echo "${TOOL_RESULT}" | grep -q "file1.txt"; then
echo "FAIL: list_files tool - expected 'file1.txt' in output"
echo "Got: ${TOOL_RESULT}"
@@ -40,7 +40,7 @@ be_done 'Recursive complete' >> mock_responses2.txt
OUTPUT=$("${DRHARNESS}" -F chat.json -n -m "List recursively" -t list_files -d < "mock_responses2.txt" 2>/dev/null)
TOOL_RESULT=$(jq -r '.history[2].content' chat.json)
TOOL_RESULT=$(dr_result chat.json 2)
if ! echo "${TOOL_RESULT}" | grep -q "subdir/file2.txt"; then
echo "FAIL: list_files tool (recursive) - expected 'subdir/file2.txt' in output"
@@ -54,7 +54,7 @@ tool_call list_files '{"path":"nonexistent_dir"}' >> mock_responses3.txt
be_done 'Error complete' >> mock_responses3.txt
"${DRHARNESS}" -F chat.json -n -m "List nonexistent" -t list_files -d < "mock_responses3.txt" > /dev/null 2>&1
TOOL_RESULT=$(jq -r '.history[2].content' chat.json)
TOOL_RESULT=$(dr_result chat.json 2)
if ! echo "${TOOL_RESULT}" | grep -q "Error: path does not exist"; then
echo "FAIL: list_files tool - expected error for nonexistent path"
tests/test_tool_read_file.sh
+8 -15
View File
@@ -18,17 +18,11 @@ if [ "${OUTPUT}" != "Read complete" ]; then
exit 1
fi
TOOL_RESULT=$(jq -r '.history[2].content' chat.json)
if ! echo "${TOOL_RESULT}" | grep -q "Line 1 of test"; then
echo "FAIL: read_file tool - expected 'Line 1 of test' in output"
echo "Got: ${TOOL_RESULT}"
exit 1
fi
if ! echo "${TOOL_RESULT}" | grep -q "Line 2 of test"; then
echo "FAIL: read_file tool - expected 'Line 2 of test' in output"
echo "Got: ${TOOL_RESULT}"
RESULT=$(dr_result chat.json 2)
if [ "$RESULT" != "Line 1 of test
Line 2 of test" ]; then
echo "FAIL: read_file tool - content mismatch"
echo "Got: $RESULT"
exit 1
fi
@@ -38,11 +32,10 @@ tool_call read_file '{"path":"nonexistent.txt"}' >> mock_responses2.txt
be_done 'File error done' >> mock_responses2.txt
"${DRHARNESS}" -F chat.json -n -m "Read nonexistent" -t read_file -d < "mock_responses2.txt" >/dev/null 2>&1
TOOL_RESULT=$(jq -r '.history[2].content' chat.json)
if ! echo "${TOOL_RESULT}" | grep -q "Error: file does not exist"; then
RESULT=$(dr_result chat.json 2)
if ! echo "$RESULT" | grep -q "Error: file does not exist"; then
echo "FAIL: read_file tool - expected error for nonexistent file"
echo "Got: ${TOOL_RESULT}"
echo "Got: $RESULT"
exit 1
fi
tests/test_tool_restrictions.sh
+5 -13
View File
@@ -12,20 +12,12 @@ be_done 'Restricted done' >> mock_responses.txt
# Run harness with only 'edit' tool specified
"${DRHARNESS}" -F chat.json -n -m "Try to use read_file but only edit is allowed" -t edit -d < "mock_responses.txt" >/dev/null 2>&1
EXIT_CODE=$?
if [ $EXIT_CODE -ne 0 ]; then
# It might exit if it throws uncaught, but based on harness.cpp it catches and returns error
# Let's just check the tool result
true
fi
# Write expected error to a file to compare via helper
cat > expected_error.txt << 'EOF'
Error: unknown tool: read_file
EOF
TOOL_RESULT=$(jq -r '.history[2].content' chat.json)
if ! echo "${TOOL_RESULT}" | grep -q "Error: unknown tool: read_file"; then
echo "FAIL: tool restriction - expected tool to be rejected"
echo "Got: ${TOOL_RESULT}"
exit 1
fi
check_tool_call chat.json 2 expected_error.txt
echo "PASS: tool_restrictions"
tests/test_tool_shell.sh
+5 -6
View File
@@ -12,11 +12,10 @@ if [ "${OUTPUT}" != "Shell done" ]; then
exit 1
fi
TOOL_RESULT=$(jq -r '.history[2].content' chat.json)
if ! echo "${TOOL_RESULT}" | grep -q "Hello from shell"; then
RESULT=$(dr_result chat.json 2)
if ! echo "$RESULT" | grep -q '^Hello from shell$' ; then
echo "FAIL: run_shell tool - expected 'Hello from shell' in output"
echo "Got: ${TOOL_RESULT}"
echo "Got: $RESULT"
exit 1
fi
@@ -26,9 +25,9 @@ tool_call run_shell '{"command":"exit 42"}' >> mock_responses2.txt
be_done 'Error handled' >> mock_responses2.txt
"${DRHARNESS}" -F chat.json -n -m "Run invalid shell" -t run_shell -d < "mock_responses2.txt" >/dev/null 2>&1
TOOL_RESULT=$(jq -r '.history[2].content' chat.json)
TOOL_RESULT=$(dr_result chat.json 2)
# Note: buggy behavior returns "(no output)" instead of "[exit 42]" due to waitpid logic in shell_tool.cpp
# Note: buggy behavior returns "(no output)" instead of "[exit 42]" due to waitpid logic in shell_tool.cpp. TODO: Fix later
if ! echo "${TOOL_RESULT}" | grep -q "\[exit 42\]"; then
echo "FAIL: run_shell tool - expected exit code in output"
echo "Got: ${TOOL_RESULT}"