hal9000
02975ddfd4
- Added dr_config_base_url and dr_config_model helper functions to run_test.sh - Updated test_harness_config.sh to use be_done() and helper functions instead of inline JSON
509 lines
14 KiB
Bash
Executable File
509 lines
14 KiB
Bash
Executable File
#!/bin/bash
|
|
# run_test.sh - Common test runner with setup and teardown logic
|
|
# Usage: run_test.sh <test_script.sh>
|
|
|
|
set -eu
|
|
set -x
|
|
|
|
|
|
function check_tool_call() {
|
|
chat_file="$1"
|
|
entry="$2"
|
|
expected_out="$3"
|
|
|
|
if [ "$(cat "$chat_file" | jq --arg e "$entry" -r '.history[$e | tonumber].role')" != "tool" ]; then
|
|
echo "Expected $entry chat to be a tool call result"
|
|
exit 1
|
|
fi
|
|
|
|
diff -u "$expected_out" <(cat "$chat_file" | jq --arg e "$entry" -r '.history[$e | tonumber].content')
|
|
}
|
|
# Generates a message JSON blob
|
|
# Usage: message [options]
|
|
# Options:
|
|
# --content TEXT Set message content
|
|
# --tool-call NAME ARGS Add a tool call (name and JSON args)
|
|
# --reasoning TEXT Add reasoning field
|
|
# --reasoning-content TEXT Add reasoning_content field
|
|
function message() {
|
|
local content=""
|
|
local reasoning=""
|
|
local reasoning_content=""
|
|
local has_tool_call=0
|
|
local tool_name=""
|
|
local tool_args=""
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--content)
|
|
content="$2"
|
|
shift 2
|
|
;;
|
|
--tool-call)
|
|
has_tool_call=1
|
|
tool_name="$2"
|
|
tool_args="$3"
|
|
shift 3
|
|
;;
|
|
--reasoning)
|
|
reasoning="$2"
|
|
shift 2
|
|
;;
|
|
--reasoning-content)
|
|
reasoning_content="$2"
|
|
shift 2
|
|
;;
|
|
esac
|
|
done
|
|
|
|
local msg='{"role":"assistant"}'
|
|
|
|
if [[ $has_tool_call -eq 1 ]]; then
|
|
tool_args=$(echo "$tool_args" | jq -R)
|
|
msg=$(echo "$msg" | jq \
|
|
--arg name "$tool_name" \
|
|
--argjson args "$tool_args" \
|
|
'. + {
|
|
"tool_calls": [{"index":0, "id": "call_1", "type": "function", "function": {"name": $name, "arguments": $args}}],
|
|
"content": null
|
|
}')
|
|
else
|
|
msg=$(echo "$msg" | jq \
|
|
--arg content "$content" \
|
|
--arg reasoning "$reasoning" \
|
|
--arg reasoning_content "$reasoning_content" \
|
|
'. + {"content": $content} | if $reasoning != "" then . + {"reasoning": $reasoning} else . end | if $reasoning_content != "" then . + {"reasoning_content": $reasoning_content} else . end')
|
|
fi
|
|
|
|
echo "$msg" | jq -c
|
|
}
|
|
|
|
function api_response() {
|
|
local message_json="$1"
|
|
local finish_reason="$2"
|
|
|
|
echo "{\"choices\":[{\"delta\":${message_json},\"finish_reason\":\"${finish_reason}\"}],\"usage\":{\"completion_tokens\":336,\"prompt_tokens\":14,\"total_tokens\":350}}"
|
|
}
|
|
|
|
function tool_call() {
|
|
local name="$1"
|
|
local args="$2"
|
|
local msg
|
|
msg=$(message --tool-call "$name" "$args")
|
|
api_response "$msg" "tool_calls"
|
|
}
|
|
|
|
function tool_call_raw() {
|
|
local name="$1"
|
|
local args="$2"
|
|
local msg
|
|
msg=$(message --tool-call "$name" "$args")
|
|
api_response "$msg" "tool_calls"
|
|
}
|
|
|
|
function be_done() {
|
|
local content="$1"
|
|
local msg
|
|
msg=$(message --content "$content")
|
|
api_response "$msg" "stop"
|
|
}
|
|
|
|
function reasoning_step() {
|
|
local content="$1"
|
|
local msg
|
|
msg=$(message --reasoning "$content")
|
|
api_response "$msg" "stop"
|
|
}
|
|
|
|
function reasoning_content_step() {
|
|
local content="$1"
|
|
local msg
|
|
msg=$(message --reasoning-content "$content")
|
|
api_response "$msg" "stop"
|
|
}
|
|
|
|
export -f check_tool_call
|
|
export -f message
|
|
export -f tool_call
|
|
export -f tool_call_raw
|
|
export -f api_response
|
|
export -f be_done
|
|
export -f reasoning_step
|
|
export -f reasoning_content_step
|
|
|
|
function history_length() {
|
|
local chat_file="$1"
|
|
jq '.history | length' "$chat_file"
|
|
}
|
|
export -f history_length
|
|
|
|
function dr_result() {
|
|
local chat_file="$1"
|
|
local entry="$2"
|
|
jq -r --arg e "$entry" '.history[$e | tonumber].content' "$chat_file" 2>/dev/null || echo ""
|
|
}
|
|
export -f dr_result
|
|
|
|
function dr_role() {
|
|
local chat_file="$1"
|
|
local entry="$2"
|
|
jq -r --arg e "$entry" '.history[$e | tonumber].role' "$chat_file" 2>/dev/null || echo ""
|
|
}
|
|
export -f dr_role
|
|
|
|
function dr_reasoning() {
|
|
local chat_file="$1"
|
|
local entry="$2"
|
|
jq -r --arg e "$entry" '.history[$e | tonumber].reasoning // ""' "$chat_file" 2>/dev/null || echo ""
|
|
}
|
|
export -f dr_reasoning
|
|
|
|
function dr_reasoning_content() {
|
|
local chat_file="$1"
|
|
local entry="$2"
|
|
jq -r --arg e "$entry" '.history[$e | tonumber].reasoning_content // ""' "$chat_file" 2>/dev/null || echo ""
|
|
}
|
|
export -f dr_reasoning_content
|
|
|
|
# Create a chat file with history messages
|
|
# Usage: create_chat_file <filename> <role> <content> [role content ...]
|
|
# Example: create_chat_file chat.json user "hello" assistant "hi there"
|
|
function create_chat_file() {
|
|
local file="$1"
|
|
shift
|
|
create_history_json "$@" | jq '{history: ., tools: []}' > "$file"
|
|
}
|
|
export -f create_chat_file
|
|
|
|
# Create a JSON array of history messages (for -T flag testing)
|
|
# Usage: create_history_json role content [role content ...]
|
|
# Example: create_history_json user "hello" assistant "hi there"
|
|
function create_history_json() {
|
|
local history="[]"
|
|
while [ $# -gt 0 ]; do
|
|
local role="$1"
|
|
local content="$2"
|
|
shift 2
|
|
history=$(echo "$history" | jq --arg r "$role" --arg c "$content" '. + [{"role": $r, "content": $c}]')
|
|
done
|
|
echo "$history"
|
|
}
|
|
export -f create_history_json
|
|
|
|
# Helper functions for rebase parser testing
|
|
|
|
# Get number of lines in rebase JSON output
|
|
function rebase_line_count() {
|
|
local output="$1"
|
|
echo "$output" | jq 'length'
|
|
}
|
|
export -f rebase_line_count
|
|
|
|
# Get command at position in rebase JSON output
|
|
function rebase_command_at() {
|
|
local output="$1"
|
|
local pos="$2"
|
|
echo "$output" | jq -r ".[$pos | tonumber].command"
|
|
}
|
|
export -f rebase_command_at
|
|
|
|
# Get index at position in rebase JSON output
|
|
function rebase_index_at() {
|
|
local output="$1"
|
|
local pos="$2"
|
|
echo "$output" | jq -r ".[$pos | tonumber].index"
|
|
}
|
|
export -f rebase_index_at
|
|
|
|
# Get role at position in rebase JSON output
|
|
function rebase_role_at() {
|
|
local output="$1"
|
|
local pos="$2"
|
|
echo "$output" | jq -r ".[$pos | tonumber].role"
|
|
}
|
|
export -f rebase_role_at
|
|
|
|
# Helper functions for rebase apply testing
|
|
|
|
# Create a rebase line JSON object
|
|
# Usage: create_rebase_line <command> <index> <role>
|
|
# Example: create_rebase_line pick 0 user
|
|
function create_rebase_line() {
|
|
local cmd="$1"
|
|
local idx="$2"
|
|
local role="$3"
|
|
jq -n --arg c "$cmd" --argjson i "$idx" --arg r "$role" '{"command": $c, "index": $i, "role": $r}'
|
|
}
|
|
export -f create_rebase_line
|
|
|
|
# Create a JSON array of rebase lines
|
|
# Usage: create_rebase_lines [command index role ...]
|
|
# Example: create_rebase_lines pick 0 user drop 1 assistant edit 2 tool
|
|
function create_rebase_lines() {
|
|
local lines="[]"
|
|
while [ $# -gt 0 ]; do
|
|
local cmd="$1"
|
|
local idx="$2"
|
|
local role="$3"
|
|
shift 3
|
|
local line
|
|
line=$(create_rebase_line "$cmd" "$idx" "$role")
|
|
lines=$(echo "$lines" | jq --argjson l "$line" '. + [$l]')
|
|
done
|
|
echo "$lines"
|
|
}
|
|
export -f create_rebase_lines
|
|
|
|
# Add a field to a message JSON object
|
|
function add_message_field() {
|
|
local msg="$1"
|
|
local field="$2"
|
|
local value="$3"
|
|
case "$field" in
|
|
reasoning)
|
|
echo "$msg" | jq --arg v "$value" '. + {reasoning: $v}'
|
|
;;
|
|
reasoning_content)
|
|
echo "$msg" | jq --arg v "$value" '. + {reasoning_content: $v}'
|
|
;;
|
|
tool_call_id)
|
|
echo "$msg" | jq --arg v "$value" '. + {tool_call_id: $v}'
|
|
;;
|
|
tool_call)
|
|
local call_id="$3"
|
|
local call_name="$4"
|
|
local call_args="$5"
|
|
echo "$msg" | jq --arg id "$call_id" --arg name "$call_name" --arg args "$call_args" \
|
|
'.tool_calls = ((.tool_calls // []) + [{"index": 0, "id": $id, "type": "function", "function": {"name": $name, "arguments": $args}}])'
|
|
;;
|
|
*)
|
|
echo "ERROR: Unknown field '$field'" >&2
|
|
return 1
|
|
;;
|
|
esac
|
|
}
|
|
export -f add_message_field
|
|
|
|
# Create a message JSON object with optional fields
|
|
# Usage: create_message_json <role> <content> [field value ...]
|
|
# Supported fields: reasoning, reasoning_content, tool_call_id, tool_call
|
|
# tool_call takes 3 args: tool_call <id> <name> <arguments>
|
|
# Example: create_message_json user "hello"
|
|
# Example: create_message_json user "hello" reasoning "thought" tool_call_id "call_123"
|
|
# Example: create_message_json assistant "" tool_call "call_1" "test" "{}"
|
|
function create_message_json() {
|
|
local role="$1"
|
|
local content="$2"
|
|
shift 2
|
|
|
|
local msg
|
|
msg=$(jq -n --arg r "$role" --arg c "$content" '{"role": $r, "content": $c}')
|
|
|
|
while [ $# -gt 0 ]; do
|
|
local field="$1"
|
|
if [ "$field" = "tool_call" ]; then
|
|
local call_id="$2"
|
|
local call_name="$3"
|
|
local call_args="$4"
|
|
shift 4
|
|
msg=$(add_message_field "$msg" "$field" "$call_id" "$call_name" "$call_args")
|
|
else
|
|
local value="$2"
|
|
shift 2
|
|
msg=$(add_message_field "$msg" "$field" "$value")
|
|
fi
|
|
done
|
|
echo "$msg"
|
|
}
|
|
export -f create_message_json
|
|
|
|
# Check if a string is a known message field name
|
|
function is_message_field() {
|
|
case "$1" in
|
|
reasoning|reasoning_content|tool_call_id|tool_call) return 0 ;;
|
|
*) return 1 ;;
|
|
esac
|
|
}
|
|
export -f is_message_field
|
|
|
|
# Create a JSON array of messages
|
|
# Usage: create_messages_json <role> <content> [field value ...] [role content ...]
|
|
# Each message: role content [field value ...]
|
|
# Supported fields: reasoning, reasoning_content, tool_call_id, tool_call
|
|
# tool_call takes 3 args: tool_call <id> <name> <arguments>
|
|
# Example: create_messages_json user "hello" assistant "hi there"
|
|
# Example: create_messages_json user "hello" reasoning "thought" assistant "hi"
|
|
function create_messages_json() {
|
|
local messages="[]"
|
|
|
|
while [ $# -gt 0 ]; do
|
|
local role="$1"
|
|
local content="$2"
|
|
shift 2
|
|
|
|
# Collect field-value pairs for this message into an array
|
|
local -a field_args=()
|
|
while [ $# -gt 0 ] && is_message_field "$1"; do
|
|
if [ "$1" = "tool_call" ]; then
|
|
field_args+=("$1" "$2" "$3" "$4")
|
|
shift 4
|
|
else
|
|
field_args+=("$1" "$2")
|
|
shift 2
|
|
fi
|
|
done
|
|
|
|
local msg
|
|
msg=$(create_message_json "$role" "$content" "${field_args[@]}")
|
|
messages=$(echo "$messages" | jq --argjson m "$msg" '. + [$m]')
|
|
done
|
|
echo "$messages"
|
|
}
|
|
export -f create_messages_json
|
|
|
|
function create_state() {
|
|
local file="$1"
|
|
shift
|
|
|
|
# Build history array from role/content pairs using jq for proper escaping
|
|
local history_json="[]"
|
|
while [ $# -gt 0 ]; do
|
|
local role="$1"
|
|
local content="$2"
|
|
shift 2
|
|
history_json=$(echo "$history_json" | jq --arg r "$role" --arg c "$content" '. + [{"role": $r, "content": $c}]')
|
|
done
|
|
|
|
# Generate the complete state file
|
|
echo "$history_json" | jq '{
|
|
"history": .,
|
|
"tools": [],
|
|
"config": {
|
|
"max_tool_iterations": 16,
|
|
"verbose": false,
|
|
"depth": 0
|
|
}
|
|
}' > "$file"
|
|
}
|
|
export -f create_state
|
|
|
|
# Get message role from chat history at given index
|
|
function dr_history_role() {
|
|
local chat_file="$1"
|
|
local index="$2"
|
|
jq -r --argjson i "$index" '.history[$i].role' "$chat_file"
|
|
}
|
|
export -f dr_history_role
|
|
|
|
# Get content type (string/array) from message at given history index
|
|
function dr_content_type() {
|
|
local chat_file="$1"
|
|
local index="$2"
|
|
jq -r --argjson i "$index" '.history[$i].content | type' "$chat_file"
|
|
}
|
|
export -f dr_content_type
|
|
|
|
# Get content part type at given history and content part indices
|
|
function dr_content_part_type() {
|
|
local chat_file="$1"
|
|
local history_idx="$2"
|
|
local part_idx="$3"
|
|
jq -r --argjson h "$history_idx" --argjson p "$part_idx" '.history[$h].content[$p].type' "$chat_file"
|
|
}
|
|
export -f dr_content_part_type
|
|
|
|
# Get text content from a content part at given history and content part indices
|
|
function dr_content_part_text() {
|
|
local chat_file="$1"
|
|
local history_idx="$2"
|
|
local part_idx="$3"
|
|
jq -r --argjson h "$history_idx" --argjson p "$part_idx" '.history[$h].content[$p].text' "$chat_file"
|
|
}
|
|
export -f dr_content_part_text
|
|
|
|
# Get image URL from a content part at given history and content part indices
|
|
function dr_content_part_image_url() {
|
|
local chat_file="$1"
|
|
local history_idx="$2"
|
|
local part_idx="$3"
|
|
jq -r --argjson h "$history_idx" --argjson p "$part_idx" '.history[$h].content[$p].image_url.url' "$chat_file"
|
|
}
|
|
export -f dr_content_part_image_url
|
|
|
|
# Get image detail from a content part at given history and content part indices
|
|
function dr_content_part_image_detail() {
|
|
local chat_file="$1"
|
|
local history_idx="$2"
|
|
local part_idx="$3"
|
|
jq -r --argjson h "$history_idx" --argjson p "$part_idx" '.history[$h].content[$p].image_url.detail' "$chat_file"
|
|
}
|
|
export -f dr_content_part_image_detail
|
|
|
|
# Get content parts count from message at given history index
|
|
function dr_content_parts_count() {
|
|
local chat_file="$1"
|
|
local index="$2"
|
|
jq --argjson i "$index" '.history[$i].content | length' "$chat_file"
|
|
}
|
|
export -f dr_content_parts_count
|
|
|
|
# Get content from message at given history index (for text-only messages)
|
|
function dr_content_text() {
|
|
local chat_file="$1"
|
|
local index="$2"
|
|
jq -r --argjson i "$index" '.history[$i].content' "$chat_file"
|
|
}
|
|
export -f dr_content_text
|
|
|
|
# Get base_url from config in chat file
|
|
function dr_config_base_url() {
|
|
local chat_file="$1"
|
|
jq -r '.config.base_url' "$chat_file"
|
|
}
|
|
export -f dr_config_base_url
|
|
|
|
# Get model from config in chat file
|
|
function dr_config_model() {
|
|
local chat_file="$1"
|
|
jq -r '.config.model' "$chat_file"
|
|
}
|
|
export -f dr_config_model
|
|
|
|
# Resolve paths relative to this script
|
|
SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
|
|
PROJECT_ROOT=$(cd "${SCRIPT_DIR}/.." && pwd)
|
|
DRHARNESS="${PWD}/drharness"
|
|
MOCK_EDITOR="${PROJECT_ROOT}/tests/mock_editor.sh"
|
|
|
|
# Validate drharness binary exists
|
|
if [ ! -f "${DRHARNESS}" ]; then
|
|
echo "ERROR: drharness not found at ${DRHARNESS}" >&2
|
|
echo "Run 'cmake --build build' first." >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Validate arguments
|
|
if [ $# -lt 1 ]; then
|
|
echo "Usage: run_test.sh <test_script.sh>" >&2
|
|
exit 1
|
|
fi
|
|
|
|
TEST_SCRIPT="$1"
|
|
TEST_TEMP_DIR=$(mktemp -d /tmp/drharness_test_XXXXXX)
|
|
|
|
# Export variables for the test script
|
|
export DRHARNESS
|
|
export TEST_TEMP_DIR
|
|
export MOCK_EDITOR
|
|
|
|
# Cleanup function
|
|
cleanup() {
|
|
rm -rf "${TEST_TEMP_DIR}"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
# Change to isolation directory and run the test
|
|
cd "${TEST_TEMP_DIR}"
|
|
bash -xeu "${TEST_SCRIPT}"
|
|
exit $?
|