drharness/tests/test_multimodal_message.sh

#!/bin/bash
# test_multimodal_message.sh - Tests for multimodal message construction with -i flag

# Create a minimal 1x1 PNG file for testing
printf '\x89\x50\x4E\x47\x0D\x0A\x1A\x0A\x00\x00\x00\x0D\x49\x48\x44\x52\x00\x00\x00\x01\x00\x00\x00\x01\x08\x02\x00\x00\x00\x90\x77\x53\xDE\x00\x00\x00\x0C\x49\x44\x41\x54\x08\xD7\x63\xF8\xCF\xC0\x00\x00\x03\x01\x01\x00\x18\xDD\x8D\xB4\x00\x00\x00\x00\x49\x45\x4E\x44\xAE\x42\x60\x82' > "${TEST_TEMP_DIR}/test.png"

# Test 1: Verify multimodal message format in chat.json with -i and -d flags
echo "$(be_done "Response")" | "${DRHARNESS}" \
    -F "${TEST_TEMP_DIR}/chat.json" \
    -n \
    -d \
    -i "${TEST_TEMP_DIR}/test.png" \
    -m "describe this image" 2>/dev/null

# Verify the first message (index 0) is a user message with multimodal content
ROLE=$(dr_history_role "${TEST_TEMP_DIR}/chat.json" 0)
if [ "$ROLE" != "user" ]; then
    echo "FAIL: Expected first message role to be 'user', got '${ROLE}'"
    exit 1
fi

# Verify content is an array (multimodal format)
CONTENT_TYPE=$(dr_content_type "${TEST_TEMP_DIR}/chat.json" 0)
if [ "$CONTENT_TYPE" != "array" ]; then
    echo "FAIL: Expected content to be an array for multimodal message, got '${CONTENT_TYPE}'"
    exit 1
fi

# Verify first content part is text type
TEXT_TYPE=$(dr_content_part_type "${TEST_TEMP_DIR}/chat.json" 0 0)
if [ "$TEXT_TYPE" != "text" ]; then
    echo "FAIL: Expected first content part type to be 'text', got '${TEXT_TYPE}'"
    exit 1
fi

# Verify text content matches the prompt
TEXT_CONTENT=$(dr_content_part_text "${TEST_TEMP_DIR}/chat.json" 0 0)
if [ "$TEXT_CONTENT" != "describe this image" ]; then
    echo "FAIL: Expected text content to be 'describe this image', got '${TEXT_CONTENT}'"
    exit 1
fi

# Verify second content part is image_url type
IMAGE_TYPE=$(dr_content_part_type "${TEST_TEMP_DIR}/chat.json" 0 1)
if [ "$IMAGE_TYPE" != "image_url" ]; then
    echo "FAIL: Expected second content part type to be 'image_url', got '${IMAGE_TYPE}'"
    exit 1
fi

# Verify image_url has a valid data URI
IMAGE_URL=$(dr_content_part_image_url "${TEST_TEMP_DIR}/chat.json" 0 1)
if ! echo "$IMAGE_URL" | grep -q "^data:image/png;base64,"; then
    echo "FAIL: Expected image URL to be a PNG data URI, got '${IMAGE_URL}'"
    exit 1
fi

# Verify image detail is "auto"
IMAGE_DETAIL=$(dr_content_part_image_detail "${TEST_TEMP_DIR}/chat.json" 0 1)
if [ "$IMAGE_DETAIL" != "auto" ]; then
    echo "FAIL: Expected image detail to be 'auto', got '${IMAGE_DETAIL}'"
    exit 1
fi

# Test 2: Verify multiple images are included
printf '\x89\x50\x4E\x47\x0D\x0A\x1A\x0A\x00\x00\x00\x0D\x49\x48\x44\x52\x00\x00\x00\x01\x00\x00\x00\x01\x08\x02\x00\x00\x00\x90\x77\x53\xDE\x00\x00\x00\x0C\x49\x44\x41\x54\x08\xD7\x63\xF8\xCF\xC0\x00\x00\x03\x01\x01\x00\x18\xDD\x8D\xB4\x00\x00\x00\x00\x49\x45\x4E\x44\xAE\x42\x60\x82' > "${TEST_TEMP_DIR}/test2.png"

echo "$(be_done "Response 2")" | "${DRHARNESS}" \
    -F "${TEST_TEMP_DIR}/chat2.json" \
    -n \
    -d \
    -i "${TEST_TEMP_DIR}/test.png" \
    -i "${TEST_TEMP_DIR}/test2.png" \
    -m "compare images" 2>/dev/null

# Verify content array has 3 parts (1 text + 2 images)
CONTENT_LENGTH=$(dr_content_parts_count "${TEST_TEMP_DIR}/chat2.json" 0)
if [ "$CONTENT_LENGTH" != "3" ]; then
    echo "FAIL: Expected 3 content parts (1 text + 2 images), got '${CONTENT_LENGTH}'"
    exit 1
fi

# Verify third content part is also image_url
THIRD_TYPE=$(dr_content_part_type "${TEST_TEMP_DIR}/chat2.json" 0 2)
if [ "$THIRD_TYPE" != "image_url" ]; then
    echo "FAIL: Expected third content part type to be 'image_url', got '${THIRD_TYPE}'"
    exit 1
fi

# Test 3: Verify backward compatibility - text-only message when no images
echo "$(be_done "Response 3")" | "${DRHARNESS}" \
    -F "${TEST_TEMP_DIR}/chat3.json" \
    -n \
    -d \
    -m "text only prompt" 2>/dev/null

# Verify content is a string (not an array) when no images
BACKWARD_TYPE=$(dr_content_type "${TEST_TEMP_DIR}/chat3.json" 0)
if [ "$BACKWARD_TYPE" != "string" ]; then
    echo "FAIL: Expected content to be a string for text-only message, got '${BACKWARD_TYPE}'"
    exit 1
fi

BACKWARD_CONTENT=$(dr_content_text "${TEST_TEMP_DIR}/chat3.json" 0)
if [ "$BACKWARD_CONTENT" != "text only prompt" ]; then
    echo "FAIL: Expected text content to be 'text only prompt', got '${BACKWARD_CONTENT}'"
    exit 1
fi

# Test 4: Verify -i works with -t tool filter flag
echo "$(be_done "Response with tool filter")" | "${DRHARNESS}" \
    -F "${TEST_TEMP_DIR}/chat4.json" \
    -n \
    -d \
    -t read_file \
    -i "${TEST_TEMP_DIR}/test.png" \
    -m "read and describe" 2>/dev/null

# Verify multimodal format still works with tool filter
TOOL_FILTER_TYPE=$(dr_content_type "${TEST_TEMP_DIR}/chat4.json" 0)
if [ "$TOOL_FILTER_TYPE" != "array" ]; then
    echo "FAIL: Expected multimodal content array with -t flag, got '${TOOL_FILTER_TYPE}'"
    exit 1
fi

# Test 5: Verify -i works with -M model override flag
echo "$(be_done "Response with model override")" | "${DRHARNESS}" \
    -F "${TEST_TEMP_DIR}/chat5.json" \
    -n \
    -d \
    -M "test-model" \
    -i "${TEST_TEMP_DIR}/test.png" \
    -m "test model override" 2>/dev/null

# Verify multimodal format still works with model override
MODEL_OVERRIDE_TYPE=$(dr_content_type "${TEST_TEMP_DIR}/chat5.json" 0)
if [ "$MODEL_OVERRIDE_TYPE" != "array" ]; then
    echo "FAIL: Expected multimodal content array with -M flag, got '${MODEL_OVERRIDE_TYPE}'"
    exit 1
fi

echo "All multimodal message tests passed"