Merge pull request 'bugfix/utf8' (#38) from bugfix/utf8 into master

Reviewed-on: https://192.168.8.63/git/darthrubik/drharness/pulls/38

CMakeLists.txt

@@ -25,6 +25,7 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Windows")
 endif()
 
 add_library(drharness_lib
+    src/encoding.cpp
     src/tool_registry.cpp
     src/ollama_client.cpp
     src/harness.cpp

include/encoding.hpp

@@ -0,0 +1,5 @@
+#pragma once
+#include <string>
+
+char const* validate_utf8(char const* s);
+std::string ensure_utf8(char const* s);

src/encoding.cpp

@@ -0,0 +1,99 @@
+#include "encoding.hpp"
+
+char const* validate_utf8(char const* s)
+{
+    auto b = reinterpret_cast<unsigned char const*>(s);
+
+    unsigned char c0 = b[0];
+
+    // ASCII
+    if (c0 < 0x80)
+        return s + 1;
+
+    // Continuation byte or invalid lead byte
+    if (c0 < 0xC2)
+        return nullptr;
+
+    // 2-byte sequence
+    if (c0 <= 0xDF)
+    {
+        unsigned char c1 = b[1];
+        if (c1 == '\0' || (c1 & 0xC0) != 0x80)
+            return nullptr;
+        return s + 2;
+    }
+
+    // 3-byte sequence
+    if (c0 <= 0xEF)
+    {
+        unsigned char c1 = b[1];
+        unsigned char c2 = b[2];
+
+        if (c1 == '\0' || c2 == '\0')
+            return nullptr;
+
+        if ((c1 & 0xC0) != 0x80 || (c2 & 0xC0) != 0x80)
+            return nullptr;
+
+        // Overlong
+        if (c0 == 0xE0 && c1 < 0xA0)
+            return nullptr;
+
+        // Surrogates
+        if (c0 == 0xED && c1 >= 0xA0)
+            return nullptr;
+
+        return s + 3;
+    }
+
+    // 4-byte sequence
+    if (c0 <= 0xF4)
+    {
+        unsigned char c1 = b[1];
+        unsigned char c2 = b[2];
+        unsigned char c3 = b[3];
+
+        if (c1 == '\0' || c2 == '\0' || c3 == '\0')
+            return nullptr;
+
+        if ((c1 & 0xC0) != 0x80 ||
+            (c2 & 0xC0) != 0x80 ||
+            (c3 & 0xC0) != 0x80)
+            return nullptr;
+
+        // Overlong
+        if (c0 == 0xF0 && c1 < 0x90)
+            return nullptr;
+
+        // > U+10FFFF
+        if (c0 == 0xF4 && c1 >= 0x90)
+            return nullptr;
+
+        return s + 4;
+    }
+
+    return nullptr;
+}
+
+std::string ensure_utf8(char const* s)
+{
+    std::string ret = "";
+
+    while (*s)
+    {
+        char const* next = validate_utf8(s);
+        if (next == 0)
+        {
+            ret += (0xc2+(((unsigned char)*s)>0xbf));
+            ret += ((((unsigned char)*s)&0x3f))+0x80;
+            s++;
+            continue;
+        }
+
+        ret += std::string(s, next);
+
+        s = next;
+    }
+
+    return ret;
+}

src/tool_registry.cpp

@@ -1,4 +1,5 @@
 #include "tool_registry.hpp"
+#include "encoding.hpp"
 
 std::expected<void, std::string> ToolRegistry::register_tool(Tool tool) {
     if (tools_.count(tool.name))
@@ -24,12 +25,37 @@ std::vector<Tool> ToolRegistry::tools() const {
     return ret;
 }
 
+
+// helper type for the visitor
+template<class... Ts>
+struct overloads : Ts... { using Ts::operator()...; };
+
 Content ToolRegistry::invoke(const std::string& name,
                              const nlohmann::json& args) const {
     auto it = tools_.find(name);
     if (it == tools_.end())
         return "Error: unknown tool: " + name;
-    return it->second.execute(args);
+    auto ret = it->second.execute(args);
+
+    ret = std::visit(overloads{
+            [](std::string const& x) -> Content { return ensure_utf8(x.c_str()); },
+            [](std::vector<ContentPart> const& x) -> Content {
+                std::vector<ContentPart> ret = x;
+
+                for (auto& i : ret)
+                {
+                    if (i.text)
+                    {
+                        std::string valid = ensure_utf8(i.text->c_str());
+                        i.text = valid;
+                    }
+                }
+
+                return ret;
+            },
+    }, ret);
+
+    return ret;
 }
 
 std::expected<ToolRegistry, std::string> ToolRegistry::create_subregistry(std::vector<std::string> const& names) const

tests/CMakeLists.txt

@@ -72,3 +72,6 @@ add_dr_test(test_multimodal_roundtrip)
 
 # Add print_history multimodal tests
 add_dr_test(test_print_history_multimodal)
+
+# Add encoding tests
+add_dr_test(test_encoding)

tests/test_encoding.sh

@@ -0,0 +1,130 @@
+#!/bin/bash
+# test_encoding.sh - Tests for UTF-8 encoding validation and conversion
+# All tests exercise encoding through the read_file tool
+
+# Test 1: Valid ASCII passes through unchanged
+printf 'Hello' > test_ascii.txt
+tool_call read_file '{"path":"test_ascii.txt"}' >> mock1.txt
+be_done 'done' >> mock1.txt
+"${DRHARNESS}" -F chat1.json -n -m "test" -t read_file -d < "mock1.txt" >/dev/null 2>&1
+RESULT=$(dr_result chat1.json 2)
+if [ "${RESULT}" != "Hello" ]; then
+    echo "FAIL: ASCII passthrough - expected 'Hello', got '${RESULT}'"
+    exit 1
+fi
+
+# Test 2: Valid UTF-8 2-byte sequence (é = c3a9) passes through unchanged
+printf '\xc3\xa9' > test_utf8_2byte.txt
+tool_call read_file '{"path":"test_utf8_2byte.txt"}' >> mock2.txt
+be_done 'done' >> mock2.txt
+"${DRHARNESS}" -F chat2.json -n -m "test" -t read_file -d < "mock2.txt" >/dev/null 2>&1
+RESULT=$(dr_result chat2.json 2)
+RESULT_HEX=$(od -An -tx1 -v <<< "${RESULT}" | tr -d ' \n' | sed 's/0a$//')
+if [ "${RESULT_HEX}" != "c3a9" ]; then
+    echo "FAIL: valid UTF-8 2-byte - expected 'c3a9', got '${RESULT_HEX}'"
+    exit 1
+fi
+
+# Test 3: Valid UTF-8 3-byte sequence (中 = e4b8ad) passes through unchanged
+printf '\xe4\xb8\xad' > test_utf8_3byte.txt
+tool_call read_file '{"path":"test_utf8_3byte.txt"}' >> mock3.txt
+be_done 'done' >> mock3.txt
+"${DRHARNESS}" -F chat3.json -n -m "test" -t read_file -d < "mock3.txt" >/dev/null 2>&1
+RESULT=$(dr_result chat3.json 2)
+RESULT_HEX=$(od -An -tx1 -v <<< "${RESULT}" | tr -d ' \n' | sed 's/0a$//')
+if [ "${RESULT_HEX}" != "e4b8ad" ]; then
+    echo "FAIL: valid UTF-8 3-byte - expected 'e4b8ad', got '${RESULT_HEX}'"
+    exit 1
+fi
+
+# Test 4: Valid UTF-8 4-byte sequence (😀 = f09f9880) passes through unchanged
+printf '\xf0\x9f\x98\x80' > test_utf8_4byte.txt
+tool_call read_file '{"path":"test_utf8_4byte.txt"}' >> mock4.txt
+be_done 'done' >> mock4.txt
+"${DRHARNESS}" -F chat4.json -n -m "test" -t read_file -d < "mock4.txt" >/dev/null 2>&1
+RESULT=$(dr_result chat4.json 2)
+RESULT_HEX=$(od -An -tx1 -v <<< "${RESULT}" | tr -d ' \n' | sed 's/0a$//')
+if [ "${RESULT_HEX}" != "f09f9880" ]; then
+    echo "FAIL: valid UTF-8 4-byte - expected 'f09f9880', got '${RESULT_HEX}'"
+    exit 1
+fi
+
+# Test 5: Latin-1 byte 0xe9 (é in Latin-1) should be converted to UTF-8 c3a9
+printf '\xe9' > test_latin1_e9.txt
+tool_call read_file '{"path":"test_latin1_e9.txt"}' >> mock5.txt
+be_done 'done' >> mock5.txt
+"${DRHARNESS}" -F chat5.json -n -m "test" -t read_file -d < "mock5.txt" >/dev/null 2>&1
+RESULT=$(dr_result chat5.json 2)
+RESULT_HEX=$(od -An -tx1 -v <<< "${RESULT}" | tr -d ' \n' | sed 's/0a$//')
+if [ "${RESULT_HEX}" != "c3a9" ]; then
+    echo "FAIL: Latin-1 e9 to UTF-8 - expected 'c3a9', got '${RESULT_HEX}'"
+    exit 1
+fi
+
+# Test 6: Multiple Latin-1 bytes (café in Latin-1 = 636166e9)
+printf '\x63\x61\x66\xe9' > test_latin1_cafe.txt
+tool_call read_file '{"path":"test_latin1_cafe.txt"}' >> mock6.txt
+be_done 'done' >> mock6.txt
+"${DRHARNESS}" -F chat6.json -n -m "test" -t read_file -d < "mock6.txt" >/dev/null 2>&1
+RESULT=$(dr_result chat6.json 2)
+RESULT_HEX=$(od -An -tx1 -v <<< "${RESULT}" | tr -d ' \n' | sed 's/0a$//')
+if [ "${RESULT_HEX}" != "636166c3a9" ]; then
+    echo "FAIL: Latin-1 café to UTF-8 - expected '636166c3a9', got '${RESULT_HEX}'"
+    exit 1
+fi
+
+# Test 7: Byte 0x80 (isolated continuation byte) should be converted to c280
+printf '\x80' > test_cont_byte.txt
+tool_call read_file '{"path":"test_cont_byte.txt"}' >> mock7.txt
+be_done 'done' >> mock7.txt
+"${DRHARNESS}" -F chat7.json -n -m "test" -t read_file -d < "mock7.txt" >/dev/null 2>&1
+RESULT=$(dr_result chat7.json 2)
+RESULT_HEX=$(od -An -tx1 -v <<< "${RESULT}" | tr -d ' \n' | sed 's/0a$//')
+if [ "${RESULT_HEX}" != "c280" ]; then
+    echo "FAIL: continuation byte 80 - expected 'c280', got '${RESULT_HEX}'"
+    exit 1
+fi
+
+# Test 8: Mixed valid UTF-8 and invalid bytes
+# "A" + invalid 0xe9 + "B" = 41e942 -> expected: 41c3a942
+printf '\x41\xe9\x42' > test_mixed.txt
+tool_call read_file '{"path":"test_mixed.txt"}' >> mock8.txt
+be_done 'done' >> mock8.txt
+"${DRHARNESS}" -F chat8.json -n -m "test" -t read_file -d < "mock8.txt" >/dev/null 2>&1
+RESULT=$(dr_result chat8.json 2)
+RESULT_HEX=$(od -An -tx1 -v <<< "${RESULT}" | tr -d ' \n' | sed 's/0a$//')
+if [ "${RESULT_HEX}" != "41c3a942" ]; then
+    echo "FAIL: mixed valid/invalid - expected '41c3a942', got '${RESULT_HEX}'"
+    exit 1
+fi
+
+# Test 9: Empty file returns "(empty file)"
+printf '' > test_empty.txt
+tool_call read_file '{"path":"test_empty.txt"}' >> mock9.txt
+be_done 'done' >> mock9.txt
+"${DRHARNESS}" -F chat9.json -n -m "test" -t read_file -d < "mock9.txt" >/dev/null 2>&1
+RESULT=$(dr_result chat9.json 2)
+if [ "${RESULT}" != "(empty file)" ]; then
+    echo "FAIL: empty file - expected '(empty file)', got '${RESULT}'"
+    exit 1
+fi
+
+# Test 10: File with newline and non-UTF8 content
+printf 'line1\n\xe9\nline3\n' > test_multiline.txt
+tool_call read_file '{"path":"test_multiline.txt"}' >> mock10.txt
+be_done 'done' >> mock10.txt
+"${DRHARNESS}" -F chat10.json -n -m "test" -t read_file -d < "mock10.txt" >/dev/null 2>&1
+RESULT=$(dr_result chat10.json 2)
+# Check that the file content was properly converted
+if ! echo "${RESULT}" | grep -q "line1"; then
+    echo "FAIL: multiline - missing 'line1' in result"
+    exit 1
+fi
+# Verify the é is properly UTF-8 encoded
+RESULT_HEX=$(od -An -tx1 -v <<< "${RESULT}" | tr -d ' \n' | sed 's/0a$//')
+if ! echo "${RESULT_HEX}" | grep -q "c3a9"; then
+    echo "FAIL: multiline - expected UTF-8 é (c3a9) in result, got '${RESULT_HEX}'"
+    exit 1
+fi
+
+echo "PASS: encoding tests"