Adding the ability to handle non utf8 encodings

CMakeLists.txt

@@ -25,6 +25,7 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Windows")
 endif()
 
 add_library(drharness_lib
+    src/encoding.cpp
     src/tool_registry.cpp
     src/ollama_client.cpp
     src/harness.cpp

include/encoding.hpp

@@ -0,0 +1,5 @@
+#pragma once
+#include <string>
+
+char const* validate_utf8(char const* s);
+std::string ensure_utf8(char const* s);

src/encoding.cpp

@@ -0,0 +1,102 @@
+#include "encoding.hpp"
+
+char const* validate_utf8(char const* s)
+{
+    auto b = reinterpret_cast<unsigned char const*>(s);
+
+    unsigned char c0 = b[0];
+
+    // ASCII
+    if (c0 < 0x80)
+        return s + 1;
+
+    // Continuation byte or invalid lead byte
+    if (c0 < 0xC2)
+        return nullptr;
+
+    // 2-byte sequence
+    if (c0 <= 0xDF)
+    {
+        unsigned char c1 = b[1];
+        if (c1 == '\0' || (c1 & 0xC0) != 0x80)
+            return nullptr;
+        return s + 2;
+    }
+
+    // 3-byte sequence
+    if (c0 <= 0xEF)
+    {
+        unsigned char c1 = b[1];
+        unsigned char c2 = b[2];
+
+        if (c1 == '\0' || c2 == '\0')
+            return nullptr;
+
+        if ((c1 & 0xC0) != 0x80 || (c2 & 0xC0) != 0x80)
+            return nullptr;
+
+        // Overlong
+        if (c0 == 0xE0 && c1 < 0xA0)
+            return nullptr;
+
+        // Surrogates
+        if (c0 == 0xED && c1 >= 0xA0)
+            return nullptr;
+
+        return s + 3;
+    }
+
+    // 4-byte sequence
+    if (c0 <= 0xF4)
+    {
+        unsigned char c1 = b[1];
+        unsigned char c2 = b[2];
+        unsigned char c3 = b[3];
+
+        if (c1 == '\0' || c2 == '\0' || c3 == '\0')
+            return nullptr;
+
+        if ((c1 & 0xC0) != 0x80 ||
+            (c2 & 0xC0) != 0x80 ||
+            (c3 & 0xC0) != 0x80)
+            return nullptr;
+
+        // Overlong
+        if (c0 == 0xF0 && c1 < 0x90)
+            return nullptr;
+
+        // > U+10FFFF
+        if (c0 == 0xF4 && c1 >= 0x90)
+            return nullptr;
+
+        return s + 4;
+    }
+
+    return nullptr;
+}
+
+std::string ensure_utf8(char const* s)
+{
+    std::string ret = "";
+
+    while (1)
+    {
+        char const* next = validate_utf8(s);
+        if (next == 0)
+        {
+            ret += (0xc2+(((unsigned char)*s)>0xbf));
+            ret += ((((unsigned char)*s)&0x3f))+0x80;
+            s++;
+            continue;
+        }
+
+        ret += std::string(s, next);
+
+        s = next;
+
+        if (!*s)
+            break;
+    }
+
+    return ret;
+}

src/tool_registry.cpp

@@ -1,4 +1,5 @@
 #include "tool_registry.hpp"
+#include "encoding.hpp"
 
 std::expected<void, std::string> ToolRegistry::register_tool(Tool tool) {
     if (tools_.count(tool.name))
@@ -24,12 +25,37 @@ std::vector<Tool> ToolRegistry::tools() const {
     return ret;
 }
 
+
+// helper type for the visitor
+template<class... Ts>
+struct overloads : Ts... { using Ts::operator()...; };
+
 Content ToolRegistry::invoke(const std::string& name,
                              const nlohmann::json& args) const {
     auto it = tools_.find(name);
     if (it == tools_.end())
         return "Error: unknown tool: " + name;
-    return it->second.execute(args);
+    auto ret = it->second.execute(args);
+
+    ret = std::visit(overloads{
+            [](std::string const& x) -> Content { return ensure_utf8(x.c_str()); },
+            [](std::vector<ContentPart> const& x) -> Content {
+                std::vector<ContentPart> ret = x;
+
+                for (auto& i : ret)
+                {
+                    if (i.text)
+                    {
+                        std::string valid = ensure_utf8(i.text->c_str());
+                        i.text = valid;
+                    }
+                }
+
+                return ret;
+            },
+    }, ret);
+
+    return ret;
 }
 
 std::expected<ToolRegistry, std::string> ToolRegistry::create_subregistry(std::vector<std::string> const& names) const