Adding read multi-modal tool

CMakeLists.txt

@@ -12,6 +12,7 @@ set(TOOL_SOURCES
     tools/shell_tool.cpp
     tools/agent_tool.cpp
     tools/edit_tool.cpp
+    tools/read_multi_modal.cpp
     tools/file_list_tool.cpp
     tools/file_read_tool.cpp
     tools/grep_tool.cpp

include/tool.hpp

@@ -11,7 +11,7 @@ template<typename ParamsType, typename F>
 std::function<Content(const nlohmann::json&)> create_tool_execute(
     F const& fn
 ) {
-    return [fn = std::move(fn)](const nlohmann::json& json) -> std::string {
+    return [fn = std::move(fn)](const nlohmann::json& json) -> Content {
         auto result = json_to<ParamsType>(json);
         if (!result) {
             return "Error: " + result.error();

src/main.cpp

@@ -11,6 +11,7 @@
 #endif
 #include "agent_tool.hpp"
 #include "edit_tool.hpp"
+#include "read_multi_modal.hpp"
 #include "file_list_tool.hpp"
 #include "file_read_tool.hpp"
 #include "grep_tool.hpp"
@@ -100,6 +101,7 @@ int main(int argc, char** argv) {
     ToolRegistry master_registery;
     ToolRegistry agent_tools;
     expect_or_exit(master_registery.register_tool(make_edit_tool()));
+    expect_or_exit(master_registery.register_tool(make_read_multi_modal()));
     expect_or_exit(master_registery.register_tool(make_file_list_tool()));
     expect_or_exit(master_registery.register_tool(make_file_read_tool()));
     expect_or_exit(master_registery.register_tool(make_grep_tool()));

tools/read_multi_modal.cpp

@@ -0,0 +1,41 @@
+#include "tool.hpp"
+#include "read_multi_modal.hpp"
+#include "json_types.hpp"
+#include "base64_util.hpp"
+
+
+
+Tool make_read_multi_modal() {
+    return Tool{
+        .name = "read_multi_modal",
+        .description = ""
+    "Reads multi modal files (images, audio, etc)\n",
+
+        .parameters_schema = {
+            {"type", "object"},
+            {"properties", {
+                {"path", {
+                    {"type", "string"},
+                    {"description", "Absolute or relative path to multi modal file to read"}
+                }},
+            }},
+            {"required", {"filePath"}}
+        },
+        .execute = create_tool_execute<ReadFileToolParams>([](const ReadFileToolParams& params) -> Content {
+            std::vector<ContentPart> ret{};
+
+            ContentPart image_part;
+            image_part.type = "image_url";
+            ImageUrl img_url;
+            auto uri = read_image_as_data_uri(params.path);
+            if (!uri)
+                return "Failed to read image " + uri.error();
+
+            img_url.url = *uri;
+            image_part.image_url = img_url;
+            ret.push_back(image_part);
+
+            return ret;
+        })
+    };
+}

tools/read_multi_modal.hpp

@@ -0,0 +1,4 @@
+#pragma once
+#include "tool.hpp"
+
+Tool make_read_multi_modal();