Implement issue 159: Add new overload to Harness::chat for multimodal input
Add a new overload to Harness::chat that takes a std::string as the user message and a vector<std::string> const& as image paths. The method reads the image files and converts them to data URIs internally, then constructs the multimodal message. This moves all the multimodal message construction logic from main.cpp into Harness, simplifying main.cpp.
This commit is contained in:
@@ -16,6 +16,7 @@ public:
|
||||
|
||||
std::expected<std::string, std::string> chat(const std::string& user_message);
|
||||
std::expected<std::string, std::string> chat(const Message& user_message);
|
||||
std::expected<std::string, std::string> chat(const std::string& user_message, const std::vector<std::string>& image_paths);
|
||||
void reset();
|
||||
|
||||
HarnessState serialize() const;
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
#include "harness.hpp"
|
||||
#include "json_types.hpp"
|
||||
#include "base64_util.hpp"
|
||||
#include <iostream>
|
||||
|
||||
Harness::Harness(std::unique_ptr<Client> client, const ToolRegistry& registry,
|
||||
@@ -46,6 +47,35 @@ std::expected<std::string, std::string> Harness::chat(const std::string& user_me
|
||||
return chat(message);
|
||||
}
|
||||
|
||||
std::expected<std::string, std::string> Harness::chat(const std::string& user_message, const std::vector<std::string>& image_paths) {
|
||||
Message message{};
|
||||
message.role = "user";
|
||||
std::vector<ContentPart> content_parts;
|
||||
|
||||
// Add text content part
|
||||
ContentPart text_part;
|
||||
text_part.type = "text";
|
||||
text_part.text = user_message;
|
||||
content_parts.push_back(text_part);
|
||||
|
||||
// Read image files and add image content parts
|
||||
for (const auto& path : image_paths) {
|
||||
auto result = read_image_as_data_uri(path);
|
||||
if (!result) {
|
||||
return std::unexpected("failed to read image '" + path + "': " + result.error());
|
||||
}
|
||||
ContentPart image_part;
|
||||
image_part.type = "image_url";
|
||||
ImageUrl img_url;
|
||||
img_url.url = result.value();
|
||||
image_part.image_url = img_url;
|
||||
content_parts.push_back(image_part);
|
||||
}
|
||||
|
||||
message.content = content_parts;
|
||||
return chat(message);
|
||||
}
|
||||
|
||||
std::expected<std::string, std::string> Harness::chat(const Message& user_message) {
|
||||
push_message(user_message);
|
||||
|
||||
|
||||
+2
-35
@@ -337,17 +337,6 @@ int main(int argc, char** argv) {
|
||||
|
||||
expect_or_exit(master_registery.register_tool(make_skill_tool(enabled_skills)));
|
||||
|
||||
// Validate and read image files
|
||||
std::vector<std::string> image_data_uris;
|
||||
for (const auto& path : image_paths) {
|
||||
auto result = read_image_as_data_uri(path);
|
||||
if (!result) {
|
||||
std::cerr << "Error: " << result.error() << std::endl;
|
||||
return 1;
|
||||
}
|
||||
image_data_uris.push_back(result.value());
|
||||
}
|
||||
|
||||
if (debug_paths) {
|
||||
print_discovered_paths(discovered_paths);
|
||||
return 0;
|
||||
@@ -559,30 +548,8 @@ int main(int argc, char** argv) {
|
||||
|
||||
std::expected<std::string, std::string> reply;
|
||||
|
||||
if (!image_data_uris.empty()) {
|
||||
// Construct multimodal message with text and image content parts
|
||||
Message message{};
|
||||
message.role = "user";
|
||||
std::vector<ContentPart> content_parts;
|
||||
|
||||
// Add text content part
|
||||
ContentPart text_part;
|
||||
text_part.type = "text";
|
||||
text_part.text = final_prompt;
|
||||
content_parts.push_back(text_part);
|
||||
|
||||
// Add image content parts
|
||||
for (const auto& data_uri : image_data_uris) {
|
||||
ContentPart image_part;
|
||||
image_part.type = "image_url";
|
||||
ImageUrl img_url;
|
||||
img_url.url = data_uri;
|
||||
image_part.image_url = img_url;
|
||||
content_parts.push_back(image_part);
|
||||
}
|
||||
|
||||
message.content = content_parts;
|
||||
reply = harness.chat(message);
|
||||
if (!image_paths.empty()) {
|
||||
reply = harness.chat(final_prompt, image_paths);
|
||||
} else {
|
||||
reply = harness.chat(final_prompt);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user