darthrubik/llama-cpp
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

chat : refactor API (#30210)

Browse Source
This commit is contained in:
Aldehir Rojas
2026-10-09 10:06:22 -05:00
committed by GitHub
parent a518119d30
commit 8b54361025
41 changed files with 562 additions and 736 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/chat-auto-parser-generator.cpp
+2 -3
View File
@@ -61,8 +61,7 @@ common_chat_params peg_generator::generate_parser(const common_chat_template &
data.prompt += data.generation_prompt;
}
auto parser = autoparser.build_parser(inputs, parser_generation_prompt);
data.parser = parser.save();
data.parser = autoparser.build_parser(inputs, parser_generation_prompt);
// Build grammar if tools are present
bool has_tools =
@@ -78,7 +77,7 @@ common_chat_params peg_generator::generate_parser(const common_chat_template &
if (include_grammar) {
data.grammar_lazy = !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
parser.build_grammar(builder, data.grammar_lazy);
data.parser.build_grammar(builder, data.grammar_lazy);
});
// Set grammar triggers based on tool section markers (fall back to per-call markers)
common/chat.cpp
+178 -71
View File
@@ -9,6 +9,7 @@
#include "json.h"
#include "log.h"
#include "parsers/parsers.h"
#include "sampling.h"
#include "jinja/value.h"
#include "jinja/runtime.h"
@@ -112,38 +113,6 @@ const char * common_chat_role_to_string(common_chat_role role) {
return "";
}
json common_chat_msg_delimiters::to_json() const {
json result = json::array();
for (const auto & d : delimiters) {
result.push_back({
{ "role", common_chat_role_to_string(d.role) },
{ "delimiter", d.delimiter },
});
}
return result;
}
common_chat_msg_delimiters common_chat_msg_delimiters_parse(const json & delimiters) {
common_chat_msg_delimiters result;
if (!delimiters.is_array()) {
return result;
}
result.delimiters.reserve(delimiters.size());
for (const auto & d : delimiters) {
if (!d.is_object()) {
continue;
}
result.delimiters.push_back({
common_chat_role_from_string(d.value("role", std::string())),
d.value("delimiter", std::string()),
});
}
return result;
}
void common_chat_msg_delimiters::tokenize(const llama_vocab * vocab) {
for (auto & d : delimiters) {
d.tokens = common_tokenize(vocab, d.delimiter, false, true);
@@ -620,8 +589,11 @@ std::vector<common_chat_tool> common_chat_tools_parse_oaicompat(const json & too
}
common_chat_continuation common_chat_continuation_parse(const common_json & value) {
if (value.is_boolean() && value.get<bool>()) {
return COMMON_CHAT_CONTINUATION_AUTO;
if (value.is_null()) {
return COMMON_CHAT_CONTINUATION_NONE;
}
if (value.is_boolean()) {
return value.get<bool>() ? COMMON_CHAT_CONTINUATION_AUTO : COMMON_CHAT_CONTINUATION_NONE;
}
if (value.is_string()) {
auto value_str = value.get<std::string>();
@@ -632,7 +604,7 @@ common_chat_continuation common_chat_continuation_parse(const common_json & valu
return COMMON_CHAT_CONTINUATION_CONTENT;
}
}
return COMMON_CHAT_CONTINUATION_NONE;
throw std::invalid_argument("Invalid continue_final_message: expected a boolean, \"content\" or \"reasoning_content\"");
}
bool common_chat_verify_template(const std::string & tmpl, bool use_jinja) {
@@ -1087,41 +1059,55 @@ static json common_chat_extra_context() {
return ctx;
}
std::optional<common_chat_params> common_chat_try_specialized_template(
const common_chat_template & tmpl,
const std::string & src,
autoparser::generation_params & params) {
static common_chat_params common_chat_params_init_lfm2_tokens(const common_chat_template & tmpl, const autoparser::generation_params & inputs) {
return common_chat_params_init_lfm2(tmpl, inputs, /* tool_list_tokens = */ true);
}
static common_chat_params common_chat_params_init_lfm2_5(const common_chat_template & tmpl, const autoparser::generation_params & inputs) {
return common_chat_params_init_lfm2(tmpl, inputs, /* tool_list_tokens = */ false);
}
// Older gemma4 templates need their tool responses rewritten before rendering
static common_chat_params common_chat_params_init_gemma4_legacy(const common_chat_template & tmpl, const autoparser::generation_params & inputs) {
auto adjusted = inputs;
workaround::convert_tool_responses_gemma4(adjusted.messages);
return common_chat_params_init_gemma4(tmpl, adjusted);
}
// Pick the dedicated handler for a template from its source, or null for the autoparser.
// Order matters: the first match wins, and later checks assume the earlier ones did not match.
static common_chat_params_init_fn common_chat_template_detect_params_init(const std::string & src) {
// Ministral/Mistral Large 3 - uses special reasoning structure fixes, can't use autoparser
// Note: Mistral Small 3.2 uses [CALL_ID] which Ministral doesn't have, so we can distinguish them
if (src.find("[SYSTEM_PROMPT]") != std::string::npos && src.find("[TOOL_CALLS]") != std::string::npos &&
src.find("[ARGS]") != std::string::npos && src.find("[CALL_ID]") == std::string::npos) {
LOG_DBG("Using specialized template: Ministral/Magistral Large 3\n");
return common_chat_params_init_ministral_3(tmpl, params);
return common_chat_params_init_ministral_3;
}
// LLM-jp-4.1 - GPT-OSS dialect (spaces after special tokens, <|end|>-separated parallel calls)
if (src.find("chat_format=llm-jp-harmony-v1") != std::string::npos) {
LOG_DBG("Using specialized template: LLM-jp Harmony v1\n");
return common_chat_params_init_llm_jp_harmony(tmpl, params);
return common_chat_params_init_llm_jp_harmony;
}
// GPT-OSS - has unique channel-based structure that needs dedicated handler
if (src.find("<|channel|>") != std::string::npos) {
LOG_DBG("Using specialized template: GPT-OSS\n");
return common_chat_params_init_gpt_oss(tmpl, params);
return common_chat_params_init_gpt_oss;
}
// Muse Glimmer format using " to=<recipient>" recipients and <|eom|>/<|eot|> message terminators.
if (src.find("<atem:function_calls>") != std::string::npos && src.find("<|eom|>") != std::string::npos) {
LOG_DBG("Using specialized template: Muse Glimmer\n");
return common_chat_params_init_muse_glimmer(tmpl, params);
return common_chat_params_init_muse_glimmer;
}
// Functionary v3.2 - uses recipient-based format with >>>recipient\n{content}
// Detection: template has ">>>all" for content and ">>>" prefix for tool calls
if (src.find(">>>all") != std::string::npos && src.find(">>>${recipient}") != std::string::npos) {
LOG_DBG("Using specialized template: Functionary v3.2\n");
return common_chat_params_init_functionary_v3_2(tmpl, params);
return common_chat_params_init_functionary_v3_2;
}
// Kimi K2 Thinking - uses unique tool call ID format: functions.<name>:<index>
@@ -1129,14 +1115,14 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
if (src.find("<|tool_calls_section_begin|>") != std::string::npos &&
src.find("<|tool_call_begin|>") != std::string::npos) {
LOG_DBG("Using specialized template: Kimi K2 Thinking\n");
return common_chat_params_init_kimi_k2(tmpl, params);
return common_chat_params_init_kimi_k2;
}
// Kimi K3 - the <|open|>/<|close|>/<|end_of_msg|> markers are unique to it
if (src.find("<|open|>") != std::string::npos && src.find("<|close|>") != std::string::npos &&
src.find("<|end_of_msg|>") != std::string::npos) {
LOG_DBG("Using specialized template: Kimi K3\n");
return common_chat_params_init_kimi_k3(tmpl, params);
return common_chat_params_init_kimi_k3;
}
// K2 Horizon - <|ifm|im_start|> turns, <ifm|think*> reasoning picked by reasoning_effort and
@@ -1144,7 +1130,7 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
if (src.find("<|ifm|im_start|>") != std::string::npos &&
src.find("<ifm|tool_calls>") != std::string::npos) {
LOG_DBG("Using specialized template: K2 Horizon\n");
return common_chat_params_init_k2_horizon(tmpl, params);
return common_chat_params_init_k2_horizon;
}
// Ling 3.0 / Bailing V3 - <role>X</role> sections with <arg_key>/<arg_value> tagged
@@ -1152,7 +1138,7 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
if (src.find("<role>ASSISTANT</role>") != std::string::npos &&
src.find("<arg_key>") != std::string::npos) {
LOG_DBG("Using specialized template: Ling 3.0 (Bailing V3)\n");
return common_chat_params_init_ling3(tmpl, params);
return common_chat_params_init_ling3;
}
// Cohere2 MoE / North Code - marker-wrapped format with <|START_TEXT|> content and
@@ -1161,19 +1147,19 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
if (src.find("<|START_TEXT|>") != std::string::npos &&
src.find("<|START_ACTION|>") != std::string::npos) {
LOG_DBG("Using specialized template: Cohere2 MoE\n");
return common_chat_params_init_cohere2moe(tmpl, params);
return common_chat_params_init_cohere2moe;
}
if (is_lfm2_template(src)) {
LOG_DBG("Using specialized template: LFM2\n");
return common_chat_params_init_lfm2(tmpl, params, /* tool_list_tokens = */ true);
return common_chat_params_init_lfm2_tokens;
}
// LFM2.5 format detection: template uses plain "List of tools: [...]" with no special tokens
if (src.find("List of tools: [") != std::string::npos &&
src.find("<|tool_list_start|>") == std::string::npos) {
LOG_DBG("Using specialized template: LFM2.5\n");
return common_chat_params_init_lfm2(tmpl, params, /* tool_list_tokens = */ false);
return common_chat_params_init_lfm2_5;
}
// GigaChatV3 format detection
@@ -1181,7 +1167,7 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
src.find("<|message_sep|>") != std::string::npos &&
src.find("<|function_call|>") == std::string::npos) {
LOG_DBG("Using specialized template: GigaChatV3\n");
return common_chat_params_init_gigachat_v3(tmpl, params);
return common_chat_params_init_gigachat_v3;
}
// MiniMax-M3: the namespace token "]<]minimax[>[" collides with the autoparser's
@@ -1190,7 +1176,7 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
src.find("<tool_call>") != std::string::npos &&
src.find("<invoke name=") != std::string::npos) {
LOG_DBG("Using specialized template: MiniMax-M3\n");
return common_chat_params_init_minimax_m3(tmpl, params);
return common_chat_params_init_minimax_m3;
}
// DeepSeek V3.2/V4 format detection: template defines dsml_token and uses it for tool calls.
@@ -1201,18 +1187,18 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
(src.find("function_calls") != std::string::npos ||
src.find("tool_calls") != std::string::npos)) {
LOG_DBG("Using specialized template: DeepSeek V3.2/V4\n");
return common_chat_params_init_deepseek_v3_2(tmpl, params);
return common_chat_params_init_deepseek_v3_2;
}
// Gemma4 format detection
if (src.find("'<|tool_call>call:'") != std::string::npos) {
LOG_DBG("Using specialized template: Gemma4\n");
if (src.find("{#- OpenAI Chat Completions:") == std::string::npos) {
// apply workarounds if using the older gemma4 templates
LOG_WRN("%s: detected an outdated gemma4 chat template, applying compatibility workarounds. "
"Consider updating to the official template.\n", __func__);
workaround::convert_tool_responses_gemma4(params.messages);
return common_chat_params_init_gemma4_legacy;
}
return common_chat_params_init_gemma4(tmpl, params);
return common_chat_params_init_gemma4;
}
// MiniCPM5 - XML tool calls with <function name="..."><param name="...">...</param></function>
@@ -1220,14 +1206,14 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
src.find("<function name=\"") != std::string::npos &&
src.find("<param name=\"") != std::string::npos) {
LOG_DBG("Using specialized template: MiniCPM5\n");
return common_chat_params_init_minicpm5(tmpl, params);
return common_chat_params_init_minicpm5;
}
// TranslateGemma - user content must follow a custom schema with language codes
if (src.find("[source_lang_code]") != std::string::npos &&
src.find("[target_lang_code]") != std::string::npos) {
LOG_DBG("Using specialized template: TranslateGemma\n");
return common_chat_params_init_translate_gemma(tmpl, params);
return common_chat_params_init_translate_gemma;
}
// Qwen3-Coder XML tool calls, also used by Nemotron Nano 3, Qwen3.5 and StepFun-3.5-Flash
@@ -1237,10 +1223,51 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
// Exclude models that don't use \n between tags
src.find("'<tool_call><function=' ~ tool_call.name ~ '>'") == std::string::npos) {
LOG_DBG("Using specialized template: Qwen3-Coder\n");
return common_chat_params_init_qwen3_coder(tmpl, params);
return common_chat_params_init_qwen3_coder;
}
return std::nullopt;
return nullptr;
}
common_chat_template::common_chat_template(const std::string & src, const std::string & bos_token, const std::string & eos_token) {
jinja::lexer lexer;
auto lexer_res = lexer.tokenize(src);
this->prog = jinja::parse_from_tokens(lexer_res);
this->src = lexer_res.source;
this->bos_tok = bos_token;
this->eos_tok = eos_token;
this->caps = jinja::caps_get(prog);
// LOG_INF("%s: caps:\n%s\n", __func__, this->caps.to_string().c_str());
this->params_init = common_chat_template_detect_params_init(this->src);
if (this->params_init) {
return;
}
// The analysis depends only on the template, so run it once here instead of on every apply.
// A failure is kept for apply to report, so a bad template still loads like it did before.
try {
analysis = std::make_unique<autoparser::autoparser>();
analysis->analyze_template(*this);
} catch (const std::exception & e) {
analysis.reset();
analysis_error = e.what();
}
}
common_chat_template::~common_chat_template() = default;
common_chat_template::common_chat_template(common_chat_template &&) = default;
common_chat_template & common_chat_template::operator=(common_chat_template &&) = default;
std::optional<common_chat_params> common_chat_try_specialized_template(
const common_chat_template & tmpl,
const autoparser::generation_params & params) {
if (!tmpl.params_init) {
return std::nullopt;
}
return tmpl.params_init(tmpl, params);
}
static common_chat_params common_chat_templates_apply_jinja(const struct common_chat_templates * tmpls,
@@ -1342,21 +1369,23 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_
data.prompt = common_chat_template_direct_apply_impl(tmpl, params_copy);
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, params);
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
auto parser = build_chat_peg_parser([&data](common_chat_peg_builder &p) {
data.parser = build_chat_peg_parser([&data](common_chat_peg_builder &p) {
return p.literal(data.generation_prompt) << p.content(p.rest());
});
data.parser = parser.save();
return data;
}
if (auto result = common_chat_try_specialized_template(tmpl, src, params)) {
if (auto result = common_chat_try_specialized_template(tmpl, params)) {
return *result;
}
if (!tmpl.analysis) {
throw std::invalid_argument("Unable to generate parser for this template. Automatic parser generation failed: " + tmpl.analysis_error);
}
try {
LOG_DBG("%s: using differential autoparser\n", __func__);
struct autoparser::autoparser autoparser;
autoparser.analyze_template(tmpl);
const auto & autoparser = *tmpl.analysis;
auto auto_params = autoparser::peg_generator::generate_parser(tmpl, params, autoparser);
common_chat_msg_delimiters delimiters;
@@ -1377,8 +1406,7 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_
auto_params.thinking_end_tags = {std::move(end_tag)};
}
}
common_peg_arena arena;
arena.load(auto_params.parser);
const auto & arena = auto_params.parser;
LOG_DBG("%s: generated parser:\n%s\n\nparser generation prompt: %s\n", __func__, arena.dump(arena.root()).c_str(), auto_params.generation_prompt.c_str());
return auto_params;
} catch (const std::exception & e) {
@@ -1525,9 +1553,10 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars
const common_chat_input & input,
bool is_partial,
const common_chat_parser_params & params) {
const common_peg_arena & parser = src_parser.empty() ?
build_chat_peg_parser([](common_chat_peg_builder & p) { return p.content(p.rest()) + p.end(); }) :
src_parser;
// both branches must be lvalues, a temporary here would copy the arena on every call
static const common_peg_arena content_only =
build_chat_peg_parser([](common_chat_peg_builder & p) { return p.content(p.rest()) + p.end(); });
const common_peg_arena & parser = src_parser.empty() ? content_only : src_parser;
if (src_parser.empty()) {
LOG_DBG("No parser definition detected, assuming pure content parser.");
@@ -1598,6 +1627,84 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars
return msg;
}
common_chat_session::common_chat_session(const common_chat_templates * tmpls,
const llama_vocab * vocab,
const common_chat_templates_inputs & inputs,
const common_chat_session_params & params) {
auto applied = common_chat_templates_apply(tmpls, inputs);
templated = true;
prompt_text = std::move(applied.prompt);
result.role = "assistant";
grammar_text = std::move(applied.grammar);
grammar_lazy = applied.grammar_lazy;
stops = std::move(applied.additional_stops);
generation_prompt_text = applied.generation_prompt;
thinking_start = std::move(applied.thinking_start_tag);
thinking_ends = std::move(applied.thinking_end_tags);
parser_params.format = applied.format;
parser_params.generation_prompt = vocab ? common_chat_input_tokenize(vocab, applied.generation_prompt)
: common_chat_input(applied.generation_prompt);
parser_params.debug = params.debug;
parser_params.parser = std::move(applied.parser);
delimiters = std::move(applied.message_delimiters);
if (vocab) {
common_params_sampling resolved;
resolved.grammar_lazy = applied.grammar_lazy;
common_sampling_add_preserved_tokens(resolved, vocab, applied.preserved_tokens);
common_sampling_add_grammar_triggers(resolved, vocab, std::move(applied.grammar_triggers));
preserved_tokens = std::move(resolved.preserved_tokens);
grammar_triggers = std::move(resolved.grammar_triggers);
delimiters.tokenize(vocab);
} else {
grammar_triggers = std::move(applied.grammar_triggers);
}
if (inputs.continue_final_message != COMMON_CHAT_CONTINUATION_NONE && !params.echo) {
// start from the prefill so it is not emitted as part of the first delta
result = common_chat_parse(input, true, parser_params);
}
}
void common_chat_session::apply_sampling(common_params_sampling & sampling) const {
if (!templated) {
return;
}
if (!grammar_text.empty()) {
sampling.grammar = {COMMON_GRAMMAR_TYPE_TOOL_CALLS, grammar_text};
}
sampling.grammar_lazy = grammar_lazy;
sampling.generation_prompt = generation_prompt_text;
sampling.preserved_tokens.insert(preserved_tokens.begin(), preserved_tokens.end());
sampling.grammar_triggers.insert(sampling.grammar_triggers.end(), grammar_triggers.begin(), grammar_triggers.end());
}
const common_chat_msg & common_chat_session::feed(const common_chat_input & chunk) {
GGML_ASSERT(!finished && "feed() after finish()");
input.append(chunk);
auto msg = common_chat_parse(input, true, parser_params);
if (!msg.empty()) {
result = std::move(msg);
}
return result;
}
const common_chat_msg & common_chat_session::finish(const common_chat_input & chunk) {
GGML_ASSERT(!finished && "finish() called twice");
finished = true;
input.append(chunk);
auto msg = common_chat_parse(input, false, parser_params);
if (!msg.empty()) {
result = std::move(msg);
}
return result;
}
std::map<std::string, bool> common_chat_templates_get_caps(const common_chat_templates * chat_templates) {
GGML_ASSERT(chat_templates != nullptr);
GGML_ASSERT(chat_templates->template_default != nullptr);
common/chat.h
+79 -27
View File
@@ -22,8 +22,16 @@ struct common_chat_templates;
namespace autoparser {
struct generation_params;
struct autoparser;
} // namespace autoparser
struct common_chat_params;
struct common_chat_template;
// Builds the prompt and parser for a template that has a dedicated handler (see common/parsers)
using common_chat_params_init_fn = common_chat_params (*)(const common_chat_template & tmpl,
const autoparser::generation_params & inputs);
struct common_chat_tool_call {
std::string name;
std::string arguments;
@@ -54,19 +62,20 @@ struct common_chat_template {
std::string eos_tok;
std::string src;
chat_template_caps caps;
// Dedicated handler picked once from the source, null when the differential autoparser is used
common_chat_params_init_fn params_init = nullptr;
common_chat_template(const std::string & src, const std::string & bos_token, const std::string & eos_token) {
jinja::lexer lexer;
auto lexer_res = lexer.tokenize(src);
this->prog = jinja::parse_from_tokens(lexer_res);
// Differential analysis, run once here when there is no dedicated handler. Null when there
// is one, or when the analysis failed, in which case analysis_error says why.
std::unique_ptr<autoparser::autoparser> analysis;
std::string analysis_error;
this->src = lexer_res.source;
this->bos_tok = bos_token;
this->eos_tok = eos_token;
common_chat_template(const std::string & src, const std::string & bos_token, const std::string & eos_token);
this->caps = jinja::caps_get(prog);
// LOG_INF("%s: caps:\n%s\n", __func__, this->caps.to_string().c_str());
}
// autoparser is incomplete here, so these are defined where it is complete
~common_chat_template();
common_chat_template(common_chat_template &&);
common_chat_template & operator=(common_chat_template &&);
const std::string & source() const { return src; }
const std::string & bos_token() const { return bos_tok; }
@@ -209,8 +218,6 @@ struct common_chat_msg_delimiters {
// split tokens into message spans. skips maps a start index to a length of a region to jump over without matching
common_chat_msg_spans split(const llama_tokens & tokens, const std::map<size_t, size_t> & skips = {}) const;
common_json to_json() const;
};
struct common_chat_tool {
@@ -278,7 +285,7 @@ struct common_chat_params {
std::vector<common_grammar_trigger> grammar_triggers;
std::vector<std::string> preserved_tokens;
std::vector<std::string> additional_stops;
std::string parser;
common_peg_arena parser;
common_chat_msg_delimiters message_delimiters;
};
@@ -310,16 +317,10 @@ common_chat_input common_chat_input_tokenize(const llama_vocab * vocab, const st
// per-message parsing syntax
// should be derived from common_chat_params
struct common_chat_parser_params {
common_chat_format format = COMMON_CHAT_FORMAT_CONTENT_ONLY;
common_reasoning_format reasoning_format = COMMON_REASONING_FORMAT_NONE; // TODO: refactor this to "bool parse_reasoning"
// Whether reasoning_content should be inlined in the content (e.g. for reasoning_format=deepseek in stream mode)
bool reasoning_in_content = false;
common_chat_input generation_prompt;
bool parse_tool_calls = true;
bool is_continuation = false;
bool echo = false; // Include assistant prefilled msg in output
bool debug = false; // Enable debug output for PEG parser
common_peg_arena parser = {};
common_chat_format format = COMMON_CHAT_FORMAT_CONTENT_ONLY;
common_chat_input generation_prompt;
bool debug = false; // Enable debug output for PEG parser
common_peg_arena parser = {};
common_chat_parser_params() = default;
common_chat_parser_params(const common_chat_params & chat_params) {
format = chat_params.format;
@@ -365,6 +366,60 @@ const char * common_chat_format_name(common_chat_format format);
common_chat_msg common_chat_parse(const common_chat_input & input, bool is_partial, const common_chat_parser_params & params);
common_chat_msg common_chat_peg_parse(const common_peg_arena & src_parser, const common_chat_input & input, bool is_partial, const common_chat_parser_params & params);
struct common_chat_session_params {
bool echo = false; // include the assistant prefill in the output when continuing a message
bool debug = false; // enable debug output for the PEG parser
};
class common_chat_session {
public:
common_chat_session() { result.role = "assistant"; }
common_chat_session(const common_chat_templates * tmpls,
const llama_vocab * vocab,
const common_chat_templates_inputs & inputs,
const common_chat_session_params & params = {});
const std::string & prompt() const { return prompt_text; }
common_chat_format format() const { return parser_params.format; }
const common_chat_msg & msg() const { return result; }
const common_peg_arena & parser() const { return parser_params.parser; }
const std::string & grammar() const { return grammar_text; }
const std::string & generation_prompt() const { return generation_prompt_text; }
const std::string & thinking_start_tag() const { return thinking_start; }
const std::vector<std::string> & thinking_end_tags() const { return thinking_ends; }
const std::vector<std::string> & additional_stops() const { return stops; }
const common_chat_msg_delimiters & message_delimiters() const { return delimiters; }
void apply_sampling(common_params_sampling & sampling) const;
bool has_template() const { return templated; }
const common_chat_msg & feed(const common_chat_input & chunk);
const common_chat_msg & finish(const common_chat_input & chunk = {});
private:
std::string prompt_text;
std::string grammar_text;
bool grammar_lazy = false;
std::vector<common_grammar_trigger> grammar_triggers;
std::set<llama_token> preserved_tokens;
std::vector<std::string> stops;
std::string generation_prompt_text;
std::string thinking_start;
std::vector<std::string> thinking_ends;
common_chat_parser_params parser_params;
common_chat_msg_delimiters delimiters;
common_chat_input input;
common_chat_msg result;
bool templated = false;
bool finished = false;
};
// used by arg and server
const char * common_reasoning_format_name(common_reasoning_format format);
common_reasoning_format common_reasoning_format_from_name(const std::string & format);
@@ -401,8 +456,7 @@ std::string common_chat_template_generation_prompt(
std::optional<common_chat_params> common_chat_try_specialized_template(
const common_chat_template & tmpl,
const std::string & src,
autoparser::generation_params & params);
const autoparser::generation_params & params);
// specialized per-task preset
@@ -412,5 +466,3 @@ struct common_chat_prompt_preset {
};
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates);
common_chat_msg_delimiters common_chat_msg_delimiters_parse(const common_json & delimiters);
common/parsers/cohere2moe.cpp
+2 -4
View File
@@ -77,7 +77,7 @@ common_chat_params common_chat_params_init_cohere2moe(const common_chat_template
data.prompt += data.generation_prompt;
}
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto generation_prompt = p.literal(GEN_PREFIX);
auto end = p.end();
@@ -124,12 +124,10 @@ common_chat_params common_chat_params_init_cohere2moe(const common_chat_template
return generation_prompt + reasoning + body + p.optional(p.literal(TURN_END)) + end;
});
data.parser = parser.save();
if (include_grammar) {
data.grammar_lazy = !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
parser.build_grammar(builder, data.grammar_lazy);
data.parser.build_grammar(builder, data.grammar_lazy);
});
data.grammar_triggers = {
common/parsers/deepseek.cpp
+2 -4
View File
@@ -145,7 +145,7 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templ
bool require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;
bool has_tool_calls = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto generation_prompt = p.literal(GEN_PROMPT);
auto end = p.end();
@@ -256,12 +256,10 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templ
generation_prompt + reasoning + content_before_tools + tool_calls + end;
});
data.parser = parser.save();
if (include_grammar) {
data.grammar_lazy = has_tools && !require_tools;
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
parser.build_grammar(builder, data.grammar_lazy);
data.parser.build_grammar(builder, data.grammar_lazy);
});
data.grammar_triggers = {
common/parsers/functionary-v3-2.cpp
+2 -4
View File
@@ -21,7 +21,7 @@ common_chat_params common_chat_params_init_functionary_v3_2(const common_chat_te
data.prompt += data.generation_prompt;
}
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
// Functionary v3.2 format:
// - Normal content: >>>all\n{content}
// - Tool calls: >>>function_name\n{json_args}
@@ -76,13 +76,11 @@ common_chat_params common_chat_params_init_functionary_v3_2(const common_chat_te
return generation_prompt + ret;
});
data.parser = parser.save();
if (include_grammar) {
data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
parser.build_grammar(builder, data.grammar_lazy);
data.parser.build_grammar(builder, data.grammar_lazy);
});
// Grammar trigger for when the model starts outputting a tool call
common/parsers/gemma4.cpp
+2 -4
View File
@@ -198,7 +198,7 @@ common_chat_params common_chat_params_init_gemma4(const common_chat_template &
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto start = p.rule("start", p.optional(p.literal("<|turn>model\n")));
if (extract_reasoning) {
@@ -290,12 +290,10 @@ common_chat_params common_chat_params_init_gemma4(const common_chat_template &
return start + p.one_or_more(message);
});
data.parser = parser.save();
if (include_grammar) {
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
parser.build_grammar(builder, data.grammar_lazy);
data.parser.build_grammar(builder, data.grammar_lazy);
});
data.grammar_triggers = {
common/parsers/gigachat-v3.cpp
+2 -4
View File
@@ -25,7 +25,7 @@ common_chat_params common_chat_params_init_gigachat_v3(
auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
const auto *tool_call_start_prefix = "<|message_sep|>\n\nfunction call<|role_sep|>\n";
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto ret = p.eps();
if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
// Build a choice of all available tools
@@ -60,13 +60,11 @@ common_chat_params common_chat_params_init_gigachat_v3(
return p.literal("assistant<|role_sep|>\n") + ret;
});
data.parser = parser.save();
if (include_grammar) {
data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
parser.build_grammar(builder, data.grammar_lazy);
data.parser.build_grammar(builder, data.grammar_lazy);
});
data.grammar_triggers = {
common/parsers/gpt-oss.cpp
+3 -6
View File
@@ -45,8 +45,7 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template &
data.thinking_start_tag = "<|channel|>analysis<|message|>";
data.thinking_end_tags = {"<|end|>"};
// These special tokens are required to parse properly, so we include them
// even if parse_tool_calls is false.
// These special tokens are required to parse properly
data.preserved_tokens = {
"<|channel|>", "<|constrain|>", "<|message|>", "<|start|>", "<|end|>",
};
@@ -68,7 +67,7 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template &
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto start = p.rule("start", p.literal("<|start|>assistant"));
auto end = p.rule("end", p.literal("<|end|>"));
auto content = p.rule("message-content", p.until("<|end|>"));
@@ -138,12 +137,10 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template &
return p.zero_or_more(start + any) + start + (final_msg | unsolicited);
});
data.parser = parser.save();
if (include_grammar) {
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
parser.build_grammar(builder, data.grammar_lazy);
data.parser.build_grammar(builder, data.grammar_lazy);
});
data.grammar_triggers = {
common/parsers/k2-horizon.cpp
+2 -4
View File
@@ -77,7 +77,7 @@ common_chat_params common_chat_params_init_k2_horizon(const common_chat_template
data.prompt += data.generation_prompt;
}
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto generation_prompt = p.literal(GEN_PREFIX);
auto think_end = p.choice();
@@ -174,12 +174,10 @@ common_chat_params common_chat_params_init_k2_horizon(const common_chat_template
return generation_prompt + (reasoning << content << tool_calls);
});
data.parser = parser.save();
if (include_grammar) {
data.grammar_lazy = !(has_response_format || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED);
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
parser.build_grammar(builder, data.grammar_lazy);
data.parser.build_grammar(builder, data.grammar_lazy);
});
if (data.grammar_lazy) {
common/parsers/kimi-k2.cpp
+2 -4
View File
@@ -48,7 +48,7 @@ common_chat_params common_chat_params_init_kimi_k2(const common_chat_template &
data.prompt += data.generation_prompt;
}
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
// Kimi K2 Thinking format:
// - Reasoning: <think>{reasoning}</think>
// - Content: text after reasoning
@@ -111,12 +111,10 @@ common_chat_params common_chat_params_init_kimi_k2(const common_chat_template &
return generation_prompt + reasoning + content_before_tools + tool_calls + end;
});
data.parser = parser.save();
if (include_grammar) {
data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
parser.build_grammar(builder, data.grammar_lazy);
data.parser.build_grammar(builder, data.grammar_lazy);
});
data.grammar_triggers = {
common/parsers/kimi-k3.cpp
+2 -4
View File
@@ -66,7 +66,7 @@ common_chat_params common_chat_params_init_kimi_k3(const common_chat_template &
data.prompt += data.generation_prompt;
}
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto end = p.end();
auto start = p.optional(p.literal(MSG_START));
@@ -151,12 +151,10 @@ common_chat_params common_chat_params_init_kimi_k3(const common_chat_template &
return start + reasoning + response + tools + trailer + end;
});
data.parser = parser.save();
if (include_grammar) {
data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
parser.build_grammar(builder, data.grammar_lazy);
data.parser.build_grammar(builder, data.grammar_lazy);
});
data.grammar_triggers = {
common/parsers/lfm2.cpp
+2 -4
View File
@@ -64,7 +64,7 @@ common_chat_params common_chat_params_init_lfm2(const common_chat_template &
data.prompt += data.generation_prompt;
}
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto generation_prompt = p.literal(GEN_PROMPT);
auto end = p.end();
@@ -93,12 +93,10 @@ common_chat_params common_chat_params_init_lfm2(const common_chat_template &
return generation_prompt + reasoning + content + tool_calls + end;
});
data.parser = parser.save();
if (include_grammar) {
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
parser.build_grammar(builder, data.grammar_lazy);
data.parser.build_grammar(builder, data.grammar_lazy);
});
data.grammar_triggers = {
common/parsers/ling3.cpp
+2 -4
View File
@@ -80,7 +80,7 @@ common_chat_params common_chat_params_init_ling3(const common_chat_template &
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto end = p.end();
// the effective parse input is generation_prompt + model output, so the
@@ -185,12 +185,10 @@ common_chat_params common_chat_params_init_ling3(const common_chat_template &
return opener + reasoning + content + tools + tail + end;
});
data.parser = parser.save();
if (include_grammar) {
data.grammar_lazy = !has_response_format && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
parser.build_grammar(builder, data.grammar_lazy);
data.parser.build_grammar(builder, data.grammar_lazy);
});
data.grammar_triggers = {
common/parsers/llm-jp-harmony.cpp
+3 -6
View File
@@ -48,8 +48,7 @@ common_chat_params common_chat_params_init_llm_jp_harmony(const common_chat_temp
data.thinking_start_tag = "<|channel|>analysis<|message|>";
data.thinking_end_tags = {"<|end|>"};
// These special tokens are required to parse properly, so we include them
// even if parse_tool_calls is false.
// These special tokens are required to parse properly
data.preserved_tokens = {
"<|channel|>", "<|constrain|>", "<|message|>", "<|start|>", "<|end|>",
};
@@ -71,7 +70,7 @@ common_chat_params common_chat_params_init_llm_jp_harmony(const common_chat_temp
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
// tokenizer space after special tokens; not p.space() since GBNF `space` allows one space only
auto sp = p.chars("[ ]", 0, -1);
auto channel_tag = p.literal("<|channel|>") + sp;
@@ -144,12 +143,10 @@ common_chat_params common_chat_params_init_llm_jp_harmony(const common_chat_temp
return p.zero_or_more(start + any) + start + final_msg;
});
data.parser = parser.save();
if (include_grammar) {
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
parser.build_grammar(builder, data.grammar_lazy);
data.parser.build_grammar(builder, data.grammar_lazy);
});
data.grammar_triggers = {
common/parsers/minicpm5.cpp
+2 -4
View File
@@ -46,7 +46,7 @@ common_chat_params common_chat_params_init_minicpm5(const common_chat_template &
data.prompt += data.generation_prompt;
}
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto generation_prompt = p.literal("<|im_start|>assistant\n");
auto reasoning = p.eps();
@@ -113,12 +113,10 @@ common_chat_params common_chat_params_init_minicpm5(const common_chat_template &
return generation_prompt + reasoning + p.content(p.rest()) + p.end();
});
data.parser = parser.save();
if (include_grammar) {
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
parser.build_grammar(builder, data.grammar_lazy);
data.parser.build_grammar(builder, data.grammar_lazy);
});
data.grammar_triggers = {
common/parsers/minimax-m3.cpp
+2 -4
View File
@@ -56,7 +56,7 @@ common_chat_params common_chat_params_init_minimax_m3(const common_chat_template
data.prompt += data.generation_prompt;
}
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto generation_prompt = p.prefix(GEN_PROMPT, THINK_START);
auto end = p.end();
@@ -213,12 +213,10 @@ common_chat_params common_chat_params_init_minimax_m3(const common_chat_template
return generation_prompt + reasoning + content_before_tools + tool_calls + end;
});
data.parser = parser.save();
if (include_grammar) {
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
parser.build_grammar(builder, data.grammar_lazy);
data.parser.build_grammar(builder, data.grammar_lazy);
});
data.grammar_triggers = {
common/parsers/ministral3.cpp
+2 -4
View File
@@ -72,7 +72,7 @@ common_chat_params common_chat_params_init_ministral_3(const common_chat_templat
data.prompt += data.generation_prompt;
}
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto generation_prompt = p.eps();
auto reasoning =
extract_reasoning ? p.optional("[THINK]" + p.reasoning(p.until("[/THINK]")) + "[/THINK]") : p.eps();
@@ -108,13 +108,11 @@ common_chat_params common_chat_params_init_ministral_3(const common_chat_templat
return generation_prompt + (reasoning << p.content(p.rest()));
});
data.parser = parser.save();
if (include_grammar) {
data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
parser.build_grammar(builder, data.grammar_lazy);
data.parser.build_grammar(builder, data.grammar_lazy);
});
data.grammar_triggers = {
common/parsers/muse-glimmer.cpp
+2 -4
View File
@@ -48,7 +48,7 @@ common_chat_params common_chat_params_init_muse_glimmer(const common_chat_templa
// Constrained grammar whenever tools are offered or a response format is requested.
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto start = p.rule("start", p.literal("<|start|>assistant"));
if (!extract_reasoning && !include_grammar) {
@@ -131,12 +131,10 @@ common_chat_params common_chat_params_init_muse_glimmer(const common_chat_templa
return p.zero_or_more(start + analysis) + start + final_msg;
});
data.parser = parser.save();
if (include_grammar) {
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
parser.build_grammar(builder, data.grammar_lazy);
data.parser.build_grammar(builder, data.grammar_lazy);
});
data.grammar_triggers = {
{ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN,
common/parsers/qwen3-coder.cpp
+2 -4
View File
@@ -71,7 +71,7 @@ common_chat_params common_chat_params_init_qwen3_coder(const common_chat_templat
});
}
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto generation_prompt = p.literal(GEN_PREFIX);
auto reasoning = p.eps();
@@ -174,13 +174,11 @@ common_chat_params common_chat_params_init_qwen3_coder(const common_chat_templat
return generation_prompt + (reasoning << p.content(p.rest()));
});
data.parser = parser.save();
if (include_grammar) {
data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
parser.build_grammar(builder, data.grammar_lazy);
data.parser.build_grammar(builder, data.grammar_lazy);
});
if (data.grammar_lazy) {
common/parsers/translate-gemma.cpp
+1 -2
View File
@@ -54,10 +54,9 @@ common_chat_params common_chat_params_init_translate_gemma(
data.prompt += data.generation_prompt;
}
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
return p.literal(data.generation_prompt) << p.content(p.rest());
});
data.parser = parser.save();
return data;
}
common/peg-parser.cpp
-303
View File
@@ -1814,309 +1814,6 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo
}
}
static common_json serialize_parser_variant(const common_peg_parser_variant & variant) {
using json = common_json;
return std::visit([](const auto & p) -> json {
using T = std::decay_t<decltype(p)>;
if constexpr (std::is_same_v<T, common_peg_epsilon_parser>) {
return json{{"type", "epsilon"}};
} else if constexpr (std::is_same_v<T, common_peg_start_parser>) {
return json{{"type", "start"}};
} else if constexpr (std::is_same_v<T, common_peg_end_parser>) {
return json{{"type", "end"}};
} else if constexpr (std::is_same_v<T, common_peg_literal_parser>) {
return json{{"type", "literal"}, {"literal", p.literal}};
} else if constexpr (std::is_same_v<T, common_peg_sequence_parser>) {
return json{{"type", "sequence"}, {"children", p.children}};
} else if constexpr (std::is_same_v<T, common_peg_choice_parser>) {
return json{{"type", "choice"}, {"children", p.children}};
} else if constexpr (std::is_same_v<T, common_peg_repetition_parser>) {
return json{
{"type", "repetition"},
{"child", p.child},
{"min_count", p.min_count},
{"max_count", p.max_count}
};
} else if constexpr (std::is_same_v<T, common_peg_and_parser>) {
return json{{"type", "and"}, {"child", p.child}};
} else if constexpr (std::is_same_v<T, common_peg_not_parser>) {
return json{{"type", "not"}, {"child", p.child}};
} else if constexpr (std::is_same_v<T, common_peg_any_parser>) {
return json{{"type", "any"}};
} else if constexpr (std::is_same_v<T, common_peg_space_parser>) {
return json{{"type", "space"}};
} else if constexpr (std::is_same_v<T, common_peg_chars_parser>) {
json ranges = json::array();
for (const auto & range : p.ranges) {
ranges.push_back({{"start", range.start}, {"end", range.end}});
}
return json{
{"type", "chars"},
{"pattern", p.pattern},
{"ranges", ranges},
{"negated", p.negated},
{"min_count", p.min_count},
{"max_count", p.max_count}
};
} else if constexpr (std::is_same_v<T, common_peg_string_parser>) {
return json{{"type", "string"}, {"delimiter", std::string(1, p.delimiter)}};
} else if constexpr (std::is_same_v<T, common_peg_until_parser>) {
return json{{"type", "until"}, {"delimiters", p.delimiters}};
} else if constexpr (std::is_same_v<T, common_peg_schema_parser>) {
return json{
{"type", "schema"},
{"child", p.child},
{"name", p.name},
{"raw", p.raw}
};
} else if constexpr (std::is_same_v<T, common_peg_rule_parser>) {
return json{
{"type", "rule"},
{"name", p.name},
{"child", p.child},
{"trigger", p.trigger}
};
} else if constexpr (std::is_same_v<T, common_peg_ref_parser>) {
return json{{"type", "ref"}, {"name", p.name}};
} else if constexpr (std::is_same_v<T, common_peg_atomic_parser>) {
return json{{"type", "atomic"}, {"child", p.child}};
} else if constexpr (std::is_same_v<T, common_peg_tag_parser>) {
return json{
{"type", "tag"},
{"child", p.child},
{"tag", p.tag}
};
} else if constexpr (std::is_same_v<T, common_peg_gbnf_parser>) {
return json{{"type", "gbnf"}, {"child", p.child}, {"grammar", p.grammar}};
} else if constexpr (std::is_same_v<T, common_peg_ac_parser>) {
return json{{"type", "ac"}, {"child", p.child}, {"delimiters", p.delimiters}};
}
}, variant);
}
common_json common_peg_arena::to_json() const {
auto parsers = common_json::array();
for (const auto & parser : parsers_) {
parsers.push_back(serialize_parser_variant(parser));
}
return common_json{
{"parsers", parsers},
{"rules", rules_},
{"root", root_}
};
}
static common_peg_parser_variant deserialize_parser_variant(const common_json & j) {
if (!j.contains("type") || !j["type"].is_string()) {
throw std::runtime_error("Parser variant JSON missing or invalid 'type' field");
}
std::string type = j["type"];
if (type == "epsilon") {
return common_peg_epsilon_parser{};
}
if (type == "start") {
return common_peg_start_parser{};
}
if (type == "end") {
return common_peg_end_parser{};
}
if (type == "literal") {
if (!j.contains("literal") || !j["literal"].is_string()) {
throw std::runtime_error("literal parser missing or invalid 'literal' field");
}
return common_peg_literal_parser{j["literal"]};
}
if (type == "sequence") {
if (!j.contains("children") || !j["children"].is_array()) {
throw std::runtime_error("sequence parser missing or invalid 'children' field");
}
return common_peg_sequence_parser{j["children"].get<std::vector<common_peg_parser_id>>()};
}
if (type == "choice") {
if (!j.contains("children") || !j["children"].is_array()) {
throw std::runtime_error("choice parser missing or invalid 'children' field");
}
return common_peg_choice_parser{j["children"].get<std::vector<common_peg_parser_id>>()};
}
if (type == "repetition") {
if (!j.contains("child") || !j.contains("min_count") || !j.contains("max_count")) {
throw std::runtime_error("repetition parser missing required fields");
}
return common_peg_repetition_parser{
j["child"].get<common_peg_parser_id>(),
j["min_count"].get<int>(),
j["max_count"].get<int>()
};
}
if (type == "and") {
if (!j.contains("child")) {
throw std::runtime_error("and parser missing 'child' field");
}
return common_peg_and_parser{j["child"].get<common_peg_parser_id>()};
}
if (type == "not") {
if (!j.contains("child")) {
throw std::runtime_error("not parser missing 'child' field");
}
return common_peg_not_parser{j["child"].get<common_peg_parser_id>()};
}
if (type == "any") {
return common_peg_any_parser{};
}
if (type == "space") {
return common_peg_space_parser{};
}
if (type == "chars") {
if (!j.contains("pattern") || !j.contains("ranges") || !j.contains("negated") ||
!j.contains("min_count") || !j.contains("max_count")) {
throw std::runtime_error("chars parser missing required fields");
}
common_peg_chars_parser parser;
parser.pattern = j["pattern"];
parser.negated = j["negated"].get<bool>();
parser.min_count = j["min_count"].get<int>();
parser.max_count = j["max_count"].get<int>();
for (const auto & range_json : j["ranges"]) {
if (!range_json.contains("start") || !range_json.contains("end")) {
throw std::runtime_error("char_range missing 'start' or 'end' field");
}
parser.ranges.push_back({
range_json["start"].get<uint32_t>(),
range_json["end"].get<uint32_t>()
});
}
return parser;
}
if (type == "string") {
if (!j.contains("delimiter")) {
throw std::runtime_error("string parser missing delimiter field.");
}
std::string delimiter = j["delimiter"];
if (delimiter.empty()) {
throw std::runtime_error("string parser delimiter is empty.");
}
return common_peg_string_parser{delimiter[0]};
}
if (type == "until") {
if (!j.contains("delimiters") || !j["delimiters"].is_array()) {
throw std::runtime_error("until parser missing or invalid 'delimiters' field");
}
return common_peg_until_parser{j["delimiters"].get<std::vector<std::string>>()};
}
if (type == "schema") {
if (!j.contains("child") || !j.contains("name") || !j.contains("raw")) {
throw std::runtime_error("schema parser missing required fields");
}
common_peg_schema_parser parser;
parser.child = j["child"].get<common_peg_parser_id>();
parser.name = j["name"];
parser.raw = j["raw"].get<bool>();
return parser;
}
if (type == "rule") {
if (!j.contains("name") || !j.contains("child") || !j.contains("trigger")) {
throw std::runtime_error("rule parser missing required fields");
}
return common_peg_rule_parser{
j["name"].get<std::string>(),
j["child"].get<common_peg_parser_id>(),
j["trigger"].get<bool>()
};
}
if (type == "ref") {
if (!j.contains("name") || !j["name"].is_string()) {
throw std::runtime_error("ref parser missing or invalid 'name' field");
}
return common_peg_ref_parser{j["name"]};
}
if (type == "atomic") {
if (!j.contains("child")) {
throw std::runtime_error("tag parser missing required fields");
}
return common_peg_atomic_parser{
j["child"].get<common_peg_parser_id>(),
};
}
if (type == "tag") {
if (!j.contains("child") || !j.contains("tag")) {
throw std::runtime_error("tag parser missing required fields");
}
return common_peg_tag_parser{
j["child"].get<common_peg_parser_id>(),
j["tag"].get<std::string>(),
};
}
if (type == "gbnf") {
if (!j.contains("child") || !j.contains("grammar")) {
throw std::runtime_error("gbnf parser missing required fields");
}
return common_peg_gbnf_parser{
j["child"].get<common_peg_parser_id>(),
j["grammar"].get<std::string>(),
};
}
if (type == "ac") {
if (!j.contains("child") || !j.contains("delimiters") || !j["delimiters"].is_array() || j["delimiters"].empty()) {
throw std::runtime_error("ac parser requires 'child' and a non-empty 'delimiters' array");
}
return common_peg_ac_parser{
j["child"].get<common_peg_parser_id>(),
j["delimiters"].get<std::vector<std::string>>(),
};
}
throw std::runtime_error("Unknown parser type: " + type);
}
common_peg_arena common_peg_arena::from_json(const common_json & j) {
if (!j.contains("parsers") || !j["parsers"].is_array()) {
throw std::runtime_error("JSON missing or invalid 'parsers' array");
}
if (!j.contains("rules") || !j["rules"].is_object()) {
throw std::runtime_error("JSON missing or invalid 'rules' object");
}
if (!j.contains("root")) {
throw std::runtime_error("JSON missing 'root' field");
}
common_peg_arena arena;
const auto & parsers_json = j["parsers"];
arena.parsers_.reserve(parsers_json.size());
for (const auto & parser_json : parsers_json) {
arena.parsers_.push_back(deserialize_parser_variant(parser_json));
}
arena.rules_ = j["rules"].get<std::unordered_map<std::string, common_peg_parser_id>>();
for (const auto & [name, id] : arena.rules_) {
if (id >= arena.parsers_.size()) {
throw std::runtime_error("Rule '" + name + "' references invalid parser ID: " + std::to_string(id));
}
}
arena.root_ = j["root"].get<common_peg_parser_id>();
if (arena.root_ != COMMON_PEG_INVALID_PARSER_ID && arena.root_ >= arena.parsers_.size()) {
throw std::runtime_error("Root references invalid parser ID: " + std::to_string(arena.root_));
}
return arena;
}
std::string common_peg_arena::save() const {
return to_json().dump();
}
void common_peg_arena::load(const std::string & data) {
*this = from_json(common_json::parse(data));
}
common_peg_arena build_peg_parser(const std::function<common_peg_parser(common_peg_parser_builder & builder)> & fn) {
common_peg_parser_builder builder;
builder.set_root(fn(builder));
common/peg-parser.h
-6
View File
@@ -357,12 +357,6 @@ class common_peg_arena {
std::string dump(common_peg_parser_id id) const;
common_json to_json() const;
static common_peg_arena from_json(const common_json & j);
std::string save() const;
void load(const std::string & data);
friend class common_peg_parser_builder;
private:
common/sampling.cpp
+38
View File
@@ -1050,3 +1050,41 @@ std::vector<common_sampler_type> common_sampler_types_from_chars(const std::stri
return samplers;
}
void common_sampling_add_preserved_tokens(common_params_sampling & sampling, const llama_vocab * vocab, const std::vector<std::string> & tokens) {
GGML_ASSERT(vocab != nullptr);
for (const auto & t : tokens) {
auto ids = common_tokenize(vocab, t, false, true);
if (ids.size() == 1) {
sampling.preserved_tokens.insert(ids[0]);
}
}
}
void common_sampling_add_grammar_triggers(common_params_sampling & sampling, const llama_vocab * vocab, std::vector<common_grammar_trigger> triggers) {
GGML_ASSERT(vocab != nullptr);
for (auto & trigger : triggers) {
if (trigger.type == COMMON_GRAMMAR_TRIGGER_TYPE_WORD) {
const auto & word = trigger.value;
auto ids = common_tokenize(vocab, word, false, true);
if (ids.size() == 1) {
auto token = ids[0];
if (std::find(sampling.preserved_tokens.begin(), sampling.preserved_tokens.end(), (llama_token) token) == sampling.preserved_tokens.end()) {
throw std::runtime_error("Grammar trigger word should be marked as preserved token: " + word);
}
common_grammar_trigger token_trigger;
token_trigger.type = COMMON_GRAMMAR_TRIGGER_TYPE_TOKEN;
token_trigger.value = word;
token_trigger.token = token;
sampling.grammar_triggers.push_back(std::move(token_trigger));
} else {
sampling.grammar_triggers.push_back({COMMON_GRAMMAR_TRIGGER_TYPE_WORD, word});
}
} else {
sampling.grammar_triggers.push_back(std::move(trigger));
}
}
if (sampling.grammar_lazy && sampling.grammar_triggers.empty()) {
throw std::runtime_error("Error: no triggers set for lazy grammar!");
}
}
common/sampling.h
+6
View File
@@ -118,6 +118,12 @@ std::string common_sampler_type_to_str(enum common_sampler_type cnstr);
std::vector<enum common_sampler_type> common_sampler_types_from_names(const std::vector<std::string> & names);
std::vector<enum common_sampler_type> common_sampler_types_from_chars(const std::string & chars);
// add the strings that are a single token in the vocab to the preserved tokens
void common_sampling_add_preserved_tokens(common_params_sampling & sampling, const llama_vocab * vocab, const std::vector<std::string> & tokens);
// add grammar triggers, a trigger word that is a single token becomes a token trigger and must be a preserved token
void common_sampling_add_grammar_triggers(common_params_sampling & sampling, const llama_vocab * vocab, std::vector<common_grammar_trigger> triggers);
llama_sampler * llama_sampler_init_llg(const llama_vocab * vocab,
const char * grammar_kind, const char * grammar_data);
tests/CMakeLists.txt
-1
View File
@@ -245,7 +245,6 @@ llama_build_and_test(
peg-parser/test-basic.cpp
peg-parser/test-gbnf-generation.cpp
peg-parser/test-json-parser.cpp
peg-parser/test-json-serialization.cpp
peg-parser/test-python-dict-parser.cpp
peg-parser/test-unicode.cpp
peg-parser/tests.h
tests/peg-parser/test-json-serialization.cpp
-28
View File
@@ -1,28 +0,0 @@
#include "tests.h"
void test_json_serialization(testing &t) {
auto original = build_peg_parser([](common_peg_parser_builder & p) {
return "<tool_call>" + p.json() + "</tool_call>";
});
auto json_serialized = original.to_json().dump();
t.test("compare before/after", [&](testing &t) {
auto deserialized = common_peg_arena::from_json(common_json::parse(json_serialized));
// Test complex JSON
std::string input = R"({"name": "test", "values": [1, 2, 3], "nested": {"a": true}})";
common_peg_parse_context ctx1(input);
common_peg_parse_context ctx2(input);
auto result1 = original.parse(ctx1);
auto result2 = deserialized.parse(ctx2);
t.assert_equal("both_succeed", result1.success(), result2.success());
t.assert_equal("same_end_pos", result1.end, result2.end);
});
t.bench("deserialize", [&]() {
auto deserialized = common_peg_arena::from_json(common_json::parse(json_serialized));
}, 100);
}
tests/peg-parser/tests.h
-1
View File
@@ -21,5 +21,4 @@ void test_basic(testing &t);
void test_json_parser(testing &t);
void test_gbnf_generation(testing &t);
void test_unicode(testing &t);
void test_json_serialization(testing &t);
void test_python_dict_parser(testing &t);
tests/test-chat-auto-parser.cpp
+2 -3
View File
@@ -450,7 +450,7 @@ static int debug_single_template(const debug_options & opts) {
generation_params params = prepare_debug_params(opts, tools);
common_chat_params parser_data;
if (std::optional<common_chat_params> spec_tmpl =
common_chat_try_specialized_template(chat_template, template_source, params)) {
common_chat_try_specialized_template(chat_template, params)) {
LOG_ERR("\n");
LOG_ERR("This template uses a specialized parser, analysis results will not be available.\n");
parser_data = *spec_tmpl;
@@ -484,8 +484,7 @@ static int debug_single_template(const debug_options & opts) {
if (!std::empty(parser_data.parser)) {
LOG_ERR("\n=== Generated Parser ===\n");
common_peg_arena arena;
arena.load(parser_data.parser);
const common_peg_arena & arena = parser_data.parser;
LOG_ERR("%s\n", arena.dump(arena.root()).c_str());
LOG_ERR("\n=== Generated Grammar ===\n");
tests/test-chat.cpp
+114 -51
View File
@@ -1090,26 +1090,6 @@ struct peg_test_case {
std::vector<std::string> expect_rules;
};
struct make_peg_parser {
common_chat_params params_;
common_peg_arena arena_;
bool detailed_debug_;
make_peg_parser(common_chat_templates * tmpls,
const common_chat_templates_inputs & inputs,
bool detailed_debug = false) {
detailed_debug_ = detailed_debug;
params_ = common_chat_templates_apply(tmpls, inputs);
arena_.load(params_.parser);
}
common_chat_msg parse(const std::string & msg, bool is_partial) const {
common_chat_parser_params parser_params(params_);
parser_params.debug = detailed_debug_;
return common_chat_peg_parse(arena_, common_chat_input(msg), is_partial, parser_params);
}
};
// Global template filter for --template flag
static std::string g_template_filter;
@@ -1161,14 +1141,19 @@ static void test_peg_parser(common_chat_templates * tmpls,
tc.expect.role = "assistant";
}
auto parser = make_peg_parser(tmpls, tc.params, detailed_debug);
common_chat_session_params session_params;
session_params.debug = detailed_debug;
common_chat_session session(tmpls, nullptr, tc.params, session_params);
const auto & parser = session.parser();
common_params_sampling sampling;
session.apply_sampling(sampling);
if (detailed_debug) {
LOG_DBG("Using parser: \n%s\n", parser.arena_.dump(parser.arena_.root()).c_str());
LOG_DBG("Generation prompt: '%s'\n", parser.params_.generation_prompt.c_str());
LOG_DBG("Using parser: \n%s\n", parser.dump(parser.root()).c_str());
LOG_DBG("Generation prompt: '%s'\n", session.generation_prompt().c_str());
}
for (const auto & rule : tc.expect_rules) {
if (!parser.arena_.has_rule(rule)) {
if (!parser.has_rule(rule)) {
LOG_ERR("Missing rule: %s\n", rule.c_str());
common_log_flush(common_log_main());
throw std::runtime_error("Test failed");
@@ -1179,12 +1164,14 @@ static void test_peg_parser(common_chat_templates * tmpls,
common_chat_msg msg_prev;
msg_accum.role = msg_prev.role = "assistant";
size_t fed = 0;
for (size_t i = 1; i <= tc.input.size(); ++i) {
auto is_partial = i < tc.input.size() || tc.is_partial;
// Use UTF-8 safe truncation to avoid corrupting multi-byte characters
size_t safe_len = utf8_truncate_safe_len(std::string_view(tc.input).substr(0, i));
std::string prefix = tc.input.substr(0, safe_len);
common_chat_msg msg_current = parser.parse(prefix, is_partial);
common_chat_input chunk(tc.input.substr(fed, safe_len - fed));
fed = safe_len;
const common_chat_msg & msg_current = is_partial ? session.feed(chunk) : session.finish(chunk);
for (const auto & diff : common_chat_msg_diff::compute_diffs(msg_prev, msg_current)) {
if (!diff.reasoning_content_delta.empty()) {
@@ -1222,30 +1209,33 @@ static void test_peg_parser(common_chat_templates * tmpls,
}
if (!tc.is_partial) {
assert_msg_equals(tc.expect, parser.parse(tc.input, false), true);
if (tc.input.empty()) {
session.finish();
}
assert_msg_equals(tc.expect, session.msg(), true);
}
assert_msg_equals(tc.expect, msg_accum, true);
// A response format must be enforced by an eager grammar
if (!tc.params.json_schema.empty()) {
if (parser.params_.grammar.empty()) {
if (session.grammar().empty()) {
throw std::runtime_error("json_schema is set but no grammar was produced");
}
if (parser.params_.grammar_lazy) {
if (sampling.grammar_lazy) {
throw std::runtime_error("json_schema is set but the grammar is lazy");
}
}
// Test grammar if present in params
if (!parser.params_.grammar.empty()) {
auto grammar = build_grammar(parser.params_.grammar);
if (!session.grammar().empty()) {
auto grammar = build_grammar(session.grammar());
if (!grammar) {
throw std::runtime_error("Failed to build grammar: " + parser.params_.grammar);
throw std::runtime_error("Failed to build grammar: " + session.grammar());
}
// In production, grammar triggers match against the full generated text
// including the generation prompt. All positions are in full_input coordinates.
const auto & gen_prompt = parser.params_.generation_prompt;
const auto & gen_prompt = sampling.generation_prompt;
std::string full_input = gen_prompt + tc.input;
// Determine whether the reasoning-budget sampler path applies: tool-call grammar
@@ -1253,9 +1243,9 @@ static void test_peg_parser(common_chat_templates * tmpls,
// budget sampler inhibits grammar application while inside thinking blocks —
// triggers inside <think>...</think> are suppressed.
bool use_reasoning_budget_path = false;
if (parser.params_.grammar_lazy && !parser.params_.thinking_end_tags.empty()) {
if (sampling.grammar_lazy && !session.thinking_end_tags().empty()) {
use_reasoning_budget_path = true;
for (const auto & trigger : parser.params_.grammar_triggers) {
for (const auto & trigger : sampling.grammar_triggers) {
if (trigger.type != COMMON_GRAMMAR_TRIGGER_TYPE_WORD) {
use_reasoning_budget_path = false;
break;
@@ -1270,8 +1260,8 @@ static void test_peg_parser(common_chat_templates * tmpls,
// Reasoning-budget path: simulate thinking-aware trigger detection.
// Walk through full_input tracking thinking state; only match triggers
// when outside thinking blocks.
const auto & think_start = parser.params_.thinking_start_tag;
const auto & think_ends = parser.params_.thinking_end_tags;
const auto & think_start = session.thinking_start_tag();
const auto & think_ends = session.thinking_end_tags();
bool in_thinking = false;
for (size_t i = 0; i < full_input.size(); ++i) {
@@ -1292,7 +1282,7 @@ static void test_peg_parser(common_chat_templates * tmpls,
continue;
}
// Outside thinking — check if any trigger word starts here
for (const auto & trigger : parser.params_.grammar_triggers) {
for (const auto & trigger : sampling.grammar_triggers) {
if (full_input.compare(i, trigger.value.size(), trigger.value) == 0) {
if (earliest_trigger_pos == std::string::npos || i < earliest_trigger_pos) {
earliest_trigger_pos = i;
@@ -1314,7 +1304,7 @@ static void test_peg_parser(common_chat_templates * tmpls,
if (!use_reasoning_budget_path) {
// Legacy path: find triggers without thinking-awareness
for (const auto & trigger : parser.params_.grammar_triggers) {
for (const auto & trigger : sampling.grammar_triggers) {
size_t pos = std::string::npos;
std::smatch match;
switch (trigger.type) {
@@ -1370,16 +1360,16 @@ static void test_peg_parser(common_chat_templates * tmpls,
// If the test expects tool calls and the grammar is lazy, the trigger must fire.
// Otherwise the grammar would never activate in production and tool calls wouldn't
// be constrained. A silent skip here would hide broken triggers.
if (parser.params_.grammar_lazy && !tc.expect.tool_calls.empty() && !tc.is_partial
if (sampling.grammar_lazy && !tc.expect.tool_calls.empty() && !tc.is_partial
&& earliest_trigger_pos == std::string::npos) {
std::string trigger_desc;
for (const auto & trigger : parser.params_.grammar_triggers) {
for (const auto & trigger : sampling.grammar_triggers) {
trigger_desc += "\n [type=" + std::to_string(trigger.type) + "] " + trigger.value;
}
throw std::runtime_error(
"Grammar trigger did not fire, but test expects tool calls (lazy grammar).\n"
">>> Input: " + full_input + "\n"
">>> Triggers (" + std::to_string(parser.params_.grammar_triggers.size()) + "):" + trigger_desc);
">>> Triggers (" + std::to_string(sampling.grammar_triggers.size()) + "):" + trigger_desc);
}
// Determine the constrained portion of input to test against grammar.
@@ -1392,7 +1382,7 @@ static void test_peg_parser(common_chat_templates * tmpls,
auto constrain_from = std::max(earliest_trigger_pos, gen_prompt.size());
constrained = full_input.substr(constrain_from);
grammar_triggered = true;
} else if (!parser.params_.grammar_lazy) {
} else if (!sampling.grammar_lazy) {
// For non-lazy grammars, the entire input should match
grammar_triggered = true;
}
@@ -1411,7 +1401,7 @@ static void test_peg_parser(common_chat_templates * tmpls,
std::to_string(result.matched_codepoints) + " codepoints): " +
(result.matched_prefix.size() > 100 ? result.matched_prefix.substr(0, 100) + "..." : result.matched_prefix) +
"\n\n>>> Expected next: " + result.expected_description +
"\n\n>>> Grammar: " + parser.params_.grammar;
"\n\n>>> Grammar: " + session.grammar();
} else {
error_msg =
"Grammar match failed:\n\n"
@@ -1422,7 +1412,7 @@ static void test_peg_parser(common_chat_templates * tmpls,
(result.matched_prefix.size() > 100 ? result.matched_prefix.substr(0, 100) + "..." : result.matched_prefix) +
"\n\n>>> Failing character: " + result.failing_char +
"\n\n>>> Expected: " + result.expected_description +
"\n\n>>> Grammar: " + parser.params_.grammar;
"\n\n>>> Grammar: " + session.grammar();
}
throw std::runtime_error(error_msg);
}
@@ -1437,7 +1427,7 @@ static void test_peg_parser(common_chat_templates * tmpls,
// Start from tc.expect but copy tool call arguments from the actual parser
// output, which preserves original JSON formatting (e.g. {"arg1":1} vs {"arg1": 1}).
auto reconstruction_msg = tc.expect;
auto parsed_msg = parser.parse(tc.input, false);
const auto & parsed_msg = session.msg();
for (size_t i = 0; i < reconstruction_msg.tool_calls.size() && i < parsed_msg.tool_calls.size(); i++) {
reconstruction_msg.tool_calls[i].arguments = parsed_msg.tool_calls[i].arguments;
}
@@ -1446,7 +1436,7 @@ static void test_peg_parser(common_chat_templates * tmpls,
reconstruction_inputs.add_generation_prompt = false;
auto reconstruction_params = common_chat_templates_apply(tmpls, reconstruction_inputs);
std::string expected_text = parser.params_.prompt + tc.input;
std::string expected_text = session.prompt() + tc.input;
bool match = reconstruction_params.prompt == expected_text ||
(reconstruction_params.prompt.size() > expected_text.size() &&
reconstruction_params.prompt.compare(0, expected_text.size(), expected_text) == 0);
@@ -4644,8 +4634,7 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
inputs.messages = { msg };
auto params = common_chat_templates_apply(tmpls.get(), inputs);
common_peg_arena arena;
arena.load(params.parser);
const common_peg_arena & arena = params.parser;
common_chat_parser_params pp(params);
// generation_prompt is non-empty for thinking models, so result.end
@@ -7791,6 +7780,77 @@ static void test_deepseek_v4_tool_result_ordering() {
}
}
static void test_chat_session() {
LOG_DBG("%s\n", __func__);
auto tmpls = read_templates("models/templates/Qwen3.5-4B.jinja");
common_chat_templates_inputs inputs;
inputs.messages = { message_user };
inputs.tools = { special_function_tool };
inputs.reasoning_format = COMMON_REASONING_FORMAT_AUTO;
const std::string output =
"I'm\nthinking\n</think>\n\n"
"<tool_call>\n"
"<function=special_function>\n"
"<parameter=arg1>\n1\n</parameter>\n"
"</function>\n"
"</tool_call>";
// the session renders the prompt and parses the output fed to it in small chunks
{
common_chat_session session(tmpls.get(), nullptr, inputs);
assert_contains(session.prompt(), "\"name\": \"special_function\"");
assert_contains(session.prompt(), "<|im_start|>user\nHey there!<|im_end|>\n<|im_start|>assistant\n<think>\n");
assert_equals(false, session.grammar().empty());
assert_equals(std::string("<|im_start|>assistant\n<think>\n"), session.generation_prompt());
const std::string thinking = "I'm\nthinking\n</think>\n\n";
for (size_t i = 0; i < thinking.size(); i += 3) {
session.feed(common_chat_input(thinking.substr(i, 3)));
}
assert_msg_equals(simple_assist_msg("", "I'm\nthinking\n"), session.msg());
for (size_t i = thinking.size(); i < output.size(); i += 3) {
session.feed(common_chat_input(output.substr(i, 3)));
}
assert_msg_equals(simple_assist_msg("", "I'm\nthinking\n", "special_function", "{\"arg1\":1}"), session.finish());
}
// a copy does not see what is fed to the original
{
common_chat_session a(tmpls.get(), nullptr, inputs);
common_chat_session b = a;
a.feed(common_chat_input(output));
assert_equals(true, b.msg().empty());
b.feed(common_chat_input("I'm\nthinking\n</think>\n\nHello"));
assert_equals(std::string("Hello"), b.msg().content);
assert_equals(std::string("special_function"), a.finish().tool_calls.at(0).name);
}
// a continued message starts from the prefill, unless it is echoed
{
common_chat_templates_inputs cont;
cont.messages = { message_user, message_assist_prefill_content };
cont.add_generation_prompt = false;
cont.continue_final_message = COMMON_CHAT_CONTINUATION_CONTENT;
cont.reasoning_format = COMMON_REASONING_FORMAT_AUTO;
common_chat_session session(tmpls.get(), nullptr, cont);
assert_equals(std::string("<|im_start|>assistant\n<think>\nI'm thinking\n</think>\n\nHello, "),
session.generation_prompt());
assert_msg_equals(simple_assist_msg("Hello, ", "I'm thinking\n"), session.msg());
session.feed(common_chat_input("world!"));
assert_equals(std::string("Hello, world!"), session.msg().content);
common_chat_session_params echo;
echo.echo = true;
common_chat_session echoed(tmpls.get(), nullptr, cont, echo);
assert_equals(true, echoed.msg().empty());
}
}
static void test_reasoning_budget_tokens_per_request() {
LOG_DBG("%s\n", __func__);
// Use Qwen3 template which has <think>...</think> reasoning markers.
@@ -7811,7 +7871,8 @@ static void test_reasoning_budget_tokens_per_request() {
{"reasoning_budget_tokens", 0},
};
std::vector<raw_buffer> out_files;
auto llama_params = oaicompat_chat_params_parse(body, opt, out_files);
common_chat_session out_session;
auto llama_params = oaicompat_chat_params_parse(nullptr, body, opt, out_files, out_session);
// The per-request value must win over the server default (-1).
if (!llama_params.contains("reasoning_budget_tokens")) {
@@ -7844,7 +7905,8 @@ static void test_reasoning_budget_message_per_request() {
{"reasoning_budget_message", per_request_message},
};
std::vector<raw_buffer> out_files;
auto llama_params = oaicompat_chat_params_parse(body, opt, out_files);
common_chat_session out_session;
auto llama_params = oaicompat_chat_params_parse(nullptr, body, opt, out_files, out_session);
// The per-request value must win over the server default.
if (!llama_params.contains("reasoning_budget_message")) {
@@ -8035,6 +8097,7 @@ int main(int argc, char ** argv) {
test_deepseek_v4_tool_result_ordering();
test_template_generation_prompt();
test_reasoning_effort_caps();
test_chat_session();
test_reasoning_budget_tokens_per_request();
test_reasoning_budget_message_per_request();
test_template_output_peg_parsers(detailed_debug);
tests/test-peg-parser.cpp
-1
View File
@@ -19,7 +19,6 @@ int main(int argc, char *argv[]) {
t.test("unicode", test_unicode);
t.test("json", test_json_parser);
t.test("gbnf", test_gbnf_generation);
t.test("serialization", test_json_serialization);
t.test("python-dict", test_python_dict_parser);
return t.summary();
tools/server/README.md
-4
View File
@@ -1353,10 +1353,6 @@ The `response_format` parameter supports both plain JSON output (e.g. `{"type":
`reasoning_control`: Arms realtime reasoning control for this completion so it can be ended early via `/v1/chat/completions/control`. Defaults to `false`.
`generation_prompt`: The generation prompt that was prefilled in by the template. Prepended to model output before parsing.
`parse_tool_calls`: Whether to parse the generated tool call.
`parallel_tool_calls` : Whether to enable parallel/multiple tool calls (only supported on some models, verification is based on jinja template).
For multimodal input (typed content, `messages[i].content[j]`):
tools/server/server-common.cpp
+19 -29
View File
@@ -1266,9 +1266,11 @@ server_tokens tokenize_oai_content_array(const llama_vocab * vocab, mtmd_context
// used by /chat/completions endpoint
json oaicompat_chat_params_parse(
const llama_vocab * vocab,
json & body, /* openai api json semantics */
const server_chat_params & opt,
std::vector<raw_buffer> & out_files)
std::vector<raw_buffer> & out_files,
common_chat_session & out_session)
{
json llama_params;
@@ -1393,7 +1395,6 @@ json oaicompat_chat_params_parse(
if (body.contains("grammar")) {
throw std::invalid_argument("Cannot use custom grammar constraints with tools.");
}
llama_params["parse_tool_calls"] = true;
}
// merge the template args provided from command line with the args provided in the user request
@@ -1427,31 +1428,11 @@ json oaicompat_chat_params_parse(
inputs.force_pure_content = opt.force_pure_content;
// Apply chat template to the list of messages
auto chat_params = common_chat_templates_apply(opt.tmpls.get(), inputs);
common_chat_session_params session_params;
session_params.echo = json_value(body, "echo", false);
out_session = common_chat_session(opt.tmpls.get(), vocab, inputs, session_params);
llama_params["chat_format"] = static_cast<int>(chat_params.format);
llama_params["prompt"] = chat_params.prompt;
if (!chat_params.grammar.empty()) {
llama_params["grammar"] = chat_params.grammar;
llama_params["grammar_type"] = std::string("tool_calls");
}
llama_params["grammar_lazy"] = chat_params.grammar_lazy;
auto grammar_triggers = json::array();
for (const auto & trigger : chat_params.grammar_triggers) {
server_grammar_trigger ct(trigger);
grammar_triggers.push_back(ct.to_json());
}
llama_params["grammar_triggers"] = grammar_triggers;
llama_params["preserved_tokens"] = chat_params.preserved_tokens;
llama_params["generation_prompt"] = chat_params.generation_prompt;
for (const auto & stop : chat_params.additional_stops) {
llama_params["stop"].push_back(stop);
}
if (!chat_params.parser.empty()) {
llama_params["chat_parser"] = chat_params.parser;
}
llama_params["message_delimiters"] = chat_params.message_delimiters.to_json();
llama_params["prompt"] = out_session.prompt();
// Reasoning budget: pass parameters through to sampling layer
{
@@ -1461,10 +1442,10 @@ json oaicompat_chat_params_parse(
reasoning_budget = opt.reasoning_budget;
}
if (!chat_params.thinking_end_tags.empty()) {
if (!out_session.thinking_end_tags().empty()) {
llama_params["reasoning_budget_tokens"] = reasoning_budget;
llama_params["reasoning_budget_start_tag"] = chat_params.thinking_start_tag;
llama_params["reasoning_budget_end_tags"] = chat_params.thinking_end_tags;
llama_params["reasoning_budget_start_tag"] = out_session.thinking_start_tag();
llama_params["reasoning_budget_end_tags"] = out_session.thinking_end_tags();
llama_params["reasoning_budget_message"] = json_value(body, "reasoning_budget_message", opt.reasoning_budget_message);
llama_params["reasoning_control"] = json_value(body, "reasoning_control", false);
}
@@ -1491,6 +1472,15 @@ json oaicompat_chat_params_parse(
}
}
// the session owns these, the server applies them with server_task::apply_chat_session()
for (const char * key : { "grammar_lazy", "grammar_triggers", "preserved_tokens" }) {
llama_params.erase(key);
}
if (!out_session.grammar().empty()) {
llama_params.erase("grammar");
llama_params.erase("json_schema");
}
return llama_params;
}
tools/server/server-common.h
+3 -1
View File
@@ -355,9 +355,11 @@ json oaicompat_completion_params_parse(const json & body);
// used by /chat/completions endpoint
json oaicompat_chat_params_parse(
const llama_vocab * vocab,
json & body, /* openai api json semantics */
const server_chat_params & opt,
std::vector<raw_buffer> & out_files);
std::vector<raw_buffer> & out_files,
common_chat_session & out_session);
// used by /embeddings endpoint, content has the same format as a chat message content array
server_tokens tokenize_oai_content_array(
tools/server/server-context.cpp
+36 -18
View File
@@ -4767,7 +4767,8 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
server_task_type type,
const json & data,
const std::vector<raw_buffer> & files,
task_response_type res_type) {
task_response_type res_type,
const common_chat_session & chat_session) {
GGML_ASSERT(type == SERVER_TASK_TYPE_COMPLETION || type == SERVER_TASK_TYPE_INFILL);
auto res = create_response();
@@ -4809,11 +4810,6 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
// tasks.reserve(inputs.size()); // TODO: this is inaccurate due to child tasks
// message delimiters for checkpointing
json delims = json_value(data, "message_delimiters", json::array());
auto delimiters = common_chat_msg_delimiters_parse(delims);
delimiters.tokenize(ctx_server.vocab);
for (size_t i = 0; i < inputs.size(); i++) {
server_task task = server_task(type);
@@ -4826,7 +4822,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
meta->logit_bias_eog,
data);
task.params.message_spans = task.tokens.find_message_spans(delimiters);
task.apply_chat_session(chat_session);
task.id_slot = json_value(data, "id_slot", -1);
sse_ping_interval = task.params.sse_ping_interval;
@@ -4847,7 +4843,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
tasks.push_back(std::move(task));
}
rd.post_tasks(std::move(tasks));
rd.post_tasks(std::move(tasks), chat_session);
} catch (const std::exception & e) {
res->error(format_error_response(e.what(), ERROR_TYPE_INVALID_REQUEST));
return res;
@@ -5444,16 +5440,20 @@ void server_routes::init_routes() {
auto res = create_response();
std::vector<raw_buffer> files;
json body = json::parse(req.body);
common_chat_session session;
json body_parsed = oaicompat_chat_params_parse(
ctx_server.vocab,
body,
meta->chat_params,
files);
files,
session);
return handle_completions_impl(
req,
SERVER_TASK_TYPE_COMPLETION,
body_parsed,
files,
TASK_RESPONSE_TYPE_OAI_CHAT);
TASK_RESPONSE_TYPE_OAI_CHAT,
session);
};
this->post_chat_completions_tok = [this](const server_http_req & req) {
@@ -5503,16 +5503,20 @@ void server_routes::init_routes() {
json body = server_chat_convert_responses_to_chatcmpl(json::parse(req.body));
SRV_DBG("%s\n", "Request converted: OpenAI Responses -> OpenAI Chat Completions");
SRV_DBG("converted request: %s\n", body.dump().c_str());
common_chat_session session;
json body_parsed = oaicompat_chat_params_parse(
ctx_server.vocab,
body,
meta->chat_params,
files);
files,
session);
return handle_completions_impl(
req,
SERVER_TASK_TYPE_COMPLETION,
body_parsed,
files,
TASK_RESPONSE_TYPE_OAI_RESP);
TASK_RESPONSE_TYPE_OAI_RESP,
session);
};
this->post_responses_tok_oai = [this](const server_http_req & req) {
@@ -5535,16 +5539,20 @@ void server_routes::init_routes() {
files);
SRV_DBG("%s\n", "Request converted: OpenAI Transcriptions -> OpenAI Chat Completions");
SRV_DBG("converted request: %s\n", body.dump().c_str());
common_chat_session session;
json body_parsed = oaicompat_chat_params_parse(
ctx_server.vocab,
body,
meta->chat_params,
files);
files,
session);
return handle_completions_impl(
req,
SERVER_TASK_TYPE_COMPLETION,
body_parsed,
files,
TASK_RESPONSE_TYPE_OAI_ASR);
TASK_RESPONSE_TYPE_OAI_ASR,
session);
};
this->post_anthropic_messages = [this](const server_http_req & req) {
@@ -5553,16 +5561,20 @@ void server_routes::init_routes() {
json body = server_chat_convert_anthropic_to_oai(json::parse(req.body));
SRV_DBG("%s\n", "Request converted: Anthropic -> OpenAI Chat Completions");
SRV_DBG("converted request: %s\n", body.dump().c_str());
common_chat_session session;
json body_parsed = oaicompat_chat_params_parse(
ctx_server.vocab,
body,
meta->chat_params,
files);
files,
session);
return handle_completions_impl(
req,
SERVER_TASK_TYPE_COMPLETION,
body_parsed,
files,
TASK_RESPONSE_TYPE_ANTHROPIC);
TASK_RESPONSE_TYPE_ANTHROPIC,
session);
};
this->post_anthropic_count_tokens = [this](const server_http_req & req) {
@@ -5573,11 +5585,14 @@ void server_routes::init_routes() {
this->post_apply_template = [this](const server_http_req & req) {
auto res = create_response();
std::vector<raw_buffer> files; // dummy, unused
common_chat_session session; // dummy, unused
json body = json::parse(req.body);
json data = oaicompat_chat_params_parse(
ctx_server.vocab,
body,
meta->chat_params,
files);
files,
session);
res->ok({{ "prompt", std::move(data.at("prompt")) }});
return res;
};
@@ -6129,10 +6144,13 @@ std::unique_ptr<server_res_generator> server_routes::handle_count_tokens(const s
return res;
}
common_chat_session session; // dummy, unused
json body_parsed = oaicompat_chat_params_parse(
ctx_server.vocab,
body,
meta->chat_params,
files);
files,
session);
json prompt = body_parsed.at("prompt");
// SRV_DBG("prompt = %s\n", prompt.dump().c_str());
tools/server/server-context.h
+2 -1
View File
@@ -166,7 +166,8 @@ private:
server_task_type type,
const json & data,
const std::vector<raw_buffer> & files,
task_response_type res_type);
task_response_type res_type,
const common_chat_session & chat_session = {});
std::unique_ptr<server_res_generator> handle_slots_save(const server_http_req & req, int id_slot);
std::unique_ptr<server_res_generator> handle_slots_restore(const server_http_req & req, int id_slot);
std::unique_ptr<server_res_generator> handle_slots_erase(const server_http_req &, int id_slot);
tools/server/server-queue.cpp
+4 -4
View File
@@ -517,23 +517,23 @@ void server_response_reader::post_task(server_task && task, bool front) {
GGML_ASSERT(!task.is_parent() && "not supported, use post_tasks() instead");
task.index = 0;
id_tasks.insert(task.id);
states.push_back(task.create_state());
states.push_back(task_result_state());
queue_results.add_waiting_task_id(task.id);
queue_tasks.post(std::move(task), front);
}
void server_response_reader::post_tasks(std::vector<server_task> && tasks, bool front) {
void server_response_reader::post_tasks(std::vector<server_task> && tasks, const common_chat_session & session, bool front) {
GGML_ASSERT(id_tasks.empty() && "post_tasks() can only be called once per reader");
id_tasks = server_task::get_list_id(tasks);
states.reserve(tasks.size());
size_t index = 0;
for (auto & task : tasks) {
task.index = index++;
states.push_back(task.create_state());
states.push_back(task_result_state(session));
// for child tasks
for (auto & child_task : task.child_tasks) {
child_task.index = index++;
states.push_back(child_task.create_state());
states.push_back(task_result_state(session));
}
}
GGML_ASSERT(states.size() == id_tasks.size());
tools/server/server-queue.h
+1 -1
View File
@@ -225,7 +225,7 @@ struct server_response_reader {
// if front = true, the task will be posted to the front of the queue (high priority)
void post_task(server_task && task, bool front = false);
void post_tasks(std::vector<server_task> && tasks, bool front = false);
void post_tasks(std::vector<server_task> && tasks, const common_chat_session & session = {}, bool front = false);
bool has_next() const;
// return nullptr if should_stop() is true before receiving a result
tools/server/server-schema.cpp
+13 -78
View File
@@ -291,54 +291,23 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params &
// Chat parser params
//
// TODO: change this to string field instead
add((new field_json("chat_format"))
->set_desc("Chat format used internally by the server")
->set_handler([&](field_eval_context & ctx, const json & data) {
ctx.params.chat_parser_params.format = static_cast<common_chat_format>(data.at("chat_format").get<int>());
SRV_TRC("chat format: %s\n", common_chat_format_name(ctx.params.chat_parser_params.format));
}));
add((new field_str("reasoning_format"))
->set_desc("Reasoning format for chain-of-thought models")
->set_handler([&](field_eval_context & ctx, const json & data) {
auto reasoning_format = common_reasoning_format_from_name(data.at("reasoning_format").get<std::string>());
ctx.params.chat_parser_params.reasoning_format = reasoning_format;
ctx.params.chat_parser_params.reasoning_in_content = ctx.params.stream && (reasoning_format == COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY);
}));
add((new field_str("generation_prompt"))
->set_desc("Generation prompt appended to the chat template output")
->set_handler([&](field_eval_context & ctx, const json & data) {
std::string s = data.at("generation_prompt").get<std::string>();
ctx.params.sampling.generation_prompt = s;
if (ctx.vocab == nullptr) {
ctx.params.chat_parser_params.generation_prompt = common_chat_input(s);
return;
}
ctx.params.chat_parser_params.generation_prompt = common_chat_input_tokenize(ctx.vocab, s);
}));
add((new field_bool("parse_tool_calls", params.chat_parser_params.parse_tool_calls))
->set_desc("Whether to parse tool calls from the generated output"));
add((new field_str("chat_parser"))
->set_desc("Chat parser configuration string")
->set_handler([&](field_eval_context & ctx, const json & data) {
ctx.params.chat_parser_params.parser.load(data.at("chat_parser").get<std::string>());
ctx.params.reasoning_format = common_reasoning_format_from_name(data.at("reasoning_format").get<std::string>());
}));
add((new field_json("continue_final_message"))
->set_desc("Whether to continue the final message of the chat template")
->set_handler([&](field_eval_context & ctx, const json & data) {
auto continuation = common_chat_continuation_parse(data.at("continue_final_message"));
ctx.params.chat_parser_params.is_continuation = continuation != COMMON_CHAT_CONTINUATION_NONE;
->set_handler([&](field_eval_context &, const json & data) {
common_chat_continuation_parse(data.at("continue_final_message"));
}));
add((new field_bool("echo", params.chat_parser_params.echo))
->set_desc("Whether to echo the input tokens in the output"));
add((new field_json("echo"))
->set_desc("Whether to include the continued assistant message in the output")
->set_handler([&](field_eval_context &, const json & data) {
data.at("echo").get<bool>();
}));
//
// Token-level fields (require vocab)
@@ -347,44 +316,17 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params &
add((new field_json("preserved_tokens"))
->set_desc("List of token strings that must not be split during tokenization")
->set_handler([&](field_eval_context & ctx, const json & data) {
GGML_ASSERT(ctx.vocab != nullptr);
for (const auto & t : data.at("preserved_tokens")) {
auto ids = common_tokenize(ctx.vocab, t.get<std::string>(), false, true);
if (ids.size() == 1) {
ctx.params.sampling.preserved_tokens.insert(ids[0]);
}
}
common_sampling_add_preserved_tokens(ctx.params.sampling, ctx.vocab, data.at("preserved_tokens").get<std::vector<std::string>>());
}));
add((new field_json("grammar_triggers"))
->set_desc("List of strings or patterns that trigger grammar-constrained generation")
->set_handler([&](field_eval_context & ctx, const json & data) {
GGML_ASSERT(ctx.vocab != nullptr);
std::vector<common_grammar_trigger> triggers;
for (const auto & t : data.at("grammar_triggers")) {
server_grammar_trigger ct(t);
if (ct.value.type == COMMON_GRAMMAR_TRIGGER_TYPE_WORD) {
const auto & word = ct.value.value;
auto ids = common_tokenize(ctx.vocab, word, false, true);
if (ids.size() == 1) {
auto token = ids[0];
if (std::find(ctx.params.sampling.preserved_tokens.begin(), ctx.params.sampling.preserved_tokens.end(), (llama_token) token) == ctx.params.sampling.preserved_tokens.end()) {
throw std::runtime_error("Grammar trigger word should be marked as preserved token: " + word);
}
common_grammar_trigger trigger;
trigger.type = COMMON_GRAMMAR_TRIGGER_TYPE_TOKEN;
trigger.value = word;
trigger.token = token;
ctx.params.sampling.grammar_triggers.push_back(std::move(trigger));
} else {
ctx.params.sampling.grammar_triggers.push_back({COMMON_GRAMMAR_TRIGGER_TYPE_WORD, word});
}
} else {
ctx.params.sampling.grammar_triggers.emplace_back(std::move(ct.value));
}
}
if (ctx.params.sampling.grammar_lazy && ctx.params.sampling.grammar_triggers.empty()) {
throw std::runtime_error("Error: no triggers set for lazy grammar!");
triggers.push_back(server_grammar_trigger(t).value);
}
common_sampling_add_grammar_triggers(ctx.params.sampling, ctx.vocab, std::move(triggers));
}));
add((new field_bool("reasoning_control", params.sampling.reasoning_control))
@@ -542,7 +484,7 @@ task_params eval_llama_cmpl_schema(
// enabling this will output extra debug information in the HTTP responses from the server
params.verbose = params_base.verbosity > 9;
params.chat_parser_params.reasoning_format = params_base.reasoning_format;
params.reasoning_format = params_base.reasoning_format;
// create context and schema
field_eval_context ctx(params);
@@ -556,13 +498,6 @@ task_params eval_llama_cmpl_schema(
f->eval(ctx, data);
}
// post-processing
{
// if "reasoning_format" is not provided, its handler will not be called, we will need to handle it here
auto reasoning_format = params.chat_parser_params.reasoning_format;
params.chat_parser_params.reasoning_in_content = params.stream && (reasoning_format == COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY);
}
// debugging
{
auto budget = params.sampling.reasoning_budget_tokens;
tools/server/server-task.cpp
+12 -20
View File
@@ -73,10 +73,10 @@ json task_params::to_json(bool only_metrics) const {
{"stream", stream},
{"n_probs", sampling.n_probs},
{"min_keep", sampling.min_keep},
{"chat_format", common_chat_format_name(chat_parser_params.format)},
{"reasoning_format", common_reasoning_format_name(chat_parser_params.reasoning_format)},
{"reasoning_in_content", chat_parser_params.reasoning_in_content},
{"generation_prompt", chat_parser_params.generation_prompt.text},
{"chat_format", common_chat_format_name(chat_format)},
{"reasoning_format", common_reasoning_format_name(reasoning_format)},
{"reasoning_in_content", stream && reasoning_format == COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY},
{"generation_prompt", sampling.generation_prompt},
{"samplers", samplers},
{"speculative.types", common_speculative_type_name_str(speculative.types)},
{"timings_per_token", timings_per_token},
@@ -132,10 +132,10 @@ json task_params::to_json(bool only_metrics) const {
{"grammar_lazy", sampling.grammar_lazy},
{"grammar_triggers", grammar_triggers},
{"preserved_tokens", sampling.preserved_tokens},
{"chat_format", common_chat_format_name(chat_parser_params.format)},
{"reasoning_format", common_reasoning_format_name(chat_parser_params.reasoning_format)},
{"reasoning_in_content", chat_parser_params.reasoning_in_content},
{"generation_prompt", chat_parser_params.generation_prompt.text},
{"chat_format", common_chat_format_name(chat_format)},
{"reasoning_format", common_reasoning_format_name(reasoning_format)},
{"reasoning_in_content", stream && reasoning_format == COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY},
{"generation_prompt", sampling.generation_prompt},
{"samplers", samplers},
{"speculative.types", common_speculative_type_name_str(speculative.types)},
{"timings_per_token", timings_per_token},
@@ -148,15 +148,12 @@ json task_params::to_json(bool only_metrics) const {
//
// task_result_state
//
task_result_state::task_result_state(const common_chat_parser_params & chat_parser_params)
: chat_parser_params(chat_parser_params)
task_result_state::task_result_state(common_chat_session session)
: chat_session(std::move(session))
, chat_msg(chat_session.msg())
, oai_resp_id("resp_" + random_string())
, oai_resp_reasoning_id("rs_" + random_string())
, oai_resp_message_id("msg_" + random_string()) {
if (chat_parser_params.is_continuation && !chat_parser_params.echo) {
// initialize chat_msg to avoid emitting a delta containing the assistant prefill
chat_msg = common_chat_parse(generated_input, true, chat_parser_params);
}
}
common_chat_msg task_result_state::update_chat_msg(
@@ -164,13 +161,8 @@ common_chat_msg task_result_state::update_chat_msg(
bool is_partial,
std::vector<common_chat_msg_diff> & diffs,
bool filter_tool_calls) {
generated_input.append(added);
auto msg_prv_copy = chat_msg;
//SRV_DBG("Parsing chat message: %s\n", generated_input.text.c_str());
auto new_msg = common_chat_parse(
generated_input,
is_partial,
chat_parser_params);
auto new_msg = is_partial ? chat_session.feed(added) : chat_session.finish(added);
if (!new_msg.empty()) {
new_msg.set_tool_call_ids(generated_tool_call_ids, gen_tool_call_id);
chat_msg = new_msg;
tools/server/server-task.h
+16 -11
View File
@@ -89,8 +89,9 @@ struct task_params {
std::string control_action;
std::string control_cmpl_id;
// per-request parameters for chat parsing
common_chat_parser_params chat_parser_params;
// reported in generation_settings, parsing itself is owned by the chat session
common_chat_format chat_format = COMMON_CHAT_FORMAT_CONTENT_ONLY;
common_reasoning_format reasoning_format = COMMON_REASONING_FORMAT_NONE;
// message spans for checkpointing
common_chat_msg_spans message_spans;
@@ -106,9 +107,8 @@ struct task_params {
struct task_result_state {
// tracking diffs for partial tool calls
std::vector<common_chat_msg_diff> diffs;
common_chat_parser_params chat_parser_params;
common_chat_session chat_session; // owns all parsing for this generation
common_chat_msg chat_msg;
common_chat_input generated_input; // append new chunks of generated text here
std::vector<std::string> generated_tool_call_ids;
std::unordered_set<size_t> sent_tool_call_names;
@@ -124,7 +124,7 @@ struct task_result_state {
const std::string oai_resp_message_id;
std::string oai_resp_fc_id; // function call ID for current args delta
task_result_state(const common_chat_parser_params & chat_parser_params);
task_result_state(common_chat_session session = {});
// parse partial tool calls and update the internal state
common_chat_msg update_chat_msg(
@@ -258,6 +258,17 @@ struct server_task {
return ids;
}
void apply_chat_session(const common_chat_session & session) {
if (!session.has_template()) {
return;
}
session.apply_sampling(params.sampling);
params.chat_format = session.format();
params.antiprompt.insert(params.antiprompt.end(), session.additional_stops().begin(), session.additional_stops().end());
params.message_spans = tokens.find_message_spans(session.message_delimiters());
}
void add_child(int id_parent, int id_child) {
server_task copy;
@@ -277,12 +288,6 @@ struct server_task {
child_tasks.push_back(std::move(copy));
}
// the task will be moved into queue, then onto slots
// however, the state must be kept by caller (e.g., HTTP thread)
task_result_state create_state() const {
return task_result_state(params.chat_parser_params);
}
bool is_parent() const {
return child_tasks.size() > 0;
}