darthrubik/llama-cpp
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

server : add POST /tts endpoint for TTS models (#26603)
Copilot Setup Steps / copilot-setup-steps (push) Has been cancelled
Check Pre-Tokenizer Hashes / pre-tokenizer-hashes (push) Has been cancelled
Python check requirements.txt / check-requirements (push) Has been cancelled
Python Type-Check / python type-check (push) Has been cancelled
Update Operations Documentation / update-ops-docs (push) Has been cancelled

Browse Source 
* mtmd: add audio out stream api

* add /tts endpoint

* add docs

* wire up mtmd_helper_model_can_chat

* upload speaker_ref via form-data

* expose seed to api

* fix header for pcm

* wip

* fix penalty_last_n

* fix server_output_limits

* fix pocket-tts stream

* wip

* add server_schema for tts

* add server routing

* server: coerce seed and stream from multipart form fields

Form fields arrive as strings, so -F seed=42 and -F stream=true were
rejected by the schema. seed joins the integer fields, parsed as a long
long so a full uint32 seed fits, and stream is read as a boolean when
it comes as a string.

---------

Co-authored-by: Pascal <admin@serveurperso.com>
This commit is contained in:
Xuan-Son Nguyen
2026-10-21 01:19:42 +02:00
committed by GitHub
parent 1242be66f6
commit 5c0b49a23b
21 changed files with 724 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/common.cpp
+3
View File
@@ -1235,6 +1235,9 @@ common_gguf_info common_get_gguf_info(const std::string & fname) {
info.n_ctx_train = gguf_get_val_u32(gguf_ctx.get(), ctx_id);
}
// same key as llama_model_chat_template(model, nullptr)
info.has_chat_tmpl = gguf_find_key(gguf_ctx.get(), "tokenizer.chat_template") >= 0;
return info;
}
common/common.h
+1
View File
@@ -985,6 +985,7 @@ common_decision_type common_get_decision_type(const struct llama_model * model);
struct common_gguf_info {
common_decision_type decision_type = COMMON_DECISION_TYPE_UNKNOWN; // UNKNOWN if the file is missing, unreadable, or invalid
uint32_t n_ctx_train = 0; // 0 if unknown
bool has_chat_tmpl = false; // has the default tokenizer.chat_template
};
common_gguf_info common_get_gguf_info(const std::string & fname);
tools/mtmd/clip.cpp
+3 -2
View File
@@ -4278,8 +4278,9 @@ struct clip_init_result clip_init(const char * fname, struct clip_context_params
struct clip_cap clip_get_cap(const char * fname) {
clip_cap res;
clip_model_loader loader(fname, /* skip_tensors= */ true);
res.has_vision = loader.has_vision;
res.has_audio = loader.has_audio;
res.has_vision = loader.has_vision;
res.has_audio = loader.has_audio;
res.has_gen_audio = loader.has_gen_audio;
return res;
}
tools/mtmd/clip.h
+1
View File
@@ -148,5 +148,6 @@ std::map<ggml_backend_dev_t, size_t> clip_get_mem_usage(const struct clip_ctx *
struct clip_cap {
bool has_vision;
bool has_audio;
bool has_gen_audio;
};
struct clip_cap clip_get_cap(const char * fname);
tools/mtmd/mtmd-helper-gen.cpp
+138 -39
View File
@@ -50,29 +50,38 @@ static llama_token find_special_token(const llama_vocab * vocab, const std::stri
return LLAMA_TOKEN_NULL;
}
static void put_bytes(std::vector<char> & buf, const void * p, size_t n) {
const char * c = (const char *) p;
buf.insert(buf.end(), c, c + n);
}
// data_sz == UINT32_MAX writes the "unknown length" sentinel (streaming), same as ffmpeg does on a pipe
static void write_wav16_header(std::vector<char> & buf, uint32_t data_sz, int32_t rate) {
const uint32_t riff_sz = data_sz == UINT32_MAX ? UINT32_MAX : 36 + data_sz;
const uint32_t fmt_sz = 16, byte_rate = (uint32_t) rate * 2;
const uint16_t fmt = 1, ch = 1, align = 2, bits = 16;
const uint32_t rate32 = (uint32_t) rate;
put_bytes(buf, "RIFF", 4); put_bytes(buf, &riff_sz, 4); put_bytes(buf, "WAVE", 4);
put_bytes(buf, "fmt ", 4); put_bytes(buf, &fmt_sz, 4);
put_bytes(buf, &fmt, 2); put_bytes(buf, &ch, 2); put_bytes(buf, &rate32, 4);
put_bytes(buf, &byte_rate, 4); put_bytes(buf, &align, 2); put_bytes(buf, &bits, 2);
put_bytes(buf, "data", 4); put_bytes(buf, &data_sz, 4);
}
static void append_wav16_pcm(std::vector<char> & buf, const float * pcm, size_t n) {
for (size_t i = 0; i < n; i++) {
int16_t s = (int16_t) (std::max(-1.0f, std::min(1.0f, pcm[i])) * 32767.0f);
put_bytes(buf, &s, 2);
}
}
static bool write_wav16(std::vector<char> & buf, const std::vector<float> & pcm, int32_t rate) {
// RIFF chunk sizes are 32-bit; refuse to emit a file with a truncated header
if (pcm.size() > ((size_t) UINT32_MAX - 36) / 2) {
return false;
}
const uint32_t data_sz = (uint32_t) (pcm.size() * 2);
const uint32_t riff_sz = 36 + data_sz;
const uint32_t fmt_sz = 16, byte_rate = (uint32_t) rate * 2;
const uint16_t fmt = 1, ch = 1, align = 2, bits = 16;
const uint32_t rate32 = (uint32_t) rate;
auto put = [&](const void * p, size_t n) {
const char * c = (const char *) p;
buf.insert(buf.end(), c, c + n);
};
put("RIFF", 4); put(&riff_sz, 4); put("WAVE", 4);
put("fmt ", 4); put(&fmt_sz, 4);
put(&fmt, 2); put(&ch, 2); put(&rate32, 4);
put(&byte_rate, 4); put(&align, 2); put(&bits, 2);
put("data", 4); put(&data_sz, 4);
for (float v : pcm) {
int16_t s = (int16_t) (std::max(-1.0f, std::min(1.0f, v)) * 32767.0f);
put(&s, 2);
}
write_wav16_header(buf, (uint32_t) (pcm.size() * 2), rate);
append_wav16_pcm(buf, pcm.data(), pcm.size());
return true;
}
@@ -92,6 +101,8 @@ public:
// set out_stop on end-of-speech, h_state_out must be null if no frame is generated
virtual int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) = 0;
virtual int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) = 0;
// forces any buffered codes through code2wav now, regardless of window_frames
virtual int32_t flush() = 0;
protected:
llama_context * lctx;
@@ -121,6 +132,9 @@ public:
prompt_batch.reset();
n_prompt = 0;
prompt_pos = 0;
stream = false;
pcm_sent = 0;
wav_header_sent = false;
}
int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
@@ -208,6 +222,7 @@ public:
top_p = inp->top_p > 0 ? inp->top_p : def.top_p;
seed = inp->seed;
out_type = inp->out_type;
stream = inp->stream;
// the prompt above holds the whole text stream up to tts_eos, so every generated
// frame adds tts_pad on top of the codes embedding
@@ -301,31 +316,60 @@ public:
}
int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override {
if (!flush_gen_wav()) {
return 1;
*out_sample_rate = info.sample_rate;
if (!stream) {
// one-shot call: force out whatever's left, regardless of window_frames
if (!flush_gen_wav()) {
return 1;
}
if (out_n_samples) {
*out_n_samples = (int64_t) audio_pcm.size();
}
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
*out_data = (const char *) audio_pcm.data();
*out_data_len = audio_pcm.size() * sizeof(float);
return 0;
}
out_buf.clear();
if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {
LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
return 1;
}
*out_data = out_buf.data();
*out_data_len = out_buf.size();
return 0;
}
*out_sample_rate = info.sample_rate;
// streaming: only return audio produced since the previous call
const size_t n_new = audio_pcm.size() - pcm_sent;
if (out_n_samples) {
*out_n_samples = (int64_t) audio_pcm.size();
*out_n_samples = (int64_t) n_new;
}
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
*out_data = (const char *) audio_pcm.data();
*out_data_len = audio_pcm.size() * sizeof(float);
*out_data = (const char *) (audio_pcm.data() + pcm_sent);
*out_data_len = n_new * sizeof(float);
pcm_sent = audio_pcm.size();
return 0;
}
out_buf.clear();
if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {
LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
return 1;
if (!wav_header_sent) {
write_wav16_header(out_buf, UINT32_MAX, info.sample_rate);
wav_header_sent = true;
}
append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new);
pcm_sent = audio_pcm.size();
*out_data = out_buf.data();
*out_data_len = out_buf.size();
return 0;
}
int32_t flush() override {
return flush_gen_wav() ? 0 : 1;
}
private:
bool ensure_cache() {
if (specials_ok) {
@@ -369,7 +413,7 @@ private:
LOG_ERR("mtmd_helper_gen_audio: mmproj has no speaker/audio encoder\n");
return false;
}
const std::string marker = mtmd_default_marker();
const std::string marker = mtmd_get_marker(mctx);
mtmd_input_text text{ marker.c_str(), marker.size(), false, true };
mtmd_input_chunks * chunks = mtmd_input_chunks_init();
const mtmd_bitmap * bptr = bitmap;
@@ -455,6 +499,9 @@ private:
std::vector<float> h_state_buf;
mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
std::vector<char> out_buf;
bool stream = false;
size_t pcm_sent = 0; // samples already returned by get_output()
bool wav_header_sent = false;
};
// settings that only live in the reference's per-pack yaml, not in the checkpoint
@@ -506,6 +553,9 @@ public:
chunk_idx = 0;
n_voice_pos = 0;
chunk_budget = 0;
stream = false;
pcm_sent = 0;
wav_header_sent = false;
}
int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
@@ -575,6 +625,7 @@ public:
seed = inp->seed;
out_type = inp->out_type;
stream = inp->stream;
return 0;
}
@@ -662,31 +713,60 @@ public:
}
int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override {
if (!flush_gen_wav()) {
return 1;
*out_sample_rate = info.sample_rate;
if (!stream) {
// one-shot call: force out whatever's left, regardless of window_frames
if (!flush_gen_wav()) {
return 1;
}
if (out_n_samples) {
*out_n_samples = (int64_t) audio_pcm.size();
}
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
*out_data = (const char *) audio_pcm.data();
*out_data_len = audio_pcm.size() * sizeof(float);
return 0;
}
out_buf.clear();
if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {
LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
return 1;
}
*out_data = out_buf.data();
*out_data_len = out_buf.size();
return 0;
}
*out_sample_rate = info.sample_rate;
// streaming: only return audio produced since the previous call
const size_t n_new = audio_pcm.size() - pcm_sent;
if (out_n_samples) {
*out_n_samples = (int64_t) audio_pcm.size();
*out_n_samples = (int64_t) n_new;
}
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
*out_data = (const char *) audio_pcm.data();
*out_data_len = audio_pcm.size() * sizeof(float);
*out_data = (const char *) (audio_pcm.data() + pcm_sent);
*out_data_len = n_new * sizeof(float);
pcm_sent = audio_pcm.size();
return 0;
}
out_buf.clear();
if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {
LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
return 1;
if (!wav_header_sent) {
write_wav16_header(out_buf, UINT32_MAX, info.sample_rate);
wav_header_sent = true;
}
append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new);
pcm_sent = audio_pcm.size();
*out_data = out_buf.data();
*out_data_len = out_buf.size();
return 0;
}
int32_t flush() override {
return flush_gen_wav() ? 0 : 1;
}
private:
bool ensure_cache() {
if (specials_ok) {
@@ -907,7 +987,7 @@ private:
LOG_ERR("mtmd_helper_gen_audio: mmproj has no voice encoder\n");
return false;
}
const std::string marker = mtmd_default_marker();
const std::string marker = mtmd_get_marker(mctx);
mtmd_input_text text{ marker.c_str(), marker.size(), false, true };
mtmd_input_chunks * chunks = mtmd_input_chunks_init();
const mtmd_bitmap * bptr = bitmap;
@@ -989,6 +1069,9 @@ private:
std::vector<float> h_state_buf;
mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
std::vector<char> out_buf;
bool stream = false;
size_t pcm_sent = 0; // samples already returned by get_output()
bool wav_header_sent = false;
};
static std::unique_ptr<mtmd_gen_audio_pipeline> make_pipeline(llama_context * lctx, mtmd_context * mctx) {
@@ -1017,11 +1100,20 @@ void mtmd_helper_gen_audio_free(mtmd_helper_gen_audio * ctx) {
}
void mtmd_helper_gen_audio_reset(mtmd_helper_gen_audio * ctx) {
if (ctx->pipeline) {
if (ctx && ctx->pipeline) {
ctx->pipeline->reset();
}
}
struct mtmd_helper_gen_audio_inp mtmd_helper_gen_audio_inp_default(void) {
mtmd_helper_gen_audio_inp inp{};
inp.top_k = 50;
inp.top_p = 1.0f;
inp.seed = UINT32_MAX; // random
inp.out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
return inp;
}
int32_t mtmd_helper_gen_audio_set_input(mtmd_helper_gen_audio * ctx, const mtmd_helper_gen_audio_inp * inp) {
if (!ctx->pipeline) {
LOG_ERR("mtmd_helper_gen_audio: unsupported or missing gen-audio pipeline\n");
@@ -1058,3 +1150,10 @@ int32_t mtmd_helper_gen_audio_get_output(mtmd_helper_gen_audio * ctx, int32_t *
}
return ctx->pipeline->get_output(out_sample_rate, out_data, out_data_len, out_n_samples);
}
int32_t mtmd_helper_gen_audio_flush(mtmd_helper_gen_audio * ctx) {
if (!ctx->pipeline) {
return 1;
}
return ctx->pipeline->flush();
}
tools/mtmd/mtmd-helper.h
+46 -1
View File
@@ -204,6 +204,7 @@ enum mtmd_helper_gen_audio_outtype {
MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV, // WAV PCM 16-bit LE, mono
};
struct mtmd_helper_gen_audio_inp {
bool stream; // if true, output() must be called after each step_gen()
llama_seq_id seq_id;
const char * prompt;
@@ -219,6 +220,8 @@ struct mtmd_helper_gen_audio_inp {
enum mtmd_helper_gen_audio_outtype out_type;
};
MTMD_API struct mtmd_helper_gen_audio_inp mtmd_helper_gen_audio_inp_default(void);
MTMD_API mtmd_helper_gen_audio * mtmd_helper_gen_audio_init(
struct llama_context * lctx,
struct mtmd_context * mctx);
@@ -250,6 +253,8 @@ MTMD_API int32_t mtmd_helper_gen_audio_step_gen(
// out_data valid until next get_output() or reset() call
// out_n_samples (optional, can be NULL) receives the number of generated PCM samples
// if inp->stream is true: returns only audio produced since the previous call, and
// *out_data_len == 0 whenever a full window_frames batch hasn't accumulated yet
MTMD_API int32_t mtmd_helper_gen_audio_get_output(
mtmd_helper_gen_audio * ctx,
int32_t * out_sample_rate,
@@ -257,6 +262,10 @@ MTMD_API int32_t mtmd_helper_gen_audio_get_output(
size_t * out_data_len,
int64_t * out_n_samples);
// forces any buffered codes through code2wav now, regardless of window_frames;
// call once when generation has ended, before the last get_output() in stream mode
MTMD_API int32_t mtmd_helper_gen_audio_flush(mtmd_helper_gen_audio * ctx);
#ifdef __cplusplus
} // extern "C"
#endif
@@ -283,8 +292,41 @@ struct mtmd_helper_gen_audio_deleter {
};
using gen_audio_ptr = std::unique_ptr<mtmd_helper_gen_audio, mtmd_helper_gen_audio_deleter>;
struct gen_audio {
// sub-struct, RAII wrapper for mtmd_helper_gen_audio_inp
struct inp {
mtmd_helper_gen_audio_inp data = mtmd_helper_gen_audio_inp_default();
std::string prompt_str;
std::string lang_str;
mtmd::bitmap_ptr speaker_ref_ptr;
inp() = default;
inp(inp &&) = default;
inp & operator=(inp &&) = default;
inp(const inp &) = delete;
inp & operator=(const inp &) = delete;
void set_prompt (std::string p) { prompt_str = std::move(p); }
void set_lang (std::string l) { lang_str = std::move(l); }
void set_speaker_ref(mtmd::bitmap_ptr bmp) { speaker_ref_ptr = std::move(bmp); }
// pointers are only valid as long as *this is alive
const mtmd_helper_gen_audio_inp * get() {
data.prompt = prompt_str.c_str();
data.prompt_len = prompt_str.size();
data.lang = lang_str.empty() ? nullptr : lang_str.c_str();
data.speaker_ref = speaker_ref_ptr.get();
return &data;
}
};
gen_audio_ptr ctx;
gen_audio(struct llama_context * lctx, struct mtmd_context * mctx) : ctx(mtmd_helper_gen_audio_init(lctx, mctx)) {}
void init(struct llama_context * lctx, struct mtmd_context * mctx) {
ctx.reset(mtmd_helper_gen_audio_init(lctx, mctx));
}
bool valid() const {
return ctx.get() != nullptr;
}
void reset() {
mtmd_helper_gen_audio_reset(ctx.get());
}
@@ -300,6 +342,9 @@ struct gen_audio {
int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples = nullptr) {
return mtmd_helper_gen_audio_get_output(ctx.get(), out_sample_rate, out_data, out_data_len, out_n_samples);
}
int32_t flush() {
return mtmd_helper_gen_audio_flush(ctx.get());
}
};
} // namespace mtmd_helper
tools/mtmd/mtmd.cpp
+2 -1
View File
@@ -2846,10 +2846,11 @@ struct mtmd_caps mtmd_get_cap_from_file(const char * fname) {
mtmd_caps cap;
cap.inp_audio = tmp.has_audio;
cap.inp_vision = tmp.has_vision;
cap.gen_audio = tmp.has_gen_audio;
return cap;
} catch (const std::exception & e) {
LOG_ERR("%s: failed to get capabilities from file '%s': %s\n", __func__, fname, e.what());
return mtmd_caps{ false, false };
return mtmd_caps{ false, false, false };
}
}
tools/mtmd/mtmd.h
+1
View File
@@ -360,6 +360,7 @@ MTMD_API void mtmd_log_set(ggml_log_callback log_callback, void * user_data);
struct mtmd_caps {
bool inp_vision;
bool inp_audio;
bool gen_audio;
};
MTMD_API struct mtmd_caps mtmd_get_cap_from_file(const char * mmproj_fname);
tools/server/README.md
+47 -2
View File
@@ -802,6 +802,50 @@ curl http://127.0.0.1:8012/v1/rerank \
}' | jq
```
### POST `/tts`: Generate speech audio from text
Returns raw audio bytes (`audio/wav` by default) rather than JSON. For more info, see [tts/README.md](../tts/README.md)
*Options:*
`input`: The text to speak (alias: `prompt`).
`lang`: Language code for the utterance (model-dependent, e.g. `en`, `zh`). Optional.
`speaker_ref_b64`: Base64-encoded reference audio to clone the speaker's voice. Optional. Alternatively, upload the reference audio as a `speaker_ref` file field via `multipart/form-data` (see example below) - if both are provided, the uploaded file takes precedence.
`top_k`, `top_p`: Sampling params for the acoustic code predictor. Optional, model-dependent defaults apply.
`repeat_penalty`: Repetition penalty applied to the backbone sampler over the whole generation (default `1.05`). Without this, the backbone can loop and re-generate the same utterance.
`seed`: RNG seed for both the backbone sampler and the codec/vocoder (`-1` = random, the default). Set to a fixed value for reproducible output.
`n_predict`: Max number of audio frames to generate. Defaults to `512`; generation normally stops earlier once the model emits an end-of-speech token.
`response_format`: `wav` (default) or `pcm` (raw `float32` little-endian mono samples, no header; the response `Content-Type` carries the sample rate, e.g. `audio/pcm;rate=24000;encoding=float;bits=32`).
`stream`: If `true`, the response is streamed as audio becomes available instead of waiting for the full generation to finish. WAV streaming writes an RFC-noncompliant header with an unknown (`0xFFFFFFFF`) size field, since the final length isn't known up front; most players and decoders (ffmpeg, VLC, ...) handle this by reading until EOF.
Note: it's highly recommended to always provide a speaker reference voice; otherwise, the model's performance may be degraded.
*Examples:*
```shell
curl -X POST http://127.0.0.1:9931/tts \
-H "Content-Type: application/json" \
-d '{"input": "Hello, this is a test."}' \
-o output.wav
```
With a speaker reference uploaded as a file (`multipart/form-data`), instead of base64-encoding it into the JSON body:
```shell
curl -X POST http://127.0.0.1:9931/tts \
-F "input=Hello, this is a test." \
-F "speaker_ref=@/path/to/speaker-reference.wav;type=audio/wav" \
-o output.wav
```
### POST `/infill`: For code infilling.
Takes a prefix and a suffix and returns the predicted completion as stream.
@@ -1262,11 +1306,12 @@ Each object in `data` has an `architecture` object. It has two string arrays:
- `input_modalities` lists what the model can read. It always has `text`, plus each media type that the model supports.
- `output_modalities` lists what the model can produce.
One output value is special:
Some output values are special:
| Value | Meaning |
|---|---|
| `decisions` | The model is a native decision model. Serve it with [`/v1/systemone`](#post-v1systemone-typesafe-compatible-system-one-api). |
| `audio` | The model can generate speech. Serve it with [`/tts`](#post-tts-generate-speech-audio-from-text). A TTS-only model has `["audio"]` without `text`. |
A language model that classifies with prompts does not get `decisions`. Only native decision models do.
@@ -2054,7 +2099,7 @@ Note:
- If a model is not running, it will be added or updated according to the source
2. When the model is loaded, the info from `/v1/models` is forwarded to router's `/v1/models`. This includes metadata about the model and the runtime instance.
Each object in `data` has the same `architecture` object as [`GET /v1/models`](#get-v1models-openai-compatible-model-info-api) of a direct server. The server computes both arrays offline. It does not load the model, download files, or run inference. `output_modalities` comes from the GGUF metadata. `input_modalities` comes from the projector file. A native decision model shows `decisions` before its first load, after unload, and while it sleeps:
Each object in `data` has the same `architecture` object as [`GET /v1/models`](#get-v1models-openai-compatible-model-info-api) of a direct server. The server computes both arrays offline. It does not load the model, download files, or run inference. `output_modalities` comes from the GGUF metadata, plus `audio` when the projector file can generate speech. `input_modalities` comes from the projector file. A native decision model shows `decisions` before its first load, after unload, and while it sleeps:
```json
{
tools/server/server-common.cpp
+20 -5
View File
@@ -202,7 +202,8 @@ const char * get_media_marker() {
// model output modalities
//
std::vector<std::string> server_model_output_modalities(common_decision_type decision_type) {
std::vector<std::string> server_model_output_modalities(common_decision_type decision_type, bool has_cap_chat, bool has_cap_tts) {
std::vector<std::string> out;
switch (decision_type) {
case COMMON_DECISION_TYPE_OPENJEV:
case COMMON_DECISION_TYPE_LEV:
@@ -216,13 +217,27 @@ std::vector<std::string> server_model_output_modalities(common_decision_type dec
case COMMON_DECISION_TYPE_LFM2_D1:
case COMMON_DECISION_TYPE_LFM2_D1_OMNI:
case COMMON_DECISION_TYPE_H2O_LIGHTNING:
return {"decisions"};
out = {"decisions"};
break;
case COMMON_DECISION_TYPE_GRANITE_SWITCH:
return {"text", "decisions"};
out = {"text", "decisions"};
break;
default:
// fallback when there is no decision type or the metadata is bad
return {"text"};
// a tts-only model cannot generate text
if (has_cap_chat || !has_cap_tts) {
// fallback when there is no decision type or the metadata is bad
out = {"text"};
}
break;
}
if (has_cap_tts) {
out.push_back("audio");
}
return out;
}
std::vector<std::string> server_model_output_modalities(const llama_model * model, bool has_cap_chat, bool has_cap_tts) {
return server_model_output_modalities(common_get_decision_type(model), has_cap_chat, has_cap_tts);
}
json server_model_architecture_json(
tools/server/server-common.h
+4 -1
View File
@@ -114,7 +114,10 @@ const char * get_media_marker();
//
// output modalities for architecture.output_modalities in GET /models
std::vector<std::string> server_model_output_modalities(common_decision_type decision_type);
// has_cap_chat: false for a tts-only model, has_cap_tts: the model can generate audio
std::vector<std::string> server_model_output_modalities(common_decision_type decision_type, bool has_cap_chat, bool has_cap_tts);
// same, with the decision type read from a loaded model
std::vector<std::string> server_model_output_modalities(const llama_model * model, bool has_cap_chat, bool has_cap_tts);
// architecture object of GET /models; shared by the direct server and the router
json server_model_architecture_json(
tools/server/server-context.cpp
+349 -6
View File
@@ -20,6 +20,7 @@
#include "speculative.h"
#include "mtmd.h"
#include "mtmd-helper.h"
#include "base64.hpp"
#include <algorithm>
#include <cstddef>
@@ -43,6 +44,14 @@
constexpr int HTTP_POLLING_SECONDS = 1;
static common_speculative_output_limits server_output_limits(const common_params & params) {
if (!params.mmproj.path.empty()) {
const auto mcaps = mtmd_get_cap_from_file(params.mmproj.path.c_str());
if (mcaps.gen_audio) {
// some TTS models decode with embeddings output for the whole batch
return { params.n_batch, params.n_batch };
}
}
if (params.embedding ||
(params.pooling_type != LLAMA_POOLING_TYPE_UNSPECIFIED && params.pooling_type != LLAMA_POOLING_TYPE_NONE)) {
return { params.n_batch, 1 };
@@ -257,6 +266,27 @@ struct server_slot {
mtmd_context * mctx = nullptr;
mtmd::batch_ptr mbatch = nullptr;
struct tts_ctx {
mtmd_helper::gen_audio ctx;
std::vector<float> h_state_prompt; // the first h_state after step_prompt done
const float * h_state;
llama_token sampled;
int32_t n_decoded;
bool stop;
bool is_supported() const {
return ctx.valid();
}
void reset() {
ctx.reset();
h_state_prompt.clear();
h_state = nullptr;
sampled = LLAMA_TOKEN_NULL;
n_decoded = 0;
stop = false;
}
};
tts_ctx tts;
// speculative decoding
common_speculative * spec;
@@ -421,6 +451,9 @@ struct server_slot {
// clear multimodal state
mbatch.reset();
tts.reset();
audio_stream.reset();
audio_wait = false;
inp_embd.clear();
@@ -622,6 +655,11 @@ struct server_slot {
prompt_clear();
}
// mtmd_helper always re-eval the whole prompt
if (task->type == SERVER_TASK_TYPE_TTS) {
prompt_clear();
}
callback_on_reset(*this);
reset();
@@ -975,6 +1013,14 @@ public:
mtmd_helper_init_opt init_opt = mtmd_helper_init_opt_default();
const llama_vocab * vocab = nullptr;
bool has_cap_tts() const {
return mctx != nullptr && mtmd_gen_audio_get_info(mctx).type != MTMD_GEN_AUDIO_TYPE_NONE;
}
bool has_cap_chat() const {
return mctx == nullptr || mtmd_helper_model_can_chat(ctx_tgt, mctx);
}
server_decision_context decision;
server_queue queue_tasks;
@@ -1463,6 +1509,10 @@ private:
slot.mctx = mctx;
slot.prompt.tokens.has_mtmd = mctx != nullptr;
if (has_cap_tts()) {
slot.tts.ctx.init(ctx_tgt, mctx);
}
SLT_TRC(slot, "new slot, n_ctx = %d\n", slot.n_ctx);
slot.callback_on_release = [this](int id_slot) {
@@ -1944,6 +1994,20 @@ private:
SLT_DBG(slot, "launching slot : %s\n", safe_json_to_str(slot.to_json()).c_str());
if (task.type == SERVER_TASK_TYPE_TTS) {
GGML_ASSERT(has_cap_tts()); // should already checked in route handler
if (!slot.tts.is_supported()) {
slot.tts.ctx.init(ctx_tgt, slot.mctx);
}
// mtmd_helper always re-eval the whole prompt
slot.prompt_clear();
task.tts_inp.data.seq_id = slot.id;
if (slot.tts.ctx.set_input(task.tts_inp.get()) != 0) {
send_error(task, "failed to process TTS prompt", ERROR_TYPE_SERVER);
return false;
}
}
// initialize samplers
if (task.need_sampling()) {
try {
@@ -1961,6 +2025,9 @@ private:
// TODO: getting pre sampling logits is not yet supported with backend sampling
use_backend_sampling &= !need_pre_sample_logits;
// TODO: check verify if this actually works with TTS
use_backend_sampling &= task.type != SERVER_TASK_TYPE_TTS;
// TODO: tmp until backend sampling is fully implemented
if (use_backend_sampling) {
llama_set_sampler(ctx_tgt, slot.id, common_sampler_get(slot.smpl.get()));
@@ -1986,9 +2053,13 @@ private:
slot.task = std::make_unique<const server_task>(std::move(task));
slot.state = slot.task->is_child()
? SLOT_STATE_WAIT_OTHER // wait for the parent to process prompt
: SLOT_STATE_STARTED;
if (slot.task->type == SERVER_TASK_TYPE_TTS) {
slot.state = SLOT_STATE_PROCESSING_PROMPT;
} else {
slot.state = slot.task->is_child()
? SLOT_STATE_WAIT_OTHER // wait for the parent to process prompt
: SLOT_STATE_STARTED;
}
// no prompt, the rows come from the audio stream, which audio controls reach from now on
if (slot.task->type == SERVER_TASK_TYPE_TRANSCRIBE_RT) {
@@ -2268,6 +2339,18 @@ private:
queue_results.send(std::move(res));
}
void send_tts_result(server_slot & slot, int32_t sample_rate, const char * data, size_t data_len, bool final) {
auto res = std::make_unique<server_task_result_tts>();
res->id = slot.task->id;
res->index = slot.task->index;
res->sample_rate = sample_rate;
res->audio.assign(data, data_len);
res->final = final;
queue_results.send(std::move(res));
}
void send_final_response(server_slot & slot) {
auto res = std::make_unique<server_task_result_cmpl_final>();
@@ -2872,6 +2955,7 @@ private:
case SERVER_TASK_TYPE_TRANSCRIBE_RT:
case SERVER_TASK_TYPE_EMBEDDING:
case SERVER_TASK_TYPE_RERANK:
case SERVER_TASK_TYPE_TTS:
case SERVER_TASK_TYPE_DECISION:
{
// special case: if input is provided via CLI, tokenize it first
@@ -3397,6 +3481,14 @@ private:
return;
}
// note: TTS slots bypass the shared batch entirely
try {
process_tts_slots();
} catch (const std::exception & e) {
SRV_ERR("process_tts_slots() failed: %s\n", e.what());
abort_all_slots("process_tts_slots() failed: " + std::string(e.what()));
}
GGML_ASSERT(batch.slot_batched || batch.size() == 0);
if (batch.slot_batched) {
@@ -3459,10 +3551,89 @@ private:
}
}
void process_tts_slots() {
iterate(slots, [&](server_slot & slot) {
if (!slot.is_processing() || slot.task->type != SERVER_TASK_TYPE_TTS) {
return;
}
llama_set_embeddings(ctx_tgt, true);
if (slot.state == SLOT_STATE_PROCESSING_PROMPT) {
const int32_t ret = slot.tts.ctx.step_prompt(llama_n_batch(ctx_tgt));
if (ret < 0) {
send_error(slot, "TTS prompt processing failed", ERROR_TYPE_SERVER);
slot.release();
} else if (ret == 0) {
// done prompt, do sample and save h_state_prompt
slot.tts.sampled = common_sampler_sample(slot.smpl.get(), ctx_tgt, -1);
common_sampler_accept(slot.smpl.get(), slot.tts.sampled, true);
const float * h_embd = llama_get_embeddings_ith(ctx_tgt, -1);
slot.tts.h_state_prompt.assign(h_embd, h_embd + llama_model_n_embd(model_tgt));
slot.tts.h_state = slot.tts.h_state_prompt.data();
slot.state = SLOT_STATE_GENERATING;
}
return;
}
const int32_t n_predict = slot.n_predict_max > 0 ? slot.n_predict_max : 512;
if (slot.tts.stop || slot.tts.n_decoded >= n_predict) {
int32_t sample_rate = 0;
const char * data = nullptr;
size_t data_len = 0;
// generation truly ends here: force out any sub-window remainder still buffered
if (slot.tts.ctx.flush() != 0 || slot.tts.ctx.get_output(&sample_rate, &data, &data_len) != 0) {
send_error(slot, "failed to finalize TTS output", ERROR_TYPE_SERVER);
} else {
send_tts_result(slot, sample_rate, data, data_len, true);
}
slot.release();
return;
}
const float * h_state_next = nullptr;
bool stop = false;
if (slot.tts.ctx.step_gen(slot.tts.sampled, slot.tts.h_state, &h_state_next, &stop) != 0) {
send_error(slot, "TTS generation failed", ERROR_TYPE_SERVER);
slot.release();
return;
}
if (h_state_next == nullptr) {
// end-of-speech without a new frame (e.g. pocket-tts eos head)
slot.tts.stop = true;
return;
}
slot.tts.h_state = h_state_next;
slot.tts.n_decoded++;
slot.tts.stop = stop;
if (!stop) {
slot.tts.sampled = common_sampler_sample(slot.smpl.get(), ctx_tgt, -1);
common_sampler_accept(slot.smpl.get(), slot.tts.sampled, true);
}
if (slot.task->params.stream) {
int32_t sample_rate = 0;
const char * data = nullptr;
size_t data_len = 0;
if (slot.tts.ctx.get_output(&sample_rate, &data, &data_len) != 0) {
send_error(slot, "TTS streaming output failed", ERROR_TYPE_SERVER);
slot.release();
} else if (data_len > 0) {
send_tts_result(slot, sample_rate, data, data_len, false);
}
}
});
}
void pre_decode() {
// apply context-shift if needed
// TODO: simplify and improve
iterate(slots, [&](server_slot & slot) {
if (slot.task && slot.task->type == SERVER_TASK_TYPE_TTS) {
// TTS slots drive their own decode loop in process_tts_slots(), never enter the shared batch
return;
}
if (slot.state == SLOT_STATE_GENERATING && slot.prompt.n_tokens() + 1 >= slot.n_ctx) {
if (!params_base.ctx_shift) {
// this check is redundant (for good)
@@ -3535,7 +3706,7 @@ private:
// determine which slots are generating and drafting
iterate(slots, [&](server_slot & slot) {
if (slot.state != SLOT_STATE_GENERATING) {
if (slot.state != SLOT_STATE_GENERATING || slot.task->type == SERVER_TASK_TYPE_TTS) {
return;
}
@@ -3681,7 +3852,7 @@ private:
return; // batch is full, skip remaining slots
}
if (!slot.is_processing()) {
if (!slot.is_processing() || slot.task->type == SERVER_TASK_TYPE_TTS) {
return;
}
@@ -4926,11 +5097,13 @@ server_context_meta server_context::get_meta() const {
/* model_aliases */ impl->model_aliases,
/* model_tags */ impl->model_tags,
/* model_path */ impl->params_base.model.path,
/* model_output_modalities */ server_model_output_modalities(common_get_decision_type(impl->model_tgt)),
/* model_output_modalities */ server_model_output_modalities(impl->model_tgt, impl->has_cap_chat(), impl->has_cap_tts()),
/* has_mtmd */ impl->mctx != nullptr,
/* has_inp_image */ impl->chat_params.allow_image,
/* has_inp_audio */ impl->chat_params.allow_audio,
/* has_inp_video */ impl->chat_params.allow_video,
/* has_cap_chat */ impl->has_cap_chat(),
/* has_cap_tts */ impl->has_cap_tts(),
/* has_audio_stream */ impl->mctx != nullptr && mtmd_support_audio_stream(impl->mctx),
/* json_ui_settings */ impl->json_ui_settings,
/* slot_n_ctx */ impl->n_ctx_slot(),
@@ -5006,6 +5179,11 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
res->set_req(&req); // will also set spipe if needed
if (!ctx_server.has_cap_chat()) {
res->error(format_error_response("this server does not support chat/completions", ERROR_TYPE_NOT_SUPPORTED));
return res;
}
int32_t sse_ping_interval = params.sse_ping_interval;
try {
@@ -6175,6 +6353,171 @@ void server_routes::init_routes() {
return res;
};
this->post_tts = [this](const server_http_req & req) {
auto res = create_response();
res->set_req(&req); // will also set spipe if needed
if (!ctx_server.has_cap_tts()) {
res->error(format_error_response("this server does not support audio generation", ERROR_TYPE_NOT_SUPPORTED));
return res;
}
const auto info = mtmd_gen_audio_get_info(ctx_server.mctx);
json body = json::parse(req.body);
// multipart form fields arrive as strings, correct the types of the numeric and boolean params
for (const char * k : { "top_k", "n_predict", "max_tokens", "seed" }) {
if (body.contains(k) && body[k].is_string()) {
body[k] = std::stoll(body[k].get<std::string>());
}
}
for (const char * k : { "top_p", "repeat_penalty" }) {
if (body.contains(k) && body[k].is_string()) {
body[k] = std::stof(body[k].get<std::string>());
}
}
if (body.contains("stream") && body["stream"].is_string()) {
body["stream"] = body["stream"] == "true";
}
std::string prompt = json_value(body, "input", json_value(body, "prompt", std::string()));
if (prompt.empty()) {
res->error(format_error_response("\"input\" must be a non-empty string", ERROR_TYPE_INVALID_REQUEST));
return res;
}
const std::string response_format = json_value(body, "response_format", std::string("wav"));
server_task task(SERVER_TASK_TYPE_TTS);
server_schema::eval_tts_schema(params, task, body);
const bool stream = task.params.stream;
task.tts_inp.set_prompt(prompt);
task.tts_inp.set_lang(json_value(body, "lang", std::string()));
task.tts_inp.data.stream = stream;
task.tts_inp.data.out_type = response_format == "pcm"
? MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM
: MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
// note: -1 is clamped to 0 by the sampler, it will disable the penalty
task.params.sampling.penalty_last_n = task.params.n_predict > 0 ? task.params.n_predict : 512;
task.params.sampling.seed = task.tts_inp.data.seed; // same seed for the codec/vocoder RNG
if (task.tts_inp.data.top_k > 0) {
task.params.sampling.top_k = task.tts_inp.data.top_k;
}
if (task.tts_inp.data.top_p > 0) {
task.params.sampling.top_p = task.tts_inp.data.top_p;
}
// speaker reference: either an uploaded form file ("speaker_ref") or a base64 JSON field ("speaker_ref_b64")
const unsigned char * speaker_ref_data = nullptr;
size_t speaker_ref_len = 0;
std::string speaker_ref_b64_decoded;
auto speaker_ref_file = req.files.find("speaker_ref");
if (speaker_ref_file != req.files.end()) {
speaker_ref_data = speaker_ref_file->second.data.data();
speaker_ref_len = speaker_ref_file->second.data.size();
} else {
std::string speaker_ref_b64 = json_value(body, "speaker_ref_b64", std::string());
if (!speaker_ref_b64.empty()) {
try {
speaker_ref_b64_decoded = base64::decode(speaker_ref_b64);
} catch (const std::exception &) {
res->error(format_error_response("\"speaker_ref_b64\" is not valid base64", ERROR_TYPE_INVALID_REQUEST));
return res;
}
speaker_ref_data = (const unsigned char *) speaker_ref_b64_decoded.data();
speaker_ref_len = speaker_ref_b64_decoded.size();
}
}
if (speaker_ref_len > 0) {
auto wrapper = mtmd_helper_bitmap_init_from_buf(ctx_server.mctx, speaker_ref_data, speaker_ref_len, false, ctx_server.init_opt);
if (!wrapper.bitmap) {
res->error(format_error_response("failed to decode \"speaker_ref\"", ERROR_TYPE_INVALID_REQUEST));
return res;
}
task.tts_inp.set_speaker_ref(mtmd::bitmap_ptr(wrapper.bitmap));
} else {
SRV_WRN("%s", "no speaker reference provided, the model may behave randomly\n");
}
auto & rd = res->rd;
task.id = rd.get_new_id();
rd.post_task(std::move(task));
// raw float32 LE mono samples; audio/L16 would imply 16-bit big-endian (RFC 2586)
const std::string content_type = response_format == "pcm"
? "audio/pcm;rate=" + std::to_string(info.sample_rate) + ";encoding=float;bits=32"
: "audio/wav";
if (!stream) {
auto result = rd.next(req.should_stop);
if (!result) {
GGML_ASSERT(req.should_stop());
return res; // connection is closed
}
if (result->is_error()) {
res->error(result->to_json());
return res;
}
auto * tts_res = dynamic_cast<server_task_result_tts *>(result.get());
GGML_ASSERT(tts_res != nullptr);
res->status = 200;
res->content_type = content_type;
res->data = std::move(tts_res->audio);
return res;
} else {
auto first_result = rd.next(req.should_stop);
if (!first_result) {
GGML_ASSERT(req.should_stop());
return res; // connection is closed
}
if (first_result->is_error()) {
res->error(first_result->to_json());
return res;
}
auto * first_tts_res = dynamic_cast<server_task_result_tts *>(first_result.get());
GGML_ASSERT(first_tts_res != nullptr);
res->status = 200;
res->content_type = content_type;
res->data = std::move(first_tts_res->audio);
bool is_done = first_tts_res->final;
res->set_next([res_this = res.get(), is_done](std::string & output) mutable -> bool {
// flush buffered audio before is_done - the first result can already be final
if (!res_this->data.empty()) {
output = std::move(res_this->data);
res_this->data.clear();
return true;
}
if (is_done) {
return false;
}
if (res_this->should_stop()) {
return false;
}
server_response_reader & rd = res_this->rd;
if (!rd.has_next()) {
return false;
}
auto result = rd.next([&res_this]() { return res_this->should_stop(); });
if (!result || result->is_error()) {
return false;
}
auto * tts_res = dynamic_cast<server_task_result_tts *>(result.get());
GGML_ASSERT(tts_res != nullptr);
output = std::move(tts_res->audio);
is_done = tts_res->final;
return true;
});
}
return res;
};
this->post_systemone = [this](const server_http_req & req) {
auto res = create_response();
const auto & decision = ctx_server.decision;
tools/server/server-context.h
+3
View File
@@ -24,6 +24,8 @@ struct server_context_meta {
bool has_inp_image;
bool has_inp_audio;
bool has_inp_video;
bool has_cap_chat;
bool has_cap_tts;
bool has_audio_stream; // realtime transcription, see mtmd_support_audio_stream()
json json_ui_settings;
int slot_n_ctx;
@@ -155,6 +157,7 @@ struct server_routes {
server_http_context::handler_t post_embeddings;
server_http_context::handler_t post_embeddings_oai;
server_http_context::handler_t post_rerank;
server_http_context::handler_t post_tts;
server_http_context::handler_t post_systemone;
server_http_context::handler_t get_lora_adapters;
server_http_context::handler_t post_lora_adapters;
tools/server/server-models.cpp
+12 -8
View File
@@ -521,27 +521,31 @@ void server_model_meta::update_caps(const common_params & base) {
return;
}
// read the output modalities and the trained context from the GGUF metadata
std::vector<std::string> output_modalities = {"text"};
if (!params.model.path.empty()) {
const common_gguf_info info = common_get_gguf_info(params.model.path);
output_modalities = server_model_output_modalities(info.decision_type);
n_ctx_train = info.n_ctx_train;
}
bool inp_image = false;
bool inp_audio = false;
bool gen_audio = false;
try {
if (!params.no_mmproj && !params.mmproj.path.empty()) {
mtmd_caps caps = mtmd_get_cap_from_file(params.mmproj.path.c_str());
inp_image = caps.inp_vision;
inp_audio = caps.inp_audio;
gen_audio = caps.gen_audio;
}
} catch (const std::exception & e) {
LOG_WRN("failed to read the multimodal capabilities of '%s': %s\n", name.c_str(), e.what());
// keep the output modalities from the GGUF metadata
}
// read the output modalities and the trained context from the GGUF metadata
std::vector<std::string> output_modalities = {"text"};
if (!params.model.path.empty()) {
const common_gguf_info info = common_get_gguf_info(params.model.path);
// same rule as mtmd_helper_model_can_chat()
const bool can_chat = !gen_audio || info.has_chat_tmpl;
output_modalities = server_model_output_modalities(info.decision_type, can_chat, gen_audio);
n_ctx_train = info.n_ctx_train;
}
// offline discovery cannot see video; a loaded model reports it
architecture = server_model_architecture_json(inp_image, inp_audio, false, output_modalities);
}
tools/server/server-schema.cpp
+52
View File
@@ -511,6 +511,58 @@ task_params eval_llama_cmpl_schema(
return params;
}
//
// TTS schema
//
std::vector<std::unique_ptr<field>> make_tts_schema(server_task & task) {
std::vector<std::unique_ptr<field>> fields;
auto add = [&](field * f) {
fields.emplace_back(f);
};
add((new field_num("top_k", task.tts_inp.data.top_k))
->set_limits(0, INT32_MAX)
->set_desc("Top-k for the acoustic code predictor, 0 to use the model default"));
add((new field_num("top_p", task.tts_inp.data.top_p))
->set_limits(0.0f, 1.0f)
->set_desc("Top-p for the acoustic code predictor, 0.0 to use the model default"));
add((new field_num("seed", task.tts_inp.data.seed))
->set_desc("RNG seed for the backbone sampler and the codec/vocoder (-1 = random)"));
add((new field_num("n_predict", task.params.n_predict))
->set_hard_limits(-1, INT32_MAX)
->add_alias("max_tokens")
->set_desc("Max number of audio frames to generate"));
add((new field_num("repeat_penalty", task.params.sampling.penalty_repeat))
->set_desc("Repetition penalty applied to the backbone sampler over the whole generation"));
add((new field_bool("stream", task.params.stream))
->set_desc("Stream the audio as it becomes available instead of waiting for the full generation"));
return fields;
}
void eval_tts_schema(const common_params & params_base, server_task & task, const json & data) {
// baseline defaults, individual requests can override them
// TODO @ngxson : change the default values based on model
task.params.sampling = params_base.sampling;
task.params.sampling.penalty_repeat = 1.05f;
task.params.n_predict = -1;
task.tts_inp.data.top_k = 0;
task.tts_inp.data.top_p = 0.0f;
task.tts_inp.data.seed = params_base.sampling.seed;
field_eval_context ctx(task.params);
for (const auto & f : make_tts_schema(task)) {
f->eval(ctx, data);
}
}
//
// eval() implementations
//
tools/server/server-schema.h
+8
View File
@@ -101,4 +101,12 @@ task_params eval_llama_cmpl_schema(
const std::vector<llama_logit_bias> & logit_bias_eog,
const json & data);
std::vector<std::unique_ptr<field>> make_tts_schema(server_task & task);
// evaluates the /tts request params into task.params and task.tts_inp.data
void eval_tts_schema(
const common_params & params_base,
server_task & task,
const json & data);
} // namespace server_schema
tools/server/server-task.cpp
+11
View File
@@ -1508,6 +1508,17 @@ json server_task_result_rerank::to_json() {
};
}
//
// server_task_result_tts
//
json server_task_result_tts::to_json() {
return json {
{"sample_rate", sample_rate},
{"n_bytes", audio.size()},
{"final", final},
};
}
//
// server_task_result_decision
//
tools/server/server-task.h
+16
View File
@@ -10,6 +10,7 @@
// TODO: prevent including the whole server-common.h as we only use server_tokens
#include "server-common.h"
#include "mtmd-helper.h"
enum server_task_type {
@@ -29,6 +30,7 @@ enum server_task_type {
SERVER_TASK_TYPE_SLOT_ERASE,
SERVER_TASK_TYPE_GET_LORA,
SERVER_TASK_TYPE_SET_LORA,
SERVER_TASK_TYPE_TTS,
};
// TODO: change this to more generic "response_format" to replace the "format_response_*" in server-common
@@ -219,6 +221,9 @@ struct server_task {
// used by SERVER_TASK_TYPE_SET_LORA
std::map<int, float> set_lora; // mapping adapter ID -> scale
// used by SERVER_TASK_TYPE_TTS
mtmd_helper::gen_audio::inp tts_inp;
server_task() = default;
server_task(server_task_type type) : type(type) {}
@@ -256,6 +261,7 @@ struct server_task {
switch (type) {
case SERVER_TASK_TYPE_COMPLETION:
case SERVER_TASK_TYPE_INFILL:
case SERVER_TASK_TYPE_TTS:
case SERVER_TASK_TYPE_TRANSCRIBE_RT:
return true;
default:
@@ -526,6 +532,16 @@ struct server_task_result_embd : server_task_result {
json to_json_oaicompat();
};
struct server_task_result_tts : server_task_result {
std::string audio; // raw bytes for this chunk (WAV or PCM, per request's out_type)
int32_t sample_rate = 0;
bool final = false; // true for the last chunk of a request
virtual bool is_stop() override { return final; }
virtual json to_json() override;
};
struct server_task_result_rerank : server_task_result {
float score = -1e6;
tools/server/server.cpp
+2
View File
@@ -247,6 +247,7 @@ int llama_server(common_params & params, int argc, char ** argv, server_child &
routes.post_control = models_routes->proxy_post;
routes.post_responses_oai = models_routes->proxy_post;
routes.post_transcriptions_oai = models_routes->proxy_post;
routes.post_tts = models_routes->proxy_post;
routes.ws_realtime = models_routes->proxy_ws;
routes.post_anthropic_messages = models_routes->proxy_post;
routes.post_anthropic_count_tokens = models_routes->proxy_post;
@@ -293,6 +294,7 @@ int llama_server(common_params & params, int argc, char ** argv, server_child &
ctx_http.post("/responses", ex_wrapper(routes.post_responses_oai));
ctx_http.post("/v1/audio/transcriptions", ex_wrapper(routes.post_transcriptions_oai));
ctx_http.post("/audio/transcriptions", ex_wrapper(routes.post_transcriptions_oai));
ctx_http.post("/tts", ex_wrapper(routes.post_tts));
ctx_http.ws ("/v1/realtime", routes.ws_realtime);
ctx_http.post("/v1/messages", ex_wrapper(routes.post_anthropic_messages)); // anthropic messages API
ctx_http.post("/infill", ex_wrapper(routes.post_infill));
tools/tts/README.md
+2
View File
@@ -12,6 +12,8 @@ Simple usage:
llama-tts -hf ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF -p "Hello world" --output out.wav
```
Note: it's highly recommended to always provide a speaker reference voice (via `--tts-speaker-file`); otherwise, the model's performance may be degraded.
Common params:
- Sampling params such as `--top-k`, `--top-p`, `--temp`, etc.
- `-n <number_of_frames>` limits the output length, e.g. `-n 500`. Note that how many milliseconds each frame represents varies by model
tools/tts/tts.cpp
+3 -3
View File
@@ -111,8 +111,9 @@ int main(int argc, char ** argv) {
speaker_bitmap.reset(wrapper.bitmap);
}
mtmd_helper::gen_audio gen(lctx, mctx.get());
mtmd_helper_gen_audio_inp inp{};
mtmd_helper::gen_audio gen;
gen.init(lctx, mctx.get());
mtmd_helper_gen_audio_inp inp = mtmd_helper_gen_audio_inp_default();
inp.seq_id = 0;
inp.prompt = params.prompt.c_str();
inp.prompt_len = params.prompt.size();
@@ -121,7 +122,6 @@ int main(int argc, char ** argv) {
inp.top_k = params.sampling.top_k;
inp.top_p = params.sampling.top_p;
inp.seed = params.sampling.seed;
inp.out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
//
// stage 1: process prompt via backbone model, generate semantic representation