From 0693dbfb915f33233818373b5f6969396658d02c Mon Sep 17 00:00:00 2001 From: martin-cimmino <159526529+martin-cimmino@users.noreply.github.com> Date: Thu, 22 Oct 2026 13:34:47 +0200 Subject: [PATCH] nemotron : support non-standard GQA, BPE tokenizer and domyn-small pre-type (#23343) * nemotron : support non-standard GQA, BPE tokenizer and domyn-small pre-type * gguf-py : write Metaspace prepend_scheme as GGUF metadata * vocab : parse metaspace prepend_scheme into enum, rename key * vocab : use metaspace prepend_scheme in detokenize, add TODOs for other tokenizers * vocab : use constant for metaspace escaped space --------- Co-authored-by: Martin Cimmino --- conversion/base.py | 3 +++ conversion/nemotron.py | 24 ++++++++++++++++---- convert_hf_to_gguf_update.py | 1 + gguf-py/gguf/constants.py | 2 ++ gguf-py/gguf/gguf_writer.py | 3 +++ gguf-py/gguf/vocab.py | 18 +++++++++++++++ src/llama-arch.cpp | 1 + src/llama-arch.h | 1 + src/llama-vocab.cpp | 43 +++++++++++++++++++++++++++++++++++- src/llama-vocab.h | 8 +++++++ src/models/nemotron.cpp | 7 ++++-- 11 files changed, 104 insertions(+), 7 deletions(-) diff --git a/conversion/base.py b/conversion/base.py index f3bbe7b0f..d8420eb32 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -1837,6 +1837,9 @@ class TextModel(ModelBase): if chkhsh == "1431a23e583c97432bc230bff598d103ddb5a1f89960c8f1d1051aaa944d0b35": # ref: https://huggingface.co/sapienzanlp/Minerva-7B-base-v1.0 res = "minerva-7b" + if chkhsh == "6e4d72c6e5260b90606e26a0931fa15e4b42ec6e97719d12919f2b1f93478262": + # ref: https://huggingface.co/domyn/Domyn-Small-v1.0 + res = "domyn-small" if chkhsh == "7e57df22b1fe23a7b1e1c7f3dc4e3f96d43a4eb0836d0c6bdc3436d7b2f1c664": # ref: https://huggingface.co/tencent/Hunyuan-A13B-Instruct res = "hunyuan" diff --git a/conversion/nemotron.py b/conversion/nemotron.py index c90048194..d76a45d82 100644 --- a/conversion/nemotron.py +++ b/conversion/nemotron.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re from typing import Any, Callable, Iterable, Sequence, TYPE_CHECKING import numpy as np @@ -367,9 +368,21 @@ class NemotronModel(TextModel): model_arch = gguf.MODEL_ARCH.NEMOTRON def set_vocab(self): - self._set_vocab_sentencepiece() - self.gguf_writer.add_pad_token_id(0) - self.gguf_writer.add_unk_token_id(1) + if (self.dir_model / "tokenizer.model").exists(): + self._set_vocab_sentencepiece() + self.gguf_writer.add_pad_token_id(0) + self.gguf_writer.add_unk_token_id(1) + else: + # SPM-style BPE (Metaspace + byte fallback): mark <0xXX> tokens as BYTE so they decode to raw bytes + tokens, toktypes, tokpre = self.get_vocab_base() + toktypes = [gguf.TokenType.BYTE if re.fullmatch(r"<0x[0-9A-F]{2}>", tok) else typ for tok, typ in zip(tokens, toktypes)] + self.gguf_writer.add_tokenizer_model("gpt2") + self.gguf_writer.add_tokenizer_pre(tokpre) + self.gguf_writer.add_token_list(tokens) + self.gguf_writer.add_token_types(toktypes) + + special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True) + special_vocab.add_to_gguf(self.gguf_writer) def set_gguf_parameters(self): super().set_gguf_parameters() @@ -383,7 +396,10 @@ class NemotronModel(TextModel): rot_pct = self.rope_parameters["partial_rotary_factor"] n_embd = self.find_hparam(["hidden_size", "n_embd"]) n_head = self.find_hparam(["num_attention_heads", "n_head"]) - self.gguf_writer.add_rope_dimension_count(int(rot_pct * n_embd) // n_head) + # Use explicit head_dim when available; fallback keeps standard Nemotron working. + # Without this, models where n_head * head_dim != n_embd (e.g. 48 heads, head_dim=128) produce the wrong rope dimension count. + head_dim = self.hparams.get("head_dim") or n_embd // n_head + self.gguf_writer.add_rope_dimension_count(int(rot_pct * head_dim)) # * RopeScaling for Nemotron factor = self.hparams.get("factor") or self.rope_parameters.get("factor") diff --git a/convert_hf_to_gguf_update.py b/convert_hf_to_gguf_update.py index 7d8c4fda6..37df033ef 100755 --- a/convert_hf_to_gguf_update.py +++ b/convert_hf_to_gguf_update.py @@ -178,6 +178,7 @@ pre_computed_hashes = [ {"name": "glm4", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/zai-org/GLM-4.5-Air", "chkhsh": "9ca2dd618e8afaf09731a7cf6e2105b373ba6a1821559f258b272fe83e6eb902"}, {"name": "glm4", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/zai-org/GLM-4.7-Flash", "chkhsh": "cdf5f35325780597efd76153d4d1c16778f766173908894c04afc20108536267"}, {"name": "minerva-7b", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/sapienzanlp/Minerva-7B-base-v1.0", "chkhsh": "1431a23e583c97432bc230bff598d103ddb5a1f89960c8f1d1051aaa944d0b35"}, + {"name": "domyn-small", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/domyn/Domyn-Small-v1.0", "chkhsh": "6e4d72c6e5260b90606e26a0931fa15e4b42ec6e97719d12919f2b1f93478262"}, {"name": "hunyuan", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tencent/Hunyuan-A13B-Instruct", "chkhsh": "7e57df22b1fe23a7b1e1c7f3dc4e3f96d43a4eb0836d0c6bdc3436d7b2f1c664"}, {"name": "hunyuan-dense", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tencent/Hunyuan-4B-Instruct", "chkhsh": "bba3b3366b646dbdded5dbc42d59598b849371afc42f7beafa914afaa5b70aa6"}, {"name": "hy_v4", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tencent/Hy4-preview", "chkhsh": "e6ddf9c6686791c12d698d34c31ab9be1fea9af5a3d9a6909783ab382198ae1c"}, diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 4482f7f33..0b95580bd 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -383,6 +383,8 @@ class Keys: NORMALIZER_NFC = "tokenizer.ggml.normalizer.nfc" NORMALIZER_LOWERCASE = "tokenizer.ggml.normalizer.lowercase" NORMALIZER_STRIP_ACCENTS = "tokenizer.ggml.normalizer.strip_accents" + # Metaspace constants + METASPACE_PREPEND_SCHEME = "tokenizer.ggml.metaspace.prepend_scheme" # FIM/Infill special tokens constants FIM_PRE_ID = "tokenizer.ggml.fim_pre_token_id" FIM_SUF_ID = "tokenizer.ggml.fim_suf_token_id" diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 2faaefa59..9e49a1514 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -1377,6 +1377,9 @@ class GGUFWriter: def add_normalizer_strip_accents(self, value: bool) -> None: self.add_bool(Keys.Tokenizer.NORMALIZER_STRIP_ACCENTS, value) + def add_metaspace_prepend_scheme(self, value: str) -> None: + self.add_string(Keys.Tokenizer.METASPACE_PREPEND_SCHEME, value) + def add_eot_token_id(self, id: int) -> None: self.add_uint32(Keys.Tokenizer.EOT_ID, id) diff --git a/gguf-py/gguf/vocab.py b/gguf-py/gguf/vocab.py index 6383178ea..3b82c3961 100644 --- a/gguf-py/gguf/vocab.py +++ b/gguf-py/gguf/vocab.py @@ -56,6 +56,7 @@ class SpecialVocab: normalizer_lowercase: bool | None normalizer_strip_accents: bool | None clean_spaces: bool | None + metaspace_prepend_scheme: str | None def __init__( self, path: str | os.PathLike[str], load_merges: bool = False, @@ -72,6 +73,7 @@ class SpecialVocab: self.normalizer_lowercase = None self.normalizer_strip_accents = None self.clean_spaces = None + self.metaspace_prepend_scheme = None if special_token_types is not None: self.special_token_types = special_token_types else: @@ -126,6 +128,10 @@ class SpecialVocab: if not quiet: logger.info(f'Setting clean_spaces to {self.clean_spaces}') gw.add_clean_spaces(self.clean_spaces) + if self.metaspace_prepend_scheme is not None: + if not quiet: + logger.info(f'Setting metaspace_prepend_scheme to {self.metaspace_prepend_scheme}') + gw.add_metaspace_prepend_scheme(self.metaspace_prepend_scheme) def _load(self, path: Path) -> None: self._try_load_from_tokenizer_json(path) @@ -194,6 +200,16 @@ class SpecialVocab: for norm in normalizer.get('normalizers', []): self._parse_normalizer(norm) + def _parse_pre_tokenizer(self, pre_tokenizer: dict) -> None: + # ref: https://huggingface.co/docs/tokenizers/api/pre-tokenizers + pre_tokenizer_type = pre_tokenizer.get('type') + if pre_tokenizer_type == 'Metaspace': + if 'prepend_scheme' in pre_tokenizer: + self.metaspace_prepend_scheme = pre_tokenizer['prepend_scheme'] + elif pre_tokenizer_type == 'Sequence': + for pre in pre_tokenizer.get('pretokenizers', []): + self._parse_pre_tokenizer(pre) + def _try_load_from_tokenizer_json(self, path: Path) -> bool: tokenizer = None tokenizer_file = path / 'tokenizer.json' @@ -229,6 +245,8 @@ class SpecialVocab: # Parse normalizer configuration (e.g. Lowercase) into metadata if normalizer := tokenizer.get('normalizer'): self._parse_normalizer(normalizer) + if pre_tokenizer := tokenizer.get('pre_tokenizer'): + self._parse_pre_tokenizer(pre_tokenizer) added_tokens = tokenizer.get('added_tokens', {}) else: added_tokens = {} diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index d79b18dec..a364174c1 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -445,6 +445,7 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_TOKENIZER_NORMALIZER_NFC, "tokenizer.ggml.normalizer.nfc" }, { LLM_KV_TOKENIZER_NORMALIZER_LOWERCASE, "tokenizer.ggml.normalizer.lowercase" }, { LLM_KV_TOKENIZER_NORMALIZER_STRIP_ACCENTS, "tokenizer.ggml.normalizer.strip_accents" }, + { LLM_KV_TOKENIZER_METASPACE_PREPEND_SCHEME, "tokenizer.ggml.metaspace.prepend_scheme" }, { LLM_KV_TOKENIZER_FIM_PRE_ID, "tokenizer.ggml.fim_pre_token_id" }, { LLM_KV_TOKENIZER_FIM_SUF_ID, "tokenizer.ggml.fim_suf_token_id" }, { LLM_KV_TOKENIZER_FIM_MID_ID, "tokenizer.ggml.fim_mid_token_id" }, diff --git a/src/llama-arch.h b/src/llama-arch.h index 12d8fe392..cdf978fb8 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -408,6 +408,7 @@ enum llm_kv { LLM_KV_TOKENIZER_NORMALIZER_NFC, LLM_KV_TOKENIZER_NORMALIZER_LOWERCASE, LLM_KV_TOKENIZER_NORMALIZER_STRIP_ACCENTS, + LLM_KV_TOKENIZER_METASPACE_PREPEND_SCHEME, LLM_KV_TOKENIZER_FIM_PRE_ID, LLM_KV_TOKENIZER_FIM_SUF_ID, LLM_KV_TOKENIZER_FIM_MID_ID, diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index 80a5fcdff..78512bef5 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -543,6 +543,7 @@ struct llm_tokenizer_bpe : llm_tokenizer { byte_encode = false; // uses raw UTF-8, not GPT-2 byte encoding break; case LLAMA_VOCAB_PRE_TYPE_SARVAM_MOE: + case LLAMA_VOCAB_PRE_TYPE_DOMYN_SMALL: // Sarvam uses SPM-style BPE (same shape as Gemma4): spaces replaced with U+2581 // by the normalizer, BPE merges over the whole text on raw UTF-8. regex_exprs = { @@ -2097,6 +2098,8 @@ struct llama_vocab::impl { // Tokenizer normalizer options llama_vocab::normalizer_options normalizer_opts; + llama_metaspace_prepend_scheme metaspace_prepend_scheme = LLAMA_METASPACE_PREPEND_SCHEME_NEVER; + std::unordered_map token_to_id; std::vector id_to_token; @@ -2612,6 +2615,10 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { } else if ( tokenizer_pre == "minerva-7b") { pre_type = LLAMA_VOCAB_PRE_TYPE_MINERVA; + } else if ( + tokenizer_pre == "domyn-small") { + pre_type = LLAMA_VOCAB_PRE_TYPE_DOMYN_SMALL; + escape_whitespaces = true; } else if ( tokenizer_pre == "megrez") { pre_type = LLAMA_VOCAB_PRE_TYPE_QWEN2; @@ -2956,6 +2963,24 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { normalizer_opts.strip_accents = normalizer_opts.lowercase; ml.get_key(LLM_KV_TOKENIZER_NORMALIZER_STRIP_ACCENTS, normalizer_opts.strip_accents, false); + { + std::string scheme; + if (ml.get_key(LLM_KV_TOKENIZER_METASPACE_PREPEND_SCHEME, scheme, false)) { + if (scheme == "never") { + metaspace_prepend_scheme = LLAMA_METASPACE_PREPEND_SCHEME_NEVER; + } else if (scheme == "first") { + metaspace_prepend_scheme = LLAMA_METASPACE_PREPEND_SCHEME_FIRST; + } else if (scheme == "always") { + metaspace_prepend_scheme = LLAMA_METASPACE_PREPEND_SCHEME_ALWAYS; + } else { + throw std::runtime_error(format("unknown metaspace prepend_scheme: '%s'", scheme.c_str())); + } + if (type == LLAMA_VOCAB_TYPE_BPE && metaspace_prepend_scheme == LLAMA_METASPACE_PREPEND_SCHEME_ALWAYS) { + throw std::runtime_error("metaspace prepend_scheme 'always' is not supported for BPE vocab"); + } + } + } + // suppress tokens { const int suppress_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_SUPPRESS_TOKENS).c_str()); @@ -3743,6 +3768,7 @@ std::vector llama_vocab::impl::tokenize( // tokenizer.encode('', add_special_tokens=True) returns [1] // tokenizer.encode('', add_special_tokens=False) returns [] + // TODO: honor metaspace_prepend_scheme (add_space_prefix currently behaves like "always") bool is_prev_special = true; // prefix with space if first token if (add_special && add_bos) { @@ -3813,6 +3839,13 @@ std::vector llama_vocab::impl::tokenize( llama_escape_whitespace(text); } + // Metaspace prepend_scheme "first": add U+2581 only at the start of the input + static const std::string escaped_space = "\xE2\x96\x81"; + if (metaspace_prepend_scheme == LLAMA_METASPACE_PREPEND_SCHEME_FIRST && + &fragment == &fragment_buffer.front() && text.rfind(escaped_space, 0) != 0) { + text.insert(0, escaped_space); + } + #ifdef PRETOKENIZERDEBUG LLAMA_LOG_WARN("TT: (%ld %ld %ld) '%s'\n", text.length(), fragment.offset, fragment.length, text.c_str()); #endif @@ -3860,6 +3893,7 @@ std::vector llama_vocab::impl::tokenize( GGML_ASSERT(special_bos_id != LLAMA_TOKEN_NULL); output.push_back(special_bos_id); } + // TODO: honor metaspace_prepend_scheme llm_tokenizer_ugm_session session(vocab, *static_cast(tokenizer.get())); for (const auto & fragment : fragment_buffer) { @@ -4143,10 +4177,17 @@ int32_t llama_vocab::impl::detokenize( } } + // Metaspace prepend_scheme "first": remove the leading space of the first rendered piece, so a skipped BOS does not consume it + // TODO: honor metaspace_prepend_scheme for the other vocab types + bool remove_first_space = type == LLAMA_VOCAB_TYPE_BPE && metaspace_prepend_scheme == LLAMA_METASPACE_PREPEND_SCHEME_FIRST; + for (int32_t i = 0; i < n_tokens; ++i) { GGML_ASSERT(avail >= 0); - int32_t n_chars = token_to_piece(tokens[i], text, avail, remove_space, unparse_special); + int32_t n_chars = token_to_piece(tokens[i], text, avail, remove_space || remove_first_space, unparse_special); remove_space = false; + if (unparse_special || !(token_get_attr(tokens[i]) & (LLAMA_TOKEN_ATTR_UNKNOWN | LLAMA_TOKEN_ATTR_CONTROL))) { + remove_first_space = false; + } if (n_chars < 0) { avail = 0; total -= n_chars; diff --git a/src/llama-vocab.h b/src/llama-vocab.h index a87c0de42..40357dba7 100644 --- a/src/llama-vocab.h +++ b/src/llama-vocab.h @@ -71,6 +71,14 @@ enum llama_vocab_pre_type { LLAMA_VOCAB_PRE_TYPE_MMBERT = 60, LLAMA_VOCAB_PRE_TYPE_K2_HORIZON = 61, LLAMA_VOCAB_PRE_TYPE_FLINT = 62, + LLAMA_VOCAB_PRE_TYPE_DOMYN_SMALL = 63, +}; + +// HF Metaspace pre-tokenizer prepend_scheme +enum llama_metaspace_prepend_scheme { + LLAMA_METASPACE_PREPEND_SCHEME_NEVER = 0, + LLAMA_METASPACE_PREPEND_SCHEME_FIRST = 1, + LLAMA_METASPACE_PREPEND_SCHEME_ALWAYS = 2, }; struct LLM_KV; diff --git a/src/models/nemotron.cpp b/src/models/nemotron.cpp index 129cf622e..6b5b77107 100644 --- a/src/models/nemotron.cpp +++ b/src/models/nemotron.cpp @@ -25,8 +25,11 @@ void llama_model_nemotron::load_arch_tensors(llama_model_loader &) { layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, 0); - create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0); - layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0); + // n_embd_head_k * n_head equals n_embd for standard Nemotron (32 heads), + // but differs for models with non-standard GQA (e.g. 48 Q heads, head_dim=128). + const int64_t n_embd_q = n_embd_head_k * n_head; + create_tensor_qkv(layer, i, n_embd, n_embd_q, n_embd_gqa, n_embd_gqa, 0); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_q, n_embd}, 0); // optional bias tensors layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);