darthrubik/llama-cpp
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

vocab : add clean_spaces GGUF key (#30940)
Copilot Setup Steps / copilot-setup-steps (push) Has been cancelled
Check Pre-Tokenizer Hashes / pre-tokenizer-hashes (push) Has been cancelled
Python check requirements.txt / check-requirements (push) Has been cancelled
Python Type-Check / python type-check (push) Has been cancelled
Update Operations Documentation / update-ops-docs (push) Has been cancelled

Browse Source
This commit is contained in:
tkran2
2026-10-22 02:58:08 -05:00
committed by GitHub
parent e197cef455
commit d88040c704
9 changed files with 48 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gguf-py/gguf/constants.py
+1
View File
@@ -371,6 +371,7 @@ class Keys:
DNA_AUTO_TAGS = "tokenizer.ggml.dna_auto_tags"
ADD_PREFIX = "tokenizer.ggml.add_space_prefix"
REMOVE_EXTRA_WS = "tokenizer.ggml.remove_extra_whitespaces"
CLEAN_SPACES = "tokenizer.ggml.clean_spaces"
PRECOMPILED_CHARSMAP = "tokenizer.ggml.precompiled_charsmap"
SUPPRESS_TOKENS = "tokenizer.ggml.suppress_tokens"
HF_JSON = "tokenizer.huggingface.json"
gguf-py/gguf/gguf_writer.py
+3
View File
@@ -1326,6 +1326,9 @@ class GGUFWriter:
def add_remove_extra_whitespaces(self, value: bool) -> None:
self.add_bool(Keys.Tokenizer.REMOVE_EXTRA_WS, value)
def add_clean_spaces(self, value: bool) -> None:
self.add_bool(Keys.Tokenizer.CLEAN_SPACES, value)
def add_precompiled_charsmap(self, charsmap: bytes) -> None:
self.add_array(Keys.Tokenizer.PRECOMPILED_CHARSMAP, charsmap)
gguf-py/gguf/vocab.py
+10
View File
@@ -55,6 +55,7 @@ class SpecialVocab:
normalizer_nfc: bool | None
normalizer_lowercase: bool | None
normalizer_strip_accents: bool | None
clean_spaces: bool | None
def __init__(
self, path: str | os.PathLike[str], load_merges: bool = False,
@@ -70,6 +71,7 @@ class SpecialVocab:
self.normalizer_nfc = None
self.normalizer_lowercase = None
self.normalizer_strip_accents = None
self.clean_spaces = None
if special_token_types is not None:
self.special_token_types = special_token_types
else:
@@ -120,6 +122,10 @@ class SpecialVocab:
if not quiet:
logger.info(f'Setting normalizer_strip_accents to {self.normalizer_strip_accents}')
gw.add_normalizer_strip_accents(self.normalizer_strip_accents)
if self.clean_spaces is not None:
if not quiet:
logger.info(f'Setting clean_spaces to {self.clean_spaces}')
gw.add_clean_spaces(self.clean_spaces)
def _load(self, path: Path) -> None:
self._try_load_from_tokenizer_json(path)
@@ -345,6 +351,10 @@ class SpecialVocab:
self.chat_template = chat_template
else:
logger.warning(f'Bad type for chat_template field in {tokenizer_config_file!r} - ignoring')
# decode clean-up of spaces before punctuation (transformers clean_up_tokenization_spaces)
clean_spaces = tokenizer_config.get('clean_up_tokenization_spaces')
if isinstance(clean_spaces, bool):
self.clean_spaces = clean_spaces
for typ in self.special_token_types:
add_entry = tokenizer_config.get(f'add_{typ}_token')
if isinstance(add_entry, bool):
src/llama-arch.cpp
+1
View File
@@ -437,6 +437,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
{ LLM_KV_TOKENIZER_DNA_AUTO_TAGS, "tokenizer.ggml.dna_auto_tags" },
{ LLM_KV_TOKENIZER_ADD_PREFIX, "tokenizer.ggml.add_space_prefix" },
{ LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, "tokenizer.ggml.remove_extra_whitespaces" },
{ LLM_KV_TOKENIZER_CLEAN_SPACES, "tokenizer.ggml.clean_spaces" },
{ LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP, "tokenizer.ggml.precompiled_charsmap" },
{ LLM_KV_TOKENIZER_HF_JSON, "tokenizer.huggingface.json" },
{ LLM_KV_TOKENIZER_RWKV, "tokenizer.rwkv.world" },
src/llama-arch.h
+1
View File
@@ -400,6 +400,7 @@ enum llm_kv {
LLM_KV_TOKENIZER_DNA_AUTO_TAGS,
LLM_KV_TOKENIZER_ADD_PREFIX,
LLM_KV_TOKENIZER_REMOVE_EXTRA_WS,
LLM_KV_TOKENIZER_CLEAN_SPACES,
LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP,
LLM_KV_TOKENIZER_HF_JSON,
LLM_KV_TOKENIZER_RWKV,
src/llama-model-saver.cpp
+1
View File
@@ -424,6 +424,7 @@ void llama_model_saver::add_kv_from_model() {
add_kv(LLM_KV_TOKENIZER_ADD_SEP, vocab.get_add_sep());
add_kv(LLM_KV_TOKENIZER_ADD_PREFIX, vocab.get_add_space_prefix());
add_kv(LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, vocab.get_remove_extra_whitespaces());
add_kv(LLM_KV_TOKENIZER_CLEAN_SPACES, vocab.get_clean_spaces());
add_kv(LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP, vocab.get_precompiled_charsmap());
// add_kv(LLM_KV_TOKENIZER_HF_JSON, ???);
// add_kv(LLM_KV_TOKENIZER_RWKV, ???);
src/llama-vocab.cpp
+14 -1
View File
@@ -2432,6 +2432,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
} else if (tokenizer_pre == "minicpm5") {
pre_type = LLAMA_VOCAB_PRE_TYPE_MINICPM5;
ignore_merges = true;
clean_spaces = false;
} else if (
tokenizer_pre == "llama3" ||
tokenizer_pre == "llama-v3" ||
@@ -2439,11 +2440,16 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
tokenizer_pre == "falcon3" ||
tokenizer_pre == "falcon-h1" ||
tokenizer_pre == "midm-2.0" ||
tokenizer_pre == "lfm2" ||
tokenizer_pre == "jina-v5-nano") {
pre_type = LLAMA_VOCAB_PRE_TYPE_LLAMA3;
ignore_merges = true;
add_bos = true;
} else if (
tokenizer_pre == "lfm2") {
pre_type = LLAMA_VOCAB_PRE_TYPE_LLAMA3;
ignore_merges = true;
add_bos = true;
clean_spaces = false;
} else if (
tokenizer_pre == "deepseek-llm") {
pre_type = LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM;
@@ -2494,6 +2500,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
tokenizer_pre == "granite-embed-multi-311m") {
pre_type = LLAMA_VOCAB_PRE_TYPE_GEMMA4;
escape_whitespaces = true;
clean_spaces = false;
} else if (
tokenizer_pre == "mmbert") {
pre_type = LLAMA_VOCAB_PRE_TYPE_MMBERT;
@@ -2540,6 +2547,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
} else if (
tokenizer_pre == "dbrx") {
pre_type = LLAMA_VOCAB_PRE_TYPE_DBRX;
clean_spaces = false;
} else if (
tokenizer_pre == "smaug-bpe") {
pre_type = LLAMA_VOCAB_PRE_TYPE_SMAUG;
@@ -2551,12 +2559,14 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
tokenizer_pre == "chatglm-bpe") {
pre_type = LLAMA_VOCAB_PRE_TYPE_CHATGLM4;
special_bos_id = LLAMA_TOKEN_NULL;
clean_spaces = false;
} else if (
tokenizer_pre == "glm4" ||
tokenizer_pre == "glm5") {
pre_type = LLAMA_VOCAB_PRE_TYPE_CHATGLM4;
special_bos_id = LLAMA_TOKEN_NULL;
ignore_merges = true;
clean_spaces = false;
} else if (
tokenizer_pre == "viking") {
pre_type = LLAMA_VOCAB_PRE_TYPE_VIKING;
@@ -2590,6 +2600,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
} else if (
tokenizer_pre == "exaone4") {
pre_type = LLAMA_VOCAB_PRE_TYPE_GPT2;
clean_spaces = false;
} else if (
tokenizer_pre == "exaone-moe") {
pre_type = LLAMA_VOCAB_PRE_TYPE_EXAONE_MOE;
@@ -2690,6 +2701,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
} else if (
tokenizer_pre == "mellum2") {
pre_type = LLAMA_VOCAB_PRE_TYPE_MELLUM2;
clean_spaces = false;
} else if (
tokenizer_pre == "k2-horizon") {
pre_type = LLAMA_VOCAB_PRE_TYPE_K2_HORIZON;
@@ -2726,6 +2738,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
ml.get_key(LLM_KV_TOKENIZER_ADD_PREFIX, add_space_prefix, false);
ml.get_key(LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, remove_extra_whitespaces, false);
ml.get_key(LLM_KV_TOKENIZER_CLEAN_SPACES, clean_spaces, false);
}
const int token_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_LIST).c_str());
tests/CMakeLists.txt
+1 -1
View File
@@ -142,7 +142,7 @@ llama_test(test-tokenizer-0 NAME test-tokenizer-0-command-r ARGS ${PROJE
llama_test(test-tokenizer-0 NAME test-tokenizer-0-deepseek-coder ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-deepseek-coder.gguf)
llama_test(test-tokenizer-0 NAME test-tokenizer-0-deepseek-llm ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-deepseek-llm.gguf)
llama_test(test-tokenizer-0 NAME test-tokenizer-0-falcon ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-falcon.gguf)
llama_test(test-tokenizer-0 NAME test-tokenizer-0-gemma-4 ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-gemma-4.gguf)
llama_test(test-tokenizer-0 NAME test-tokenizer-0-gemma-4 ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-gemma-4.gguf --check-decode)
llama_test(test-tokenizer-0 NAME test-tokenizer-0-gpt-2 ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-gpt-2.gguf)
llama_test(test-tokenizer-0 NAME test-tokenizer-0-llama-bpe ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-llama-bpe.gguf)
llama_test(test-tokenizer-0 NAME test-tokenizer-0-llama-spm ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-llama-spm.gguf)
tests/test-tokenizer-0.cpp
+16 -3
View File
@@ -125,7 +125,7 @@ static llama_tests read_tests(const std::string & fname_inp, const std::string &
int main(int argc, char **argv) {
if (argc < 2) {
fprintf(stderr, "Usage: %s vocab-file [text-file]\n", argv[0]);
fprintf(stderr, "Usage: %s vocab-file [text-file] [--check-decode]\n", argv[0]);
return 1;
}
@@ -135,8 +135,13 @@ int main(int argc, char **argv) {
const std::string fname_out = fname + ".out";
std::string fname_text;
if (argc > 2) {
fname_text = argv[2];
bool check_decode = false; // also check that the expected tokens detokenize to the input text
for (int i = 2; i < argc; i++) {
if (std::string(argv[i]) == "--check-decode") {
check_decode = true;
} else {
fname_text = argv[i];
}
}
fprintf(stderr, "%s : reading vocab from: '%s'\n", __func__, fname.c_str());
@@ -244,6 +249,14 @@ int main(int argc, char **argv) {
success = false;
}
if (check_decode) {
const std::string dec = common_detokenize(ctx, test_kv.second);
if (dec != test_kv.first) {
fprintf(stderr, "%s : failed decode: '%s' detokenized to '%s'\n", __func__, test_kv.first.c_str(), dec.c_str());
success = false;
}
}
}
});
}