darthrubik/llama-cpp
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

mtmd : add MiniCPM-OCR 1.0 support (#30979)
Copilot Setup Steps / copilot-setup-steps (push) Has been cancelled
Check Pre-Tokenizer Hashes / pre-tokenizer-hashes (push) Has been cancelled
Python check requirements.txt / check-requirements (push) Has been cancelled
Python Type-Check / python type-check (push) Has been cancelled
Update Operations Documentation / update-ops-docs (push) Has been cancelled

Browse Source 
Signed-off-by: tc-mb <tianchi_cai@icloud.com>
This commit is contained in:
tc-mb
2026-10-23 01:22:15 +08:00
committed by GitHub
parent 19ab5a5e65
commit a08ab75ae7
7 changed files with 46 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conversion/__init__.py
+2
View File
@@ -196,6 +196,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
"MiniCPMForCausalLM": "minicpm",
"MiniCPMV4_6ForConditionalGeneration": "minicpm",
"MiniCPMV4_7ForConditionalGeneration": "minicpm",
"MiniCPMOcrForConditionalGeneration": "minicpm",
"MiniMaxText01ForCausalLM": "minimax",
"MiniMaxM1ForCausalLM": "minimax",
"MiniMaxM2ForCausalLM": "minimax",
@@ -361,6 +362,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
"MiniMaxM3SparseForConditionalGeneration": "minimax",
"MiniCPMV4_6ForConditionalGeneration": "minicpm",
"MiniCPMV4_7ForConditionalGeneration": "minicpm",
"MiniCPMOcrForConditionalGeneration": "minicpm",
"Mistral3ForConditionalGeneration": "llava",
"Mistral3Model": "llava",
"NemotronH_Omni_Reasoning_V3": "nemotron",
conversion/minicpm.py
+15
View File
@@ -272,3 +272,18 @@ class MiniCPMV4_7VisionModel(MiniCPMV4_6VisionModel):
yield self.format_tensor_name(gguf.MODEL_TENSOR.V_TOK_EMBD_SEP, suffix=""), data_torch[ids]
return
yield from super().modify_tensors(data_torch, name, bid)
# MiniCPM-OCR 1.0: the 4.6 stack with model_type minicpm_ocr.
@ModelBase.register("MiniCPMOcrForConditionalGeneration")
@ModelBase.example("openbmb/MiniCPM-OCR-1.0")
class MiniCPMOCR1_0TextModel(MiniCPMV4_6TextModel):
model_arch = gguf.MODEL_ARCH.QWEN35
no_mtp = True
@ModelBase.register("MiniCPMOcrForConditionalGeneration")
@ModelBase.example("openbmb/MiniCPM-OCR-1.0")
class MiniCPMOCR1_0VisionModel(MiniCPMV4_6VisionModel):
projector_type = gguf.VisionProjectorType.MINICPM_OCR_1_0
gguf-py/gguf/constants.py
+1
View File
@@ -6452,6 +6452,7 @@ class VisionProjectorType:
MINIMAXM3 = "minimax_m3"
MINICPMV4_6 = "minicpmv4_6"
MINICPMV4_7 = "minicpmv4_7"
MINICPM_OCR_1_0 = "minicpm_ocr_1_0"
GRANITE_SPEECH = "granite_speech" # audio
MIMOVL = "mimovl"
MIMO_AUDIO = "mimo_audio"
tools/mtmd/clip-impl.h
+2
View File
@@ -537,6 +537,7 @@ enum projector_type {
PROJECTOR_TYPE_EXAONE4_5,
PROJECTOR_TYPE_MINICPMV4_6,
PROJECTOR_TYPE_MINICPMV4_7,
PROJECTOR_TYPE_MINICPM_OCR_1_0,
PROJECTOR_TYPE_GRANITE_SPEECH,
PROJECTOR_TYPE_MIMOVL,
PROJECTOR_TYPE_MINIMAX_M3,
@@ -611,6 +612,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_HUNYUANVL, "hunyuanvl"},
{ PROJECTOR_TYPE_MINICPMV4_6, "minicpmv4_6"},
{ PROJECTOR_TYPE_MINICPMV4_7, "minicpmv4_7"},
{ PROJECTOR_TYPE_MINICPM_OCR_1_0, "minicpm_ocr_1_0"},
{ PROJECTOR_TYPE_GRANITE_SPEECH, "granite_speech"},
{ PROJECTOR_TYPE_MIMOVL, "mimovl"},
{ PROJECTOR_TYPE_MINIMAX_M3, "minimax_m3"},
tools/mtmd/clip.cpp
+9
View File
@@ -1031,6 +1031,7 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
} break;
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
case PROJECTOR_TYPE_MINICPM_OCR_1_0:
{
builder = std::make_unique<clip_graph_minicpmv4_6>(ctx, img);
} break;
@@ -1496,6 +1497,7 @@ struct clip_model_loader {
} break;
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
case PROJECTOR_TYPE_MINICPM_OCR_1_0:
{
// MiniCPM-V 4.6/4.7 unified merger projector
// ViT merger 2x2 + final merger 2x2 = 4x spatial merge per dimension
@@ -2527,6 +2529,7 @@ struct clip_model_loader {
|| model.proj_type == PROJECTOR_TYPE_MINICPMV
|| model.proj_type == PROJECTOR_TYPE_MINICPMV4_6
|| model.proj_type == PROJECTOR_TYPE_MINICPMV4_7
|| model.proj_type == PROJECTOR_TYPE_MINICPM_OCR_1_0
) && layer.ff_up_w && layer.ff_down_w && layer.ff_down_w->ne[0] == hparams.n_embd;
if (is_ffn_swapped) {
// swap up and down weights
@@ -2630,6 +2633,7 @@ struct clip_model_loader {
} break;
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
case PROJECTOR_TYPE_MINICPM_OCR_1_0:
{
const bool merger_required = hparams.n_merge == 4;
auto get_merger_tensor = [&](const std::string & name, bool required = true) {
@@ -4348,6 +4352,7 @@ int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img) {
case PROJECTOR_TYPE_STEP3VL:
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
case PROJECTOR_TYPE_MINICPM_OCR_1_0:
return img->nx() / (params.patch_size * params.n_merge);
case PROJECTOR_TYPE_DEEPSEEKOCR:
case PROJECTOR_TYPE_DEEPSEEKOCR2:
@@ -4379,6 +4384,7 @@ int clip_n_output_tokens_y(const clip_ctx * ctx, const clip_image_f32 * img) {
case PROJECTOR_TYPE_STEP3VL:
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
case PROJECTOR_TYPE_MINICPM_OCR_1_0:
return img->ny() / (params.patch_size * params.n_merge);
default:
break;
@@ -4445,6 +4451,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
} break;
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
case PROJECTOR_TYPE_MINICPM_OCR_1_0:
{
n_patches /= params.n_merge * params.n_merge;
} break;
@@ -5169,6 +5176,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
} break;
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
case PROJECTOR_TYPE_MINICPM_OCR_1_0:
{
const bool is_4x = hparams.n_merge == 2;
@@ -6437,6 +6445,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
return ctx->model.mm_model_proj->ne[0];
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
case PROJECTOR_TYPE_MINICPM_OCR_1_0:
return ctx->model.mm_ffn_down_w->ne[1];
case PROJECTOR_TYPE_GLM_EDGE:
return ctx->model.mm_model_mlp_3_w->ne[1];
tools/mtmd/mtmd-image.h
+11
View File
@@ -190,6 +190,17 @@ protected:
}
};
// MiniCPM-OCR 1.0: find_best_resize aligns to patch_size * n_merge (28 in 4x).
struct mtmd_image_preprocessor_minicpm_ocr_1_0 : mtmd_image_preprocessor_minicpmv {
using mtmd_image_preprocessor_minicpmv::mtmd_image_preprocessor_minicpmv;
protected:
int get_slice_align() const override {
const int merge = hparams.n_merge > 0 ? hparams.n_merge : 2;
return hparams.patch_size * merge;
}
};
// custom llava-uhd slicing logic for LFM2
// tiling params (min/max tiles, tile size, pixels tolerance) come from hparams
// ref: https://github.com/huggingface/transformers/blob/v5.1.0/src/transformers/models/lfm2_vl/image_processing_lfm2_vl_fast.py
tools/mtmd/mtmd.cpp
+6 -1
View File
@@ -722,6 +722,7 @@ struct mtmd_context {
} break;
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
case PROJECTOR_TYPE_MINICPM_OCR_1_0:
{
slice_tmpl = MTMD_SLICE_TMPL_MINICPMV_2_6;
tok_ov_img_start = {lookup_token("<image>")};
@@ -732,7 +733,11 @@ struct mtmd_context {
tok_row_end_trail = false; // no trailing end-of-row token
ov_img_first = true;
use_image_id = true;
image_preproc = std::make_unique<mtmd_image_preprocessor_minicpmv>(ctx_v);
if (clip_get_projector_type(ctx_v) == PROJECTOR_TYPE_MINICPM_OCR_1_0) {
image_preproc = std::make_unique<mtmd_image_preprocessor_minicpm_ocr_1_0>(ctx_v);
} else {
image_preproc = std::make_unique<mtmd_image_preprocessor_minicpmv>(ctx_v);
}
} break;
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL: