imatrix: calculate activation-based statistics for new format (GGUF) imatrices (#14891)

* Use activations to calculate the stats * Determine calculation mode * Compute entropy for activations * Compute cosine similarity based on activations * Compute l2 norm * Add compute_layer_statistics() function * Update aggregated statistic report layout * Fix printing l2 norm when calc_mode = 1 * Refactor variable name * Compute aggregated (per layer) l2 norm * Update aggregated sum of squared activations per layer * Make ZD Score two-tailed * Update report layout * Reverse conditional logic to match convention * Rename report heading * Add --activation-statistics parameter * Add Euclidean–Cosine Score (ECS) * Add --activation-statistics logic to avoid doubling the imatrix size by default * Update stats output sort based on imatrix type * Process external NextN draft files (-md / --model-draft) * Refactor to use new llama_batch_ext Co-authored-by: compilade <git@compilade.net> Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
2026-10-04 10:24:53 +01:00
parent 8330e96967
commit 0504396140
9 changed files with 1210 additions and 288 deletions
@@ -3180,6 +3180,13 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
            params.process_output = true;
        }
    ).set_examples({LLAMA_EXAMPLE_IMATRIX}));
+    add_opt(common_arg(
+        {"--nextn"},
+        string_format("collect data for MTP/NextN layers (default: %s)", params.load_mtp ? "true" : "false"),
+        [](common_params & params) {
+            params.load_mtp = true;
+        }
+    ).set_examples({LLAMA_EXAMPLE_IMATRIX}));
    add_opt(common_arg(
        {"--ppl"},
        {"--no-ppl"},
@@ -4259,7 +4266,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
            params.speculative.draft.mparams.path = value;
            params.speculative.draft.mparams.hf_file = value; // will be used if --spec-draft-hf is set
        }
-    ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_MODEL"));
+    ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI, LLAMA_EXAMPLE_IMATRIX}).set_env("LLAMA_ARG_SPEC_DRAFT_MODEL"));
    add_opt(common_arg(
        {"--spec-type"}, common_speculative_all_types_str(),
        string_format("comma-separated list of types of speculative decoding to use (default: %s)\n",
@@ -1630,7 +1630,7 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
    mparams.progress_callback           = params.load_progress_callback;
    mparams.progress_callback_user_data = params.load_progress_callback_user_data;
    mparams.no_alloc                    = params.no_alloc;
-    mparams.load_mtp                    = std::find(params.speculative.types.begin(), params.speculative.types.end(), COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end();
+    mparams.load_mtp                    = params.load_mtp || std::find(params.speculative.types.begin(), params.speculative.types.end(), COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end();

    return mparams;
 }
@@ -586,6 +586,7 @@ struct common_params {
    bool no_op_offload     = false; // globally disable offload host tensor operations to device
    bool no_extra_bufts    = false; // disable extra buffer types (used for weight repacking)
    bool no_host           = false; // bypass host buffer allowing extra buffers to be used
+    bool load_mtp          = false; // load MTP/NextN layers

    bool single_turn       = false; // single turn chat conversation

@@ -724,10 +725,11 @@ struct common_params {
    int32_t i_chunk     =  0; // start processing from this chunk
    int8_t  imat_dat    =  0; // whether the legacy imatrix.dat format should be output (gguf <= 0 < dat)

-    bool process_output  = false; // collect data for the output tensor
-    bool compute_ppl     = true;  // whether to compute perplexity
-    bool show_statistics = false; // show imatrix statistics per tensor
-    bool parse_special   = false; // whether to parse special tokens during imatrix tokenization
+    bool process_output         = false; // collect data for the output tensor
+    bool compute_ppl            = true;  // whether to compute perplexity
+    bool show_statistics        = false; // show imatrix statistics per tensor
+    bool activation_statistics  = false; // generate data to calculate activation based statistics
+    bool parse_special          = false; // whether to parse special tokens during imatrix tokenization

    // cvector-generator params
    int n_pca_batch = 100;
@@ -98,9 +98,10 @@ bool common_imatrix_load(const std::string & fname, common_imatrix & imatrix) {
        return false;
    }

-    const int64_t datasets_key   = gguf_find_key(ctx_gguf, LLM_KV_IMATRIX_DATASETS);
+    const int64_t datasets_key    = gguf_find_key(ctx_gguf, LLM_KV_IMATRIX_DATASETS);
    const int64_t chunk_count_key = gguf_find_key(ctx_gguf, LLM_KV_IMATRIX_CHUNK_COUNT);
    const int64_t chunk_size_key  = gguf_find_key(ctx_gguf, LLM_KV_IMATRIX_CHUNK_SIZE);
+    const int64_t nextn_key       = gguf_find_key(ctx_gguf, LLM_KV_IMATRIX_N_LAYER_NEXTN);

    if (datasets_key != -1 && gguf_get_kv_type(ctx_gguf, datasets_key) == GGUF_TYPE_ARRAY &&
        gguf_get_arr_type(ctx_gguf, datasets_key) == GGUF_TYPE_STRING) {
@@ -111,33 +112,42 @@ bool common_imatrix_load(const std::string & fname, common_imatrix & imatrix) {
        }
    }

-    imatrix.has_metadata = (datasets_key != -1 && chunk_count_key != -1 && chunk_size_key != -1);
-    imatrix.chunk_count  = (chunk_count_key != -1) ? gguf_get_val_u32(ctx_gguf, chunk_count_key) : 0;
-    imatrix.chunk_size   = (chunk_size_key  != -1) ? gguf_get_val_u32(ctx_gguf, chunk_size_key)  : 0;
+    imatrix.has_metadata  = datasets_key != -1 && chunk_count_key != -1 && chunk_size_key != -1;
+    imatrix.chunk_count   = chunk_count_key != -1 ? gguf_get_val_u32(ctx_gguf, chunk_count_key) : 0;
+    imatrix.chunk_size    = chunk_size_key  != -1 ? gguf_get_val_u32(ctx_gguf, chunk_size_key) : 0;
+    imatrix.n_layer_nextn = nextn_key       != -1 ? gguf_get_val_u32(ctx_gguf, nextn_key) : 0;

+    const std::string in_sum_suffix{ ".in_sum" };
    const std::string in_sum2_suffix{ ".in_sum2" };
    const std::string counts_suffix{ ".counts" };

-    std::map<std::string, std::pair<struct ggml_tensor *, struct ggml_tensor *>> sums_counts_for;
+    struct sum_tensors {
+        struct ggml_tensor * in_sum  = nullptr;
+        struct ggml_tensor * in_sum2 = nullptr;
+        struct ggml_tensor * counts  = nullptr;
+    };

+    std::map<std::string, sum_tensors> sums_counts_for;
    for (struct ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {
        std::string name = cur->name;
-
        if (name.empty()) { continue; }

-        if (string_remove_suffix(name, in_sum2_suffix)) {
-            sums_counts_for[std::move(name)].first = cur;
+        if (string_remove_suffix(name, in_sum_suffix)) {
+            sums_counts_for[std::move(name)].in_sum = cur;
+        } else if (string_remove_suffix(name, in_sum2_suffix)) {
+            sums_counts_for[std::move(name)].in_sum2 = cur;
        } else if (string_remove_suffix(name, counts_suffix)) {
-            sums_counts_for[std::move(name)].second = cur;
+            sums_counts_for[std::move(name)].counts = cur;
        }
    }

    for (const auto & sc : sums_counts_for) {
        const std::string &        name    = sc.first;
-        const struct ggml_tensor * in_sum2 = sc.second.first;
-        const struct ggml_tensor * counts  = sc.second.second;
+        const struct ggml_tensor * in_sum  = sc.second.in_sum;
+        const struct ggml_tensor * in_sum2 = sc.second.in_sum2;
+        const struct ggml_tensor * counts  = sc.second.counts;

-        if (!in_sum2 || !counts) {
+        if (!in_sum2 || !counts || (in_sum != nullptr && ggml_nelements(in_sum) != ggml_nelements(in_sum2))) {
            LOG_ERR("%s: mismatched sums and counts for %s\n", __func__, name.c_str());
            gguf_free(ctx_gguf);
            ggml_free(ctx);
@@ -165,6 +175,13 @@ bool common_imatrix_load(const std::string & fname, common_imatrix & imatrix) {
        for (int64_t j = 0; j < ncounts; ++j) {
            e.counts[j] = std::lround(((const float *) counts->data)[j]);
        }
+
+        if (in_sum && ggml_nelements(in_sum) == nval) {
+            e.activations.resize(nval);
+            for (int64_t j = 0; j < nval; ++j) {
+                e.activations[j] = ((const float *) in_sum->data)[j];
+            }
+        }
    }

    gguf_free(ctx_gguf);
@@ -8,9 +8,12 @@
 inline constexpr const char * LLM_KV_IMATRIX_DATASETS    = "imatrix.datasets";
 inline constexpr const char * LLM_KV_IMATRIX_CHUNK_COUNT = "imatrix.chunk_count";
 inline constexpr const char * LLM_KV_IMATRIX_CHUNK_SIZE  = "imatrix.chunk_size";
+inline constexpr const char * LLM_KV_IMATRIX_STATS_SCHEMA = "imatrix.stats_schema";
+inline constexpr const char * LLM_KV_IMATRIX_N_LAYER_NEXTN = "imatrix.n_layer_nextn";

 struct common_imatrix_entry {
    std::vector<float>   sums;
+    std::vector<float>   activations;
    std::vector<int64_t> counts;
 };

@@ -19,6 +22,7 @@ struct common_imatrix {
    std::vector<std::string> datasets;
    int32_t chunk_count    = 0;
    int32_t chunk_size     = 0;
+    int32_t n_layer_nextn  = 0;
    bool    is_legacy      = false;
    bool    has_metadata   = false;
 };
@@ -103,7 +103,7 @@ struct common_speculative_config {
            const common_params_speculative & p = common_params_speculative{}) : type(t), params(p) {}
 };

-static bool common_speculative_are_compatible(
+bool common_speculative_are_compatible(
    const llama_model * model_tgt,
    const llama_model * model_dft) {
    const llama_vocab * vocab_tgt = llama_model_get_vocab(model_tgt);
@@ -46,6 +46,9 @@ struct common_speculative_output_limits {
 common_speculative_output_limits common_speculative_get_output_limits(
        int32_t n_batch, int32_t n_parallel, int32_t n_draft);

+// return true if the target and draft models have compatible vocabs
+bool common_speculative_are_compatible(const llama_model * model_tgt, const llama_model * model_dft);
+
 common_speculative * common_speculative_init(common_params_speculative & params, uint32_t n_seq);

 void common_speculative_free(common_speculative * spec);
@@ -8,9 +8,9 @@ More information is available in <https://github.com/ggml-org/llama.cpp/pull/486
 ```
 ./llama-imatrix \
    -m model.gguf -f some-text.txt [-o imatrix.gguf] [--output-format {gguf,dat}] [--no-ppl] \
-    [--process-output] [--chunk 123] [--save-frequency 0] [--output-frequency 10] \
-    [--in-file imatrix-prev-0.gguf --in-file imatrix-prev-1.gguf ...] [--parse-special] \
-    [--show-statistics] [...]
+    [--process-output] [--nextn] [--model-draft draft.gguf] [--chunk 123] [--save-frequency 0] \
+    [--output-frequency 10] [--in-file imatrix-prev-0.gguf --in-file imatrix-prev-1.gguf ...] \
+    [--parse-special] [--show-statistics] [...]
 ```

 Here `-m | --model` with a model name and `-f | --file` with a file containing calibration data (such as e.g. `wiki.train.raw`) are mandatory.
@@ -20,13 +20,15 @@ The parameters in square brackets are optional and have the following meaning:
 * `-lv | --verbosity` specifies the verbosity level. If set to `0`, no output other than the perplexity of the processed chunks will be generated. If set to `1`, each time the results are saved a message is written to `stderr`. If `>=2`, a message is output each time data is collected for any tensor. Default verbosity level is `1`.
 * `-o | --output-file` specifies the name of the file where the computed data will be stored. If missing `imatrix.gguf` is used.
 * `-ofreq | --output-frequency` specifies how often the so far computed result is saved to disk. Default is 10 (i.e., every 10 chunks)
-* `--output-format` specifies the output format of the generated imatrix file. Either "gguf", or "dat" (the legacy format). Defaults to "gguf".
+* `--output-format` specifies the output format of the generated imatrix file. Either `gguf`, or `dat` (the legacy format). Defaults to `gguf`.
 * `--save-frequency` specifies how often to save a copy of the imatrix in a separate file. Default is 0 (i.e., never)
-* `--process-output` specifies if data will be collected for the `output.weight` tensor. Typically, it is better not to utilize the importance matrix when quantizing `output.weight`, so this is set to `false` by default.
+* `--process-output` specifies if data will be collected for `output.weight` and `token_embd.weight` (when the model ties its embeddings to the output). Typically, it is better not to utilize the importance matrix when quantizing these tensors, so it's set to `false` by default.
+* `--nextn` collects data for MTP/NextN layers if present. Needs a single sequence per batch (i.e. set `--batch-size` ≤ `--ctx-size`). Cannot be used on models where NextN layers share the trunk's KV cache.
+* `-md | --model-draft FNAME` specifies an external NextN draft file. Use only if the model does not already include NextN layers.
 * `--in-file` one or more existing imatrix files to load and combine. Useful for merging files from multiple runs/datasets.
 * `--parse-special` enables parsing of special tokens (e.g., `<|im_start|>` in some models). Useful for models with custom tokenizers.
 * `--chunk | --from-chunk` to skip the first `n` chunks of tokens from the input data. Useful for resuming or skipping initial low-quality data.
-* `--chunks` maximum number of chunks to process. Default is -1 for all available chunks.
+* `--chunks` maximum number of chunks to process. Default is `-1` for all available chunks.
 * `--no-ppl` disables the calculation of perplexity for the processed chunks. Useful if you want to speed up the processing and do not care about perplexity.
 * `--show-statistics` displays imatrix file's statistics.

@@ -44,6 +46,16 @@ Recent versions of `llama-imatrix` store data in GGUF format by default. For the
 ./llama-quantize --imatrix imatrix.gguf ggml-model-f16.gguf ./ggml-model-q4_k_m.gguf q4_k_m
 ```

+```bash
+# generate importance matrix including embedded NextN layers
+./llama-imatrix -m ggml-model-f16.gguf -f calibration-data.txt --nextn -o imatrix-mtp.gguf -ngl 99
+```
+
+```bash
+# generate importance matrix including stand-alone NextN draft
+./llama-imatrix -m ggml-model-f16.gguf -f calibration-data.txt --model-draft draft.gguf -o imatrix-mtp.gguf -ngl 99
+```
+
 ```bash
 # generate and save the imatrix using legacy format
 ./llama-imatrix -m ggml-model-f16.gguf -f calibration-data.txt --output-format dat -o imatrix-legcy-format.dat -ngl 99
@@ -70,29 +82,49 @@ Recent versions of `llama-imatrix` store data in GGUF format by default. For the
 ```

 ```bash
-# analyse imatrix file and display summary statistics instead of running inference
-./llama-imatrix --in-file imatrix.gguf --show-statistics
+# analyze imatrix file and display summary statistics instead of running inference
+./llama-imatrix -m ggml-model-f16.gguf --in-file imatrix.gguf --show-statistics
 ```

-`--show-statistics` will display the following statistics:
+## Statistics
+Please note that if a value lacks statistical interpretability, **nan** will be shown instead.

 #### Per tensor
+Statistical properties of a single tensor's average activation or activation energy (squared magnitude).

-* Σ(Act²): sum of all squared activations (the importance scores)
-* Min & Max: minimum and maximum squared activations values
-* μ & σ: Squared activations' mean and standard deviation
-* % Active: proportion of elements whose average squared activation exceeds a small threshold (1e-5). Helpful to determine how alive/dormant the tensor is during inference
-* N: number of squared activations
-* Entropy: entropy of the squared activation distribution, in bits (standard Shannon entropy measurement) $S = -\sum_{i=1}^N p_i \log_2 p_i$
-* E (norm): Normalized entropy. $E(norm)=\frac{-\sum_{i=1}^N p_i \log_2 p_i}{log_2 N}$. These two metrics can be used to determine how well a prompt "exercises" the model's capabilities
-* ZD Score: z-score distribution as described in _3.1 Layer Importance Scores_ of [Layer-Wise Quantization](https://arxiv.org/abs/2406.17415)
-* CosSim: cosine similarity with respect to the previous layer's tensor. Useful to determine how similar the squared activations of the current layer are to the previous layer's squared activations.
+* **Mean / StdDev**: $\mu = \frac{1}{N} \sum v_i$ and $\sigma = \sqrt{\frac{1}{N} \sum (v_i - \mu)^2}$
+  - Establishes the baseline distribution of the tensor's outputs. Low variance means the tensor outputs a mostly constant projection; high variance implies high information density across dimensions.
+* **Skewness & Kurtosis**: $skew = \frac{\frac{1}{N} \sum (v_i - \mu)^3}{\sigma^3}$ and $kurt = \frac{\frac{1}{N} \sum (v_i - \mu)^4}{\sigma^4} - 3.0$
+  - Skewness measures the asymmetry of a distribution around its mean. Kurtosis measures the "tailedness" of the feature activations. A high kurtosis indicates a highly sparse/heavy-tailed activation distribution (e.g., outlier features). High-kurtosis tensors typically require higher precision quantization to prevent outlier degradation.
+* **H Norm**: $H = -\sum_{i} P_i \log_2(P_i)$, where $P_i = \frac{v_{val_i}}{\sum v_{val_i}}$
+  - Shannon Entropy normalized over log₂(N). Used to determine how well a prompt "exercises" the model's capabilities. Higher values indicate more uniform distribution of activations. Every neuron is firing equally; hard to prune.
+* **$\sum E[A^2]$**: $\sum E[x_i^2]$
+  - The sum of squares of activations (Energy) for the tensor. Tensors with high "energy" contribute most to the final output. Quantization errors here propagate strongly. These tensors usually need higher precision.
+
+#### Intra-layer
+These statistics compare the same tensor between the current layer $L$ and the nearest previous layer that also has it (e.g., `blk.1.attn_v` vs `blk.0.attn_v`). That is usually $L-1$, but a model that does not carry every tensor in every layer can pair across a gap.
+
+* **Gain**: $G = \frac{\sqrt{\sum C_i^2 / N_{curr}}}{\sqrt{\sum P_i^2 / N_{prev}}}$
+  - Indicates if a layer acts as an "amplifier" ($G > 1$) or a "dampener" ($G < 1$).
+* **L2 Distance**: $L2 = \sqrt{ \sum (C_i - P_i)^2 }$ where $C$ is the current layer and $P$ is the previous layer's tensor.
+  - Measures absolute representational shift. Huge leaps in L2 distance indicate that a layer fundamentally transforms the hidden states.
+* **Pearson Correlation Coefficient (PCC)**: $r = \frac{\sum (C_i - \bar{C})(P_i - \bar{P})}{\sqrt{\sum (C_i - \bar{C})^2} \sqrt{\sum (P_i - \bar{P})^2}}$
+  - Similar to Cosine Similarity, but invariant to scalar or offset biases (centers the data first). Highly correlated adjacent layers signify structural repetition.
+* **Covariance (Cov)**: $\frac{1}{N}\sum (c_i-\bar c)(p_i-\bar p)$
+  - The **unnormalized covariance** between current and previous layer activations. Captures both the correlation structure and the magnitude of the joint variation. Large absolute covariance indicates the layers are jointly processing strong, correlated signals.

 #### Per layer
+Aggregated metrics per block/layer:

-Weighted averages of Σ(Act²), ZD Score and CosSim are also calculated.
+* **$\sum E[A^2]$**: Total energy of the layer's concatenated tensors. Indicates the layer's overall contribution amplitude.
+* **Gain**: $G = \frac{\sqrt{\sum E_{curr} / \sum N_{curr}}}{\sqrt{\sum E_{prev} / \sum N_{prev}}}$
+  - Indicates if a layer acts as an "amplifier" ($G > 1$) or a "dampener" ($G < 1$). Only tensors that have a match in a previous layer are counted, and both sides of the ratio use that same set of tensors.
+* **L2 Distance**: Euclidean Distance of the layer's concatenated tensors from the previous layer's. Global measure of transformation magnitude.
+* **CosSim**: $\frac{\sum (C \cdot P)}{\sqrt{\sum C^2} \sqrt{\sum P^2}}$
+  - Cosine Similarity of the current layer's concatenated tensors with the previous layer.
+* **PCC**: $r = \frac{\sum Cov}{\sqrt{\sum Var_{curr}} \sqrt{\sum Var_{prev}}}$
+  - Pooled Pearson Correlation over the tensors in the layer.
+* **Cov**: $\frac{\sum Cov}{\sum N}$, where $N$ is the number of elements with data in both layers.
+  - The **unnormalized covariance** between the current layer's tensors and the previous layer's.

-#### Important note on the computed Statistics
-
-When using these statistics, please note that they are computed on the squared activations, **not on the actual (raw) activations**.
-Whilst the results are still useful, they're less reliable than using the raw values, and in the case of the cosine similarity, could be misleading if the tensor contains opposite vectors.
+More information is available in https://github.com/ggml-org/llama.cpp/pull/14891