From 0416ed2bb88735bfc2edacf3e4bf35ee4a6187c7 Mon Sep 17 00:00:00 2001 From: Francis Couture-Harpin Date: Sun, 3 Aug 2025 16:28:37 -0400 Subject: [PATCH 1/3] quantize : configurable neutral imatrix prior --- tools/quantize/quantize.cpp | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/tools/quantize/quantize.cpp b/tools/quantize/quantize.cpp index 0e89a2b81b2a8..48d6747934e32 100644 --- a/tools/quantize/quantize.cpp +++ b/tools/quantize/quantize.cpp @@ -132,6 +132,7 @@ static void usage(const char * executable) { printf(" --prune-layers L0,L1,L2...comma-separated list of layer numbers to prune from the model\n"); printf(" Advanced option to remove all tensors from the given layers\n"); printf(" --keep-split: will generate quantized model in the same shards as input\n"); + printf(" --prior-weight N: how many tokens the neutral prior is worth (when using imatrix)\n"); printf(" --override-kv KEY=TYPE:VALUE\n"); printf(" Advanced option to override model metadata by key in the quantized model. May be specified multiple times.\n"); printf("Note: --include-weights and --exclude-weights cannot be used together\n"); @@ -213,7 +214,7 @@ static int load_legacy_imatrix(const std::string & imatrix_file, std::vector & imatrix_datasets, std::unordered_map> & imatrix_data) { +static int load_imatrix(const std::string & imatrix_file, std::vector & imatrix_datasets, std::unordered_map> & imatrix_data, float prior_weight) { struct ggml_context * ctx = nullptr; struct gguf_init_params meta_gguf_params = { @@ -289,7 +290,7 @@ static int load_imatrix(const std::string & imatrix_file, std::vectordata)[j]; if (count > 0.0f) { for (int64_t i = 0; i < ne0; ++i) { - e[j*ne0 + i] = ((const float *) sums->data)[j*ne0 + i] / count; + e[j*ne0 + i] = (((const float *) sums->data)[j*ne0 + i] + prior_weight) / (count + prior_weight); } } else { // Partial imatrix data, this tensor never got any input during calibration @@ -331,10 +332,11 @@ static int prepare_imatrix(const std::string & imatrix_file, std::vector & imatrix_dataset, const std::vector & included_weights, const std::vector & excluded_weights, - std::unordered_map> & imatrix_data) { + std::unordered_map> & imatrix_data, + float prior_weight) { int m_last_call = -1; if (!imatrix_file.empty()) { - m_last_call = load_imatrix(imatrix_file, imatrix_dataset, imatrix_data); + m_last_call = load_imatrix(imatrix_file, imatrix_dataset, imatrix_data, prior_weight); } if (imatrix_data.empty()) { return m_last_call; @@ -452,6 +454,7 @@ int main(int argc, char ** argv) { std::vector kv_overrides; std::vector tensor_types; std::vector prune_layers; + float prior_weight = 1.0f; for (; arg_idx < argc && strncmp(argv[arg_idx], "--", 2) == 0; arg_idx++) { if (strcmp(argv[arg_idx], "--leave-output-tensor") == 0) { @@ -510,6 +513,16 @@ int main(int argc, char ** argv) { } } else if (strcmp(argv[arg_idx], "--keep-split") == 0) { params.keep_split = true; + } else if (strcmp(argv[arg_idx], "--prior-weight") == 0) { + if (arg_idx < argc-1) { + try { + prior_weight = std::stof(argv[++arg_idx]); + } catch (...) { + usage(argv[0]); + } + } else { + usage(argv[0]); + } } else { usage(argv[0]); } @@ -525,7 +538,7 @@ int main(int argc, char ** argv) { std::vector imatrix_datasets; std::unordered_map> imatrix_data; - int m_last_call = prepare_imatrix(imatrix_file, imatrix_datasets, included_weights, excluded_weights, imatrix_data); + int m_last_call = prepare_imatrix(imatrix_file, imatrix_datasets, included_weights, excluded_weights, imatrix_data, prior_weight); if (!imatrix_data.empty()) { params.imatrix = &imatrix_data; { From 92383bfab3e4cdab0f85b60b127c7002866ca255 Mon Sep 17 00:00:00 2001 From: Francis Couture-Harpin Date: Mon, 4 Aug 2025 01:47:00 -0400 Subject: [PATCH 2/3] quantize : store metadata for prior weight used for imatrix --- tools/quantize/quantize.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tools/quantize/quantize.cpp b/tools/quantize/quantize.cpp index 48d6747934e32..a6a528b74aff6 100644 --- a/tools/quantize/quantize.cpp +++ b/tools/quantize/quantize.cpp @@ -69,6 +69,7 @@ static const char * const LLM_KV_QUANTIZE_IMATRIX_FILE = "quantize.imatrix static const char * const LLM_KV_QUANTIZE_IMATRIX_DATASET = "quantize.imatrix.dataset"; static const char * const LLM_KV_QUANTIZE_IMATRIX_N_ENTRIES = "quantize.imatrix.entries_count"; static const char * const LLM_KV_QUANTIZE_IMATRIX_N_CHUNKS = "quantize.imatrix.chunks_count"; +static const char * const LLM_KV_QUANTIZE_IMATRIX_PRIOR_W = "quantize.imatrix.prior_weight"; // TODO: share with imatrix.cpp static const char * const LLM_KV_IMATRIX_DATASETS = "imatrix.datasets"; @@ -214,7 +215,7 @@ static int load_legacy_imatrix(const std::string & imatrix_file, std::vector & imatrix_datasets, std::unordered_map> & imatrix_data, float prior_weight) { +static int load_imatrix(const std::string & imatrix_file, std::vector & imatrix_datasets, std::unordered_map> & imatrix_data, float & prior_weight) { struct ggml_context * ctx = nullptr; struct gguf_init_params meta_gguf_params = { @@ -224,6 +225,7 @@ static int load_imatrix(const std::string & imatrix_file, std::vector & included_weights, const std::vector & excluded_weights, std::unordered_map> & imatrix_data, - float prior_weight) { + float & prior_weight) { int m_last_call = -1; if (!imatrix_file.empty()) { m_last_call = load_imatrix(imatrix_file, imatrix_dataset, imatrix_data, prior_weight); @@ -574,6 +576,14 @@ int main(int argc, char ** argv) { kvo.val_i64 = m_last_call; kv_overrides.emplace_back(std::move(kvo)); } + + { + llama_model_kv_override kvo; + std::strcpy(kvo.key, LLM_KV_QUANTIZE_IMATRIX_PRIOR_W); + kvo.tag = LLAMA_KV_OVERRIDE_TYPE_FLOAT; + kvo.val_f64 = prior_weight; + kv_overrides.emplace_back(std::move(kvo)); + } } if (!kv_overrides.empty()) { kv_overrides.emplace_back(); From 46a8601140d3e1891316c93f8ea4ed4c73fe7f55 Mon Sep 17 00:00:00 2001 From: Francis Couture-Harpin Date: Tue, 5 Aug 2025 13:34:01 -0400 Subject: [PATCH 3/3] quantize : assume the neutral prior is equal imatrix weights --- tools/quantize/quantize.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tools/quantize/quantize.cpp b/tools/quantize/quantize.cpp index a6a528b74aff6..6846850bddfe9 100644 --- a/tools/quantize/quantize.cpp +++ b/tools/quantize/quantize.cpp @@ -291,8 +291,15 @@ static int load_imatrix(const std::string & imatrix_file, std::vectordata)[j]; if (count > 0.0f) { + float sumw = 0.0f; for (int64_t i = 0; i < ne0; ++i) { - e[j*ne0 + i] = (((const float *) sums->data)[j*ne0 + i] + prior_weight) / (count + prior_weight); + sumw += ((const float *) sums->data)[j*ne0 + i]; + } + // the neutral prior is equal weights, and it should reduce the variance by weighted-averaging with the mean + const float prior_value = sumw / ne0; + + for (int64_t i = 0; i < ne0; ++i) { + e[j*ne0 + i] = (((const float *) sums->data)[j*ne0 + i] + prior_value * prior_weight) / (count + prior_weight); } } else { // Partial imatrix data, this tensor never got any input during calibration