From ae2a9c8195b129819f8e80cf3aaf896f627caa13 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Thu, 8 Feb 2024 12:50:25 +0200 Subject: [PATCH] lib/mergeset: prevent from possible `too big indexBlockSize` panic This panic could occur when samples with too long label values are ingested into VictoriaMetrics. This could result in too long fistItem and commonPrefix values at blockHeader (up to 64kb each). This may inflate the maximum index block size by 4 * maxIndexBlockSize. --- docs/CHANGELOG.md | 1 + lib/mergeset/metaindex_row.go | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 88f43ccbb8..c9185b8f74 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -40,6 +40,7 @@ The sandbox cluster installation is running under the constant load generated by * BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): properly propagate [label filters](https://docs.victoriametrics.com/keyconcepts/#filtering) from multiple arguments passed to [aggregate functions](https://docs.victoriametrics.com/metricsql/#aggregate-functions). For example, `sum({job="foo"}, {job="bar"}) by (job) + a` was improperly optimized to `sum({job="foo"}, {job="bar"}) by (job) + a{job="foo"}` before being executed. This could lead to unexpected results. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5604). * BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): properly handle precision errors when calculating [changes](https://docs.victoriametrics.com/metricsql/#changes), [changes_prometheus](https://docs.victoriametrics.com/metricsql/#changes_prometheus), [increases_over_time](https://docs.victoriametrics.com/metricsql/#increases_over_time) and [resets](https://docs.victoriametrics.com/metricsql/#resets) functions. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/767). * BUGFIX: all VictoriaMetrics components: consistently return 200 http status code from [`/-/reload` endpoint](https://docs.victoriametrics.com/vmagent/#configuration-update). Previously [single-node VictoriaMetrics](https://docs.victoriametrics.com/) was returning 204 http status code. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5774). +* BUGFIX: prevent from possible `too big indexBlockSize` panic. * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix the graph dragging for Firefox and Safari. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5764). * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix handling invalid timezone. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5732). * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix the bug where the select does not open. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5728). diff --git a/lib/mergeset/metaindex_row.go b/lib/mergeset/metaindex_row.go index c06a4a1948..e80ba19a4f 100644 --- a/lib/mergeset/metaindex_row.go +++ b/lib/mergeset/metaindex_row.go @@ -72,8 +72,11 @@ func (mr *metaindexRow) Unmarshal(src []byte) ([]byte, error) { if mr.blockHeadersCount <= 0 { return src, fmt.Errorf("blockHeadersCount must be bigger than 0; got %d", mr.blockHeadersCount) } - if mr.indexBlockSize > 2*maxIndexBlockSize { - return src, fmt.Errorf("too big indexBlockSize: %d; cannot exceed %d", mr.indexBlockSize, 2*maxIndexBlockSize) + if mr.indexBlockSize > 4*maxIndexBlockSize { + // The index block size can exceed maxIndexBlockSize by up to 4x, + // since it can contain commonPrefix and firstItem at blockHeader + // with the maximum length of maxIndexBlockSize per each field. + return src, fmt.Errorf("too big indexBlockSize: %d; cannot exceed %d", mr.indexBlockSize, 4*maxIndexBlockSize) } return src, nil