mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-12-05 01:01:09 +01:00
lib/logstorage: print column names to the error log message when the number of unique columns in a block exceeds the limit
This should simplify debugging issues related to too big number of columns per block in the future. Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7568
This commit is contained in:
parent
d2cd004710
commit
4d39dfc0ee
@ -392,8 +392,9 @@ func (b *block) resizeColumns(columnsLen int) []column {
|
|||||||
|
|
||||||
func (b *block) sortColumnsByName() {
|
func (b *block) sortColumnsByName() {
|
||||||
if len(b.columns)+len(b.constColumns) > maxColumnsPerBlock {
|
if len(b.columns)+len(b.constColumns) > maxColumnsPerBlock {
|
||||||
logger.Panicf("BUG: too big number of columns detected in the block: %d; the number of columns mustn't exceed %d",
|
columnNames := b.getColumnNames()
|
||||||
len(b.columns)+len(b.constColumns), maxColumnsPerBlock)
|
logger.Panicf("BUG: too big number of columns detected in the block: %d; the number of columns mustn't exceed %d; columns: %s",
|
||||||
|
len(b.columns)+len(b.constColumns), maxColumnsPerBlock, columnNames)
|
||||||
}
|
}
|
||||||
|
|
||||||
cs := getColumnsSorter()
|
cs := getColumnsSorter()
|
||||||
@ -407,6 +408,17 @@ func (b *block) sortColumnsByName() {
|
|||||||
putConstColumnsSorter(ccs)
|
putConstColumnsSorter(ccs)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (b *block) getColumnNames() []string {
|
||||||
|
a := make([]string, 0, len(b.columns)+len(b.constColumns))
|
||||||
|
for _, c := range b.columns {
|
||||||
|
a = append(a, c.name)
|
||||||
|
}
|
||||||
|
for _, c := range b.constColumns {
|
||||||
|
a = append(a, c.Name)
|
||||||
|
}
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
|
||||||
// Len returns the number of log entries in b.
|
// Len returns the number of log entries in b.
|
||||||
func (b *block) Len() int {
|
func (b *block) Len() int {
|
||||||
return len(b.timestamps)
|
return len(b.timestamps)
|
||||||
|
@ -492,8 +492,8 @@ func (csh *columnsHeader) unmarshalNoArena(src []byte, partFormatVersion uint) e
|
|||||||
return fmt.Errorf("cannot unmarshal columnHeaders len")
|
return fmt.Errorf("cannot unmarshal columnHeaders len")
|
||||||
}
|
}
|
||||||
src = src[nSize:]
|
src = src[nSize:]
|
||||||
if n > maxColumnsPerBlock {
|
if n > 1e6 {
|
||||||
return fmt.Errorf("too many column headers: %d; mustn't exceed %d", n, maxColumnsPerBlock)
|
return fmt.Errorf("too big number of columnHeaders: %d", n)
|
||||||
}
|
}
|
||||||
|
|
||||||
chs := csh.resizeColumnHeaders(int(n))
|
chs := csh.resizeColumnHeaders(int(n))
|
||||||
@ -506,14 +506,19 @@ func (csh *columnsHeader) unmarshalNoArena(src []byte, partFormatVersion uint) e
|
|||||||
}
|
}
|
||||||
csh.columnHeaders = chs
|
csh.columnHeaders = chs
|
||||||
|
|
||||||
|
if len(chs) > maxColumnsPerBlock {
|
||||||
|
columnNames := getNamesFromColumnHeaders(chs)
|
||||||
|
return fmt.Errorf("too many column headers: %d; it mustn't exceed %d; columns: %s", len(chs), maxColumnsPerBlock, columnNames)
|
||||||
|
}
|
||||||
|
|
||||||
// unmarshal constColumns
|
// unmarshal constColumns
|
||||||
n, nSize = encoding.UnmarshalVarUint64(src)
|
n, nSize = encoding.UnmarshalVarUint64(src)
|
||||||
if nSize <= 0 {
|
if nSize <= 0 {
|
||||||
return fmt.Errorf("cannot unmarshal constColumns len")
|
return fmt.Errorf("cannot unmarshal constColumns len")
|
||||||
}
|
}
|
||||||
src = src[nSize:]
|
src = src[nSize:]
|
||||||
if n+uint64(len(csh.columnHeaders)) > maxColumnsPerBlock {
|
if n > 1e6 {
|
||||||
return fmt.Errorf("too many columns: %d; mustn't exceed %d", n+uint64(len(csh.columnHeaders)), maxColumnsPerBlock)
|
return fmt.Errorf("too big number of constColumns: %d", n)
|
||||||
}
|
}
|
||||||
|
|
||||||
ccs := csh.resizeConstColumns(int(n))
|
ccs := csh.resizeConstColumns(int(n))
|
||||||
@ -525,6 +530,14 @@ func (csh *columnsHeader) unmarshalNoArena(src []byte, partFormatVersion uint) e
|
|||||||
src = tail
|
src = tail
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if len(ccs)+len(csh.columnHeaders) > maxColumnsPerBlock {
|
||||||
|
columnNames := getNamesFromColumnHeaders(csh.columnHeaders)
|
||||||
|
for _, cc := range ccs {
|
||||||
|
columnNames = append(columnNames, cc.Name)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("too many columns: %d; mustn't exceed %d; columns: %s", len(ccs)+len(csh.columnHeaders), maxColumnsPerBlock, columnNames)
|
||||||
|
}
|
||||||
|
|
||||||
// Verify that the src is empty
|
// Verify that the src is empty
|
||||||
if len(src) > 0 {
|
if len(src) > 0 {
|
||||||
return fmt.Errorf("unexpected non-empty tail left after unmarshaling columnsHeader: len(tail)=%d", len(src))
|
return fmt.Errorf("unexpected non-empty tail left after unmarshaling columnsHeader: len(tail)=%d", len(src))
|
||||||
@ -533,6 +546,14 @@ func (csh *columnsHeader) unmarshalNoArena(src []byte, partFormatVersion uint) e
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func getNamesFromColumnHeaders(chs []columnHeader) []string {
|
||||||
|
a := make([]string, 0, len(chs))
|
||||||
|
for _, ch := range chs {
|
||||||
|
a = append(a, ch.name)
|
||||||
|
}
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
|
||||||
// columnHeaders contains information for values, which belong to a single label in a single block.
|
// columnHeaders contains information for values, which belong to a single label in a single block.
|
||||||
//
|
//
|
||||||
// The main column with an empty name is stored in messageValuesFilename,
|
// The main column with an empty name is stored in messageValuesFilename,
|
||||||
|
Loading…
Reference in New Issue
Block a user