From cfdb6762e6ac051da8a812ff1ada6e6f8dad0dfc Mon Sep 17 00:00:00 2001 From: Roman Khavronenko Date: Mon, 29 Mar 2021 10:38:03 +0100 Subject: [PATCH] deployment: add new alert `TooHighChurnRate24h` (#1154) Alert `TooHighChurnRate24h` suppose to cover cases when churn rate is low but results in multiple times higher number than total number of active series. --- deployment/docker/alerts.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/deployment/docker/alerts.yml b/deployment/docker/alerts.yml index a0a32cdbd..3f2c95d04 100644 --- a/deployment/docker/alerts.yml +++ b/deployment/docker/alerts.yml @@ -106,6 +106,23 @@ groups: High Churn Rate tightly connected with database performance and may result in unexpected OOM's or slow queries." + - alert: TooHighChurnRate24h + expr: | + sum(increase(vm_new_timeseries_created_total[24h])) by(instance) + > + (sum(vm_cache_entries{type="storage/hour_metric_ids"}) by(instance) * 3) + for: 15m + labels: + severity: warning + annotations: + dashboard: "http://localhost:3000/d/wNf0q_kZk?viewPanel=66&var-instance={{ $labels.instance }}" + summary: "Too high number of new series on \"{{ $labels.instance }}\" created over last 24h" + description: "The number of created new time series over last 24h is 3x times higher than + current number of active series on \"{{ $labels.instance }}\".\n + This effect is known as Churn Rate.\n + High Churn Rate tightly connected with database performance and may + result in unexpected OOM's or slow queries." + - alert: TooHighSlowInsertsRate expr: | (