From 87b8e3790dc6002223c9cef21645e09780a7524c Mon Sep 17 00:00:00 2001 From: Jan Fajerski Date: Thu, 20 Oct 2022 13:06:31 +0200 Subject: [PATCH] docs/node-mixin: add fsMointpointSelector to alerts and dashboards (#2446) * docs/node-mixin: add fsMountpointSelector This adds the option to add a `mountpoint` selector to filesystem related alerts. The default is `mountpoint!=""`. * docs/node-mixins: add fsMountpointSelector to dashboards Signed-off-by: Jan Fajerski --- docs/node-mixin/alerts/alerts.libsonnet | 40 ++++++++++++------------ docs/node-mixin/config.libsonnet | 8 +++++ docs/node-mixin/dashboards/use.libsonnet | 10 +++--- docs/node-mixin/lib/prom-mixin.libsonnet | 4 +-- 4 files changed, 35 insertions(+), 27 deletions(-) diff --git a/docs/node-mixin/alerts/alerts.libsonnet b/docs/node-mixin/alerts/alerts.libsonnet index 6fa14546..2382ac29 100644 --- a/docs/node-mixin/alerts/alerts.libsonnet +++ b/docs/node-mixin/alerts/alerts.libsonnet @@ -8,11 +8,11 @@ alert: 'NodeFilesystemSpaceFillingUp', expr: ||| ( - node_filesystem_avail_bytes{%(nodeExporterSelector)s,%(fsSelector)s} / node_filesystem_size_bytes{%(nodeExporterSelector)s,%(fsSelector)s} * 100 < %(fsSpaceFillingUpWarningThreshold)d + node_filesystem_avail_bytes{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s} / node_filesystem_size_bytes{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s} * 100 < %(fsSpaceFillingUpWarningThreshold)d and - predict_linear(node_filesystem_avail_bytes{%(nodeExporterSelector)s,%(fsSelector)s}[6h], 24*60*60) < 0 + predict_linear(node_filesystem_avail_bytes{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s}[6h], 24*60*60) < 0 and - node_filesystem_readonly{%(nodeExporterSelector)s,%(fsSelector)s} == 0 + node_filesystem_readonly{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s} == 0 ) ||| % $._config, 'for': '1h', @@ -28,11 +28,11 @@ alert: 'NodeFilesystemSpaceFillingUp', expr: ||| ( - node_filesystem_avail_bytes{%(nodeExporterSelector)s,%(fsSelector)s} / node_filesystem_size_bytes{%(nodeExporterSelector)s,%(fsSelector)s} * 100 < %(fsSpaceFillingUpCriticalThreshold)d + node_filesystem_avail_bytes{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s} / node_filesystem_size_bytes{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s} * 100 < %(fsSpaceFillingUpCriticalThreshold)d and - predict_linear(node_filesystem_avail_bytes{%(nodeExporterSelector)s,%(fsSelector)s}[6h], 4*60*60) < 0 + predict_linear(node_filesystem_avail_bytes{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s}[6h], 4*60*60) < 0 and - node_filesystem_readonly{%(nodeExporterSelector)s,%(fsSelector)s} == 0 + node_filesystem_readonly{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s} == 0 ) ||| % $._config, 'for': '1h', @@ -48,9 +48,9 @@ alert: 'NodeFilesystemAlmostOutOfSpace', expr: ||| ( - node_filesystem_avail_bytes{%(nodeExporterSelector)s,%(fsSelector)s} / node_filesystem_size_bytes{%(nodeExporterSelector)s,%(fsSelector)s} * 100 < %(fsSpaceAvailableWarningThreshold)d + node_filesystem_avail_bytes{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s} / node_filesystem_size_bytes{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s} * 100 < %(fsSpaceAvailableWarningThreshold)d and - node_filesystem_readonly{%(nodeExporterSelector)s,%(fsSelector)s} == 0 + node_filesystem_readonly{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s} == 0 ) ||| % $._config, 'for': '30m', @@ -66,9 +66,9 @@ alert: 'NodeFilesystemAlmostOutOfSpace', expr: ||| ( - node_filesystem_avail_bytes{%(nodeExporterSelector)s,%(fsSelector)s} / node_filesystem_size_bytes{%(nodeExporterSelector)s,%(fsSelector)s} * 100 < %(fsSpaceAvailableCriticalThreshold)d + node_filesystem_avail_bytes{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s} / node_filesystem_size_bytes{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s} * 100 < %(fsSpaceAvailableCriticalThreshold)d and - node_filesystem_readonly{%(nodeExporterSelector)s,%(fsSelector)s} == 0 + node_filesystem_readonly{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s} == 0 ) ||| % $._config, 'for': '30m', @@ -84,11 +84,11 @@ alert: 'NodeFilesystemFilesFillingUp', expr: ||| ( - node_filesystem_files_free{%(nodeExporterSelector)s,%(fsSelector)s} / node_filesystem_files{%(nodeExporterSelector)s,%(fsSelector)s} * 100 < 40 + node_filesystem_files_free{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s} / node_filesystem_files{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s} * 100 < 40 and - predict_linear(node_filesystem_files_free{%(nodeExporterSelector)s,%(fsSelector)s}[6h], 24*60*60) < 0 + predict_linear(node_filesystem_files_free{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s}[6h], 24*60*60) < 0 and - node_filesystem_readonly{%(nodeExporterSelector)s,%(fsSelector)s} == 0 + node_filesystem_readonly{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s} == 0 ) ||| % $._config, 'for': '1h', @@ -104,11 +104,11 @@ alert: 'NodeFilesystemFilesFillingUp', expr: ||| ( - node_filesystem_files_free{%(nodeExporterSelector)s,%(fsSelector)s} / node_filesystem_files{%(nodeExporterSelector)s,%(fsSelector)s} * 100 < 20 + node_filesystem_files_free{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s} / node_filesystem_files{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s} * 100 < 20 and - predict_linear(node_filesystem_files_free{%(nodeExporterSelector)s,%(fsSelector)s}[6h], 4*60*60) < 0 + predict_linear(node_filesystem_files_free{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s}[6h], 4*60*60) < 0 and - node_filesystem_readonly{%(nodeExporterSelector)s,%(fsSelector)s} == 0 + node_filesystem_readonly{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s} == 0 ) ||| % $._config, 'for': '1h', @@ -124,9 +124,9 @@ alert: 'NodeFilesystemAlmostOutOfFiles', expr: ||| ( - node_filesystem_files_free{%(nodeExporterSelector)s,%(fsSelector)s} / node_filesystem_files{%(nodeExporterSelector)s,%(fsSelector)s} * 100 < 5 + node_filesystem_files_free{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s} / node_filesystem_files{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s} * 100 < 5 and - node_filesystem_readonly{%(nodeExporterSelector)s,%(fsSelector)s} == 0 + node_filesystem_readonly{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s} == 0 ) ||| % $._config, 'for': '1h', @@ -142,9 +142,9 @@ alert: 'NodeFilesystemAlmostOutOfFiles', expr: ||| ( - node_filesystem_files_free{%(nodeExporterSelector)s,%(fsSelector)s} / node_filesystem_files{%(nodeExporterSelector)s,%(fsSelector)s} * 100 < 3 + node_filesystem_files_free{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s} / node_filesystem_files{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s} * 100 < 3 and - node_filesystem_readonly{%(nodeExporterSelector)s,%(fsSelector)s} == 0 + node_filesystem_readonly{%(nodeExporterSelector)s,%(fsSelector)s,%(fsMountpointSelector)s} == 0 ) ||| % $._config, 'for': '1h', diff --git a/docs/node-mixin/config.libsonnet b/docs/node-mixin/config.libsonnet index f16796c4..86179c8f 100644 --- a/docs/node-mixin/config.libsonnet +++ b/docs/node-mixin/config.libsonnet @@ -16,6 +16,14 @@ // alerting, you can exclude them here, e.g. 'fstype!="tmpfs"'. fsSelector: 'fstype!=""', + // Select the mountpoint for filesystem-related queries. If left + // empty, all mountpoints are selected. For example if you have a + // special purpose tmpfs instance that has a fixed size and will + // always be 100% full, but you still want alerts and dashboards for + // other tmpfs instances, you can exclude those by mountpoint prefix + // like so: 'mountpoint!~"/var/lib/foo.*"'. + fsMountpointSelector: 'mountpoint!=""', + // Select the device for disk-related queries. If left empty, all // devices are selected. If you have unusual devices you don't // want to include in dashboards and alerting, you can exclude diff --git a/docs/node-mixin/dashboards/use.libsonnet b/docs/node-mixin/dashboards/use.libsonnet index 7b937b22..1602c134 100644 --- a/docs/node-mixin/dashboards/use.libsonnet +++ b/docs/node-mixin/dashboards/use.libsonnet @@ -307,12 +307,12 @@ local diskSpaceUtilisation = ||| sum without (device) ( max without (fstype, mountpoint) (( - node_filesystem_size_bytes{%(nodeExporterSelector)s, %(fsSelector)s, %(clusterLabel)s="$cluster"} + node_filesystem_size_bytes{%(nodeExporterSelector)s, %(fsSelector)s, %(fsMountpointSelector)s, %(clusterLabel)s="$cluster"} - - node_filesystem_avail_bytes{%(nodeExporterSelector)s, %(fsSelector)s, %(clusterLabel)s="$cluster"} + node_filesystem_avail_bytes{%(nodeExporterSelector)s, %(fsSelector)s, %(fsMountpointSelector)s, %(clusterLabel)s="$cluster"} ) != 0) ) - / scalar(sum(max without (fstype, mountpoint) (node_filesystem_size_bytes{%(nodeExporterSelector)s, %(fsSelector)s, %(clusterLabel)s="$cluster"}))) + / scalar(sum(max without (fstype, mountpoint) (node_filesystem_size_bytes{%(nodeExporterSelector)s, %(fsSelector)s, %(fsMountpointSelector)s, %(clusterLabel)s="$cluster"}))) ||| % $._config, legendFormat='{{instance}}' )) ) @@ -453,10 +453,10 @@ local diskSpaceUtilisation = sum ( sum without (device) ( max without (fstype, mountpoint, instance, pod) (( - node_filesystem_size_bytes{%(nodeExporterSelector)s, %(fsSelector)s} - node_filesystem_avail_bytes{%(nodeExporterSelector)s, %(fsSelector)s} + node_filesystem_size_bytes{%(nodeExporterSelector)s, %(fsSelector)s, %(fsMountpointSelector)s) - node_filesystem_avail_bytes{%(nodeExporterSelector)s, %(fsSelector)s, %(fsMountpointSelector)s} ) != 0) ) - / scalar(sum(max without (fstype, mountpoint) (node_filesystem_size_bytes{%(nodeExporterSelector)s, %(fsSelector)s}))) + / scalar(sum(max without (fstype, mountpoint) (node_filesystem_size_bytes{%(nodeExporterSelector)s, %(fsSelector)s, %(fsMountpointSelector)s}))) ) by (%(clusterLabel)s) ||| % $._config, legendFormat='{{%(clusterLabel)s}}' % $._config )) diff --git a/docs/node-mixin/lib/prom-mixin.libsonnet b/docs/node-mixin/lib/prom-mixin.libsonnet index 6b541ac3..6c4d9904 100644 --- a/docs/node-mixin/lib/prom-mixin.libsonnet +++ b/docs/node-mixin/lib/prom-mixin.libsonnet @@ -232,7 +232,7 @@ local table = grafana70.panel.table; .addThresholdStep(color='red', value=0.9) .addTarget(prometheus.target( ||| - max by (mountpoint) (node_filesystem_size_bytes{%(nodeExporterSelector)s, instance="$instance", %(fsSelector)s}) + max by (mountpoint) (node_filesystem_size_bytes{%(nodeExporterSelector)s, instance="$instance", %(fsSelector)s, %(fsMountpointSelector)s}) ||| % config, legendFormat='', instant=true, @@ -240,7 +240,7 @@ local table = grafana70.panel.table; )) .addTarget(prometheus.target( ||| - max by (mountpoint) (node_filesystem_avail_bytes{%(nodeExporterSelector)s, instance="$instance", %(fsSelector)s}) + max by (mountpoint) (node_filesystem_avail_bytes{%(nodeExporterSelector)s, instance="$instance", %(fsSelector)s, %(fsMountpointSelector)s}) ||| % config, legendFormat='', instant=true,