Node_Exporter/node-mixin/alerts/alerts.libsonnet

{
  prometheusAlerts+:: {
    groups+: [
      {
        name: 'node',
        rules: [
          {
            alert: 'NodeFilesystemSpaceFillingUp',
            expr: |||
                predict_linear(node_filesystem_avail{%(nodeExporterSelector)s,%(fsSelectors)s}[6h], 24*60*60) < 0
              and
                node_filesystem_avail{%(nodeExporterSelector)s,%(fsSelectors)s} / node_filesystem_size{%(nodeExporterSelector)s,%(fsSelectors)s} < 0.4
              and
                node_filesystem_readonly{%(nodeExporterSelector)s,%(fsSelectors)s} == 0
            ||| % $._config,
            'for': '1h',
            labels: {
              severity: 'warning',
            },
            annotations: {
              message: 'Filesystem on {{ $labels.device }} at {{ $labels.instance }} is predicted to run out of space within the next 24 hours.',
            },
          },
          {
            alert: 'NodeFilesystemSpaceFillingUp',
            expr: |||
                predict_linear(node_filesystem_avail{%(nodeExporterSelector)s,%(fsSelectors)s}[6h], 4*60*60) < 0
              and
                node_filesystem_avail{%(nodeExporterSelector)s,%(fsSelectors)s} / node_filesystem_size{%(nodeExporterSelector)s,%(fsSelectors)s} < 0.2
              and
                node_filesystem_readonly{%(nodeExporterSelector)s,%(fsSelectors)s} == 0
            ||| % $._config,
            'for': '1h',
            labels: {
              severity: 'critical',
            },
            annotations: {
              message: 'Filesystem on {{ $labels.device }} at {{ $labels.instance }} is predicted to run out of space within the next 4 hours.',
            },
          },
          {
            alert: 'NodeFilesystemOutOfSpace',
            expr: |||
                node_filesystem_avail{%(nodeExporterSelector)s,%(fsSelectors)s} / node_filesystem_size{%(nodeExporterSelector)s,%(fsSelectors)s} * 100 < 5
              and
                node_filesystem_readonly{%(nodeExporterSelector)s,%(fsSelectors)s} == 0
            ||| % $._config,
            'for': '1h',
            labels: {
              severity: 'warning',
            },
            annotations: {
              message: 'Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ $value }}% available space left.',
            },
          },
          {
            alert: 'NodeFilesystemOutOfSpace',
            expr: |||
                node_filesystem_avail{%(nodeExporterSelector)s,%(fsSelectors)s} / node_filesystem_size{%(nodeExporterSelector)s,%(fsSelectors)s} * 100 < 3
              and
                node_filesystem_readonly{%(nodeExporterSelector)s,%(fsSelectors)s} == 0
            ||| % $._config,
            'for': '1h',
            labels: {
              severity: 'critical',
            },
            annotations: {
              message: 'Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ $value }}% available space left.',
            },
          },
          {
            alert: 'NodeFilesystemFilesFillingUp',
            expr: |||
                predict_linear(node_filesystem_files_free{%(nodeExporterSelector)s,%(fsSelectors)s}[6h], 24*60*60) < 0
              and
                node_filesystem_files_free{%(nodeExporterSelector)s,%(fsSelectors)s} / node_filesystem_files{%(nodeExporterSelector)s,%(fsSelectors)s} < 0.4
              and
                node_filesystem_readonly{%(nodeExporterSelector)s,%(fsSelectors)s} == 0
            ||| % $._config,
            'for': '1h',
            labels: {
              severity: 'warning',
            },
            annotations: {
              message: 'Filesystem on {{ $labels.device }} at {{ $labels.instance }} is predicted to run out of files within the next 24 hours.',
            },
          },
          {
            alert: 'NodeFilesystemFilesFillingUp',
            expr: |||
                predict_linear(node_filesystem_files_free{%(nodeExporterSelector)s,%(fsSelectors)s}[6h], 4*60*60) < 0
              and
                node_filesystem_files_free{%(nodeExporterSelector)s,%(fsSelectors)s} / node_filesystem_files{%(nodeExporterSelector)s,%(fsSelectors)s} < 0.2
              and
                node_filesystem_readonly{%(nodeExporterSelector)s,%(fsSelectors)s} == 0
            ||| % $._config,
            'for': '1h',
            labels: {
              severity: 'warning',
            },
            annotations: {
              message: 'Filesystem on {{ $labels.device }} at {{ $labels.instance }} is predicted to run out of files within the next 4 hours.',
            },
          },
          {
            alert: 'NodeFilesystemOutOfFiles',
            expr: |||
                node_filesystem_files_free{%(nodeExporterSelector)s,%(fsSelectors)s} / node_filesystem_files{%(nodeExporterSelector)s,%(fsSelectors)s} * 100 < 5
              and
                node_filesystem_readonly{%(nodeExporterSelector)s,%(fsSelectors)s} == 0
            ||| % $._config,
            'for': '1h',
            labels: {
              severity: 'warning',
            },
            annotations: {
              message: 'Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ $value }}% available inodes left.',
            },
          },
          {
            alert: 'NodeFilesystemOutOfSpace',
            expr: |||
                node_filesystem_files_free{%(nodeExporterSelector)s,%(fsSelectors)s} / node_filesystem_files{%(nodeExporterSelector)s,%(fsSelectors)s} * 100 < 3
              and
                node_filesystem_readonly{%(nodeExporterSelector)s,%(fsSelectors)s} == 0
            ||| % $._config,
            'for': '1h',
            labels: {
              severity: 'critical',
            },
            annotations: {
              message: 'Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ $value }}% available space left.',
            },
          },
          {
            alert: 'NodeNetworkReceiveErrs',
            expr: |||
              increase(node_network_receive_errs[2m]) > 10
            ||| % $._config,
            'for': '1h',
            labels: {
              severity: 'critical',
            },
            annotations: {
              message: '{{ $labels.instance }} interface {{ $labels.device }} shows errors while receiving packets ({{ $value }} errors in two minutes).',
            },
          },
          {
            alert: 'NodeNetworkTransmitErrs',
            expr: |||
              increase(node_network_transmit_errs[2m]) > 10
            ||| % $._config,
            'for': '1h',
            labels: {
              severity: 'critical',
            },
            annotations: {
              message: '{{ $labels.instance }} interface {{ $labels.device }} shows errors while transmitting packets ({{ $value }} errors in two minutes).',
            },
          },
        ],
      },
    ],
  },
}