829 lines
46 KiB
YAML
829 lines
46 KiB
YAML
{{- /*
|
|
Generated from 'node-exporter' group from https://github.com/prometheus-operator/kube-prometheus.git
|
|
Do not change in-place! In order to change this file first read following link:
|
|
https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack
|
|
*/ -}}
|
|
{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }}
|
|
{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.nodeExporterAlerting }}
|
|
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-exporter" | trunc 63 | trimSuffix "-" }}
|
|
namespace: {{ template "kube-prometheus-stack.namespace" . }}
|
|
labels:
|
|
app: {{ template "kube-prometheus-stack.name" . }}
|
|
{{ include "kube-prometheus-stack.labels" . | indent 4 }}
|
|
{{- if .Values.defaultRules.labels }}
|
|
{{ toYaml .Values.defaultRules.labels | indent 4 }}
|
|
{{- end }}
|
|
{{- if .Values.defaultRules.annotations }}
|
|
annotations:
|
|
{{ toYaml .Values.defaultRules.annotations | indent 4 }}
|
|
{{- end }}
|
|
spec:
|
|
groups:
|
|
- name: node-exporter
|
|
rules:
|
|
{{- if not (.Values.defaultRules.disabled.NodeFilesystemSpaceFillingUp | default false) }}
|
|
- alert: NodeFilesystemSpaceFillingUp
|
|
annotations:
|
|
{{- if .Values.defaultRules.additionalRuleAnnotations }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
|
|
{{- end }}
|
|
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }}
|
|
{{- end }}
|
|
description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up.
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemspacefillingup
|
|
summary: Filesystem is predicted to run out of space within the next 24 hours.
|
|
expr: |-
|
|
(
|
|
node_filesystem_avail_bytes{job="node-exporter",{{ $.Values.defaultRules.node.fsSelector }},mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",{{ $.Values.defaultRules.node.fsSelector }},mountpoint!=""} * 100 < 15
|
|
and
|
|
predict_linear(node_filesystem_avail_bytes{job="node-exporter",{{ $.Values.defaultRules.node.fsSelector }},mountpoint!=""}[6h], 24*60*60) < 0
|
|
and
|
|
node_filesystem_readonly{job="node-exporter",{{ $.Values.defaultRules.node.fsSelector }},mountpoint!=""} == 0
|
|
)
|
|
for: {{ dig "NodeFilesystemSpaceFillingUp" "for" "1h" .Values.customRules }}
|
|
{{- with .Values.defaultRules.keepFiringFor }}
|
|
keep_firing_for: "{{ . }}"
|
|
{{- end }}
|
|
labels:
|
|
severity: {{ dig "NodeFilesystemSpaceFillingUp" "severity" "warning" .Values.customRules }}
|
|
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- with .Values.defaultRules.additionalRuleLabels }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- if not (.Values.defaultRules.disabled.NodeFilesystemSpaceFillingUp | default false) }}
|
|
- alert: NodeFilesystemSpaceFillingUp
|
|
annotations:
|
|
{{- if .Values.defaultRules.additionalRuleAnnotations }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
|
|
{{- end }}
|
|
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }}
|
|
{{- end }}
|
|
description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up fast.
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemspacefillingup
|
|
summary: Filesystem is predicted to run out of space within the next 4 hours.
|
|
expr: |-
|
|
(
|
|
node_filesystem_avail_bytes{job="node-exporter",{{ $.Values.defaultRules.node.fsSelector }},mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",{{ $.Values.defaultRules.node.fsSelector }},mountpoint!=""} * 100 < 10
|
|
and
|
|
predict_linear(node_filesystem_avail_bytes{job="node-exporter",{{ $.Values.defaultRules.node.fsSelector }},mountpoint!=""}[6h], 4*60*60) < 0
|
|
and
|
|
node_filesystem_readonly{job="node-exporter",{{ $.Values.defaultRules.node.fsSelector }},mountpoint!=""} == 0
|
|
)
|
|
for: {{ dig "NodeFilesystemSpaceFillingUp" "for" "1h" .Values.customRules }}
|
|
{{- with .Values.defaultRules.keepFiringFor }}
|
|
keep_firing_for: "{{ . }}"
|
|
{{- end }}
|
|
labels:
|
|
severity: {{ dig "NodeFilesystemSpaceFillingUp" "severity" "critical" .Values.customRules }}
|
|
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- with .Values.defaultRules.additionalRuleLabels }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfSpace | default false) }}
|
|
- alert: NodeFilesystemAlmostOutOfSpace
|
|
annotations:
|
|
{{- if .Values.defaultRules.additionalRuleAnnotations }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
|
|
{{- end }}
|
|
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }}
|
|
{{- end }}
|
|
description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left.
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutofspace
|
|
summary: Filesystem has less than 5% space left.
|
|
expr: |-
|
|
(
|
|
node_filesystem_avail_bytes{job="node-exporter",{{ $.Values.defaultRules.node.fsSelector }},mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",{{ $.Values.defaultRules.node.fsSelector }},mountpoint!=""} * 100 < 5
|
|
and
|
|
node_filesystem_readonly{job="node-exporter",{{ $.Values.defaultRules.node.fsSelector }},mountpoint!=""} == 0
|
|
)
|
|
for: {{ dig "NodeFilesystemAlmostOutOfSpace" "for" "30m" .Values.customRules }}
|
|
{{- with .Values.defaultRules.keepFiringFor }}
|
|
keep_firing_for: "{{ . }}"
|
|
{{- end }}
|
|
labels:
|
|
severity: {{ dig "NodeFilesystemAlmostOutOfSpace" "severity" "warning" .Values.customRules }}
|
|
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- with .Values.defaultRules.additionalRuleLabels }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfSpace | default false) }}
|
|
- alert: NodeFilesystemAlmostOutOfSpace
|
|
annotations:
|
|
{{- if .Values.defaultRules.additionalRuleAnnotations }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
|
|
{{- end }}
|
|
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }}
|
|
{{- end }}
|
|
description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left.
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutofspace
|
|
summary: Filesystem has less than 3% space left.
|
|
expr: |-
|
|
(
|
|
node_filesystem_avail_bytes{job="node-exporter",{{ $.Values.defaultRules.node.fsSelector }},mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",{{ $.Values.defaultRules.node.fsSelector }},mountpoint!=""} * 100 < 3
|
|
and
|
|
node_filesystem_readonly{job="node-exporter",{{ $.Values.defaultRules.node.fsSelector }},mountpoint!=""} == 0
|
|
)
|
|
for: {{ dig "NodeFilesystemAlmostOutOfSpace" "for" "30m" .Values.customRules }}
|
|
{{- with .Values.defaultRules.keepFiringFor }}
|
|
keep_firing_for: "{{ . }}"
|
|
{{- end }}
|
|
labels:
|
|
severity: {{ dig "NodeFilesystemAlmostOutOfSpace" "severity" "critical" .Values.customRules }}
|
|
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- with .Values.defaultRules.additionalRuleLabels }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- if not (.Values.defaultRules.disabled.NodeFilesystemFilesFillingUp | default false) }}
|
|
- alert: NodeFilesystemFilesFillingUp
|
|
annotations:
|
|
{{- if .Values.defaultRules.additionalRuleAnnotations }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
|
|
{{- end }}
|
|
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }}
|
|
{{- end }}
|
|
description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up.
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemfilesfillingup
|
|
summary: Filesystem is predicted to run out of inodes within the next 24 hours.
|
|
expr: |-
|
|
(
|
|
node_filesystem_files_free{job="node-exporter",{{ $.Values.defaultRules.node.fsSelector }},mountpoint!=""} / node_filesystem_files{job="node-exporter",{{ $.Values.defaultRules.node.fsSelector }},mountpoint!=""} * 100 < 40
|
|
and
|
|
predict_linear(node_filesystem_files_free{job="node-exporter",{{ $.Values.defaultRules.node.fsSelector }},mountpoint!=""}[6h], 24*60*60) < 0
|
|
and
|
|
node_filesystem_readonly{job="node-exporter",{{ $.Values.defaultRules.node.fsSelector }},mountpoint!=""} == 0
|
|
)
|
|
for: {{ dig "NodeFilesystemFilesFillingUp" "for" "1h" .Values.customRules }}
|
|
{{- with .Values.defaultRules.keepFiringFor }}
|
|
keep_firing_for: "{{ . }}"
|
|
{{- end }}
|
|
labels:
|
|
severity: {{ dig "NodeFilesystemFilesFillingUp" "severity" "warning" .Values.customRules }}
|
|
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- with .Values.defaultRules.additionalRuleLabels }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- if not (.Values.defaultRules.disabled.NodeFilesystemFilesFillingUp | default false) }}
|
|
- alert: NodeFilesystemFilesFillingUp
|
|
annotations:
|
|
{{- if .Values.defaultRules.additionalRuleAnnotations }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
|
|
{{- end }}
|
|
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }}
|
|
{{- end }}
|
|
description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up fast.
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemfilesfillingup
|
|
summary: Filesystem is predicted to run out of inodes within the next 4 hours.
|
|
expr: |-
|
|
(
|
|
node_filesystem_files_free{job="node-exporter",{{ $.Values.defaultRules.node.fsSelector }},mountpoint!=""} / node_filesystem_files{job="node-exporter",{{ $.Values.defaultRules.node.fsSelector }},mountpoint!=""} * 100 < 20
|
|
and
|
|
predict_linear(node_filesystem_files_free{job="node-exporter",{{ $.Values.defaultRules.node.fsSelector }},mountpoint!=""}[6h], 4*60*60) < 0
|
|
and
|
|
node_filesystem_readonly{job="node-exporter",{{ $.Values.defaultRules.node.fsSelector }},mountpoint!=""} == 0
|
|
)
|
|
for: {{ dig "NodeFilesystemFilesFillingUp" "for" "1h" .Values.customRules }}
|
|
{{- with .Values.defaultRules.keepFiringFor }}
|
|
keep_firing_for: "{{ . }}"
|
|
{{- end }}
|
|
labels:
|
|
severity: {{ dig "NodeFilesystemFilesFillingUp" "severity" "critical" .Values.customRules }}
|
|
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- with .Values.defaultRules.additionalRuleLabels }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfFiles | default false) }}
|
|
- alert: NodeFilesystemAlmostOutOfFiles
|
|
annotations:
|
|
{{- if .Values.defaultRules.additionalRuleAnnotations }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
|
|
{{- end }}
|
|
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }}
|
|
{{- end }}
|
|
description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left.
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutoffiles
|
|
summary: Filesystem has less than 5% inodes left.
|
|
expr: |-
|
|
(
|
|
node_filesystem_files_free{job="node-exporter",{{ $.Values.defaultRules.node.fsSelector }},mountpoint!=""} / node_filesystem_files{job="node-exporter",{{ $.Values.defaultRules.node.fsSelector }},mountpoint!=""} * 100 < 5
|
|
and
|
|
node_filesystem_readonly{job="node-exporter",{{ $.Values.defaultRules.node.fsSelector }},mountpoint!=""} == 0
|
|
)
|
|
for: {{ dig "NodeFilesystemAlmostOutOfFiles" "for" "1h" .Values.customRules }}
|
|
{{- with .Values.defaultRules.keepFiringFor }}
|
|
keep_firing_for: "{{ . }}"
|
|
{{- end }}
|
|
labels:
|
|
severity: {{ dig "NodeFilesystemAlmostOutOfFiles" "severity" "warning" .Values.customRules }}
|
|
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- with .Values.defaultRules.additionalRuleLabels }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfFiles | default false) }}
|
|
- alert: NodeFilesystemAlmostOutOfFiles
|
|
annotations:
|
|
{{- if .Values.defaultRules.additionalRuleAnnotations }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
|
|
{{- end }}
|
|
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }}
|
|
{{- end }}
|
|
description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left.
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutoffiles
|
|
summary: Filesystem has less than 3% inodes left.
|
|
expr: |-
|
|
(
|
|
node_filesystem_files_free{job="node-exporter",{{ $.Values.defaultRules.node.fsSelector }},mountpoint!=""} / node_filesystem_files{job="node-exporter",{{ $.Values.defaultRules.node.fsSelector }},mountpoint!=""} * 100 < 3
|
|
and
|
|
node_filesystem_readonly{job="node-exporter",{{ $.Values.defaultRules.node.fsSelector }},mountpoint!=""} == 0
|
|
)
|
|
for: {{ dig "NodeFilesystemAlmostOutOfFiles" "for" "1h" .Values.customRules }}
|
|
{{- with .Values.defaultRules.keepFiringFor }}
|
|
keep_firing_for: "{{ . }}"
|
|
{{- end }}
|
|
labels:
|
|
severity: {{ dig "NodeFilesystemAlmostOutOfFiles" "severity" "critical" .Values.customRules }}
|
|
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- with .Values.defaultRules.additionalRuleLabels }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- if not (.Values.defaultRules.disabled.NodeNetworkReceiveErrs | default false) }}
|
|
- alert: NodeNetworkReceiveErrs
|
|
annotations:
|
|
{{- if .Values.defaultRules.additionalRuleAnnotations }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
|
|
{{- end }}
|
|
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }}
|
|
{{- end }}
|
|
description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} receive errors in the last two minutes.'
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodenetworkreceiveerrs
|
|
summary: Network interface is reporting many receive errors.
|
|
expr: rate(node_network_receive_errs_total{job="node-exporter"}[2m]) / rate(node_network_receive_packets_total{job="node-exporter"}[2m]) > 0.01
|
|
for: {{ dig "NodeNetworkReceiveErrs" "for" "1h" .Values.customRules }}
|
|
{{- with .Values.defaultRules.keepFiringFor }}
|
|
keep_firing_for: "{{ . }}"
|
|
{{- end }}
|
|
labels:
|
|
severity: {{ dig "NodeNetworkReceiveErrs" "severity" "warning" .Values.customRules }}
|
|
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- with .Values.defaultRules.additionalRuleLabels }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- if not (.Values.defaultRules.disabled.NodeNetworkTransmitErrs | default false) }}
|
|
- alert: NodeNetworkTransmitErrs
|
|
annotations:
|
|
{{- if .Values.defaultRules.additionalRuleAnnotations }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
|
|
{{- end }}
|
|
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }}
|
|
{{- end }}
|
|
description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} transmit errors in the last two minutes.'
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodenetworktransmiterrs
|
|
summary: Network interface is reporting many transmit errors.
|
|
expr: rate(node_network_transmit_errs_total{job="node-exporter"}[2m]) / rate(node_network_transmit_packets_total{job="node-exporter"}[2m]) > 0.01
|
|
for: {{ dig "NodeNetworkTransmitErrs" "for" "1h" .Values.customRules }}
|
|
{{- with .Values.defaultRules.keepFiringFor }}
|
|
keep_firing_for: "{{ . }}"
|
|
{{- end }}
|
|
labels:
|
|
severity: {{ dig "NodeNetworkTransmitErrs" "severity" "warning" .Values.customRules }}
|
|
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- with .Values.defaultRules.additionalRuleLabels }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- if not (.Values.defaultRules.disabled.NodeHighNumberConntrackEntriesUsed | default false) }}
|
|
- alert: NodeHighNumberConntrackEntriesUsed
|
|
annotations:
|
|
{{- if .Values.defaultRules.additionalRuleAnnotations }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
|
|
{{- end }}
|
|
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }}
|
|
{{- end }}
|
|
description: '{{`{{`}} $labels.instance {{`}}`}} {{`{{`}} $value | humanizePercentage {{`}}`}} of conntrack entries are used.'
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodehighnumberconntrackentriesused
|
|
summary: Number of conntrack are getting close to the limit.
|
|
expr: (node_nf_conntrack_entries{job="node-exporter"} / node_nf_conntrack_entries_limit) > 0.75
|
|
labels:
|
|
severity: {{ dig "NodeHighNumberConntrackEntriesUsed" "severity" "warning" .Values.customRules }}
|
|
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- with .Values.defaultRules.additionalRuleLabels }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- if not (.Values.defaultRules.disabled.NodeTextFileCollectorScrapeError | default false) }}
|
|
- alert: NodeTextFileCollectorScrapeError
|
|
annotations:
|
|
{{- if .Values.defaultRules.additionalRuleAnnotations }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
|
|
{{- end }}
|
|
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }}
|
|
{{- end }}
|
|
description: Node Exporter text file collector on {{`{{`}} $labels.instance {{`}}`}} failed to scrape.
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodetextfilecollectorscrapeerror
|
|
summary: Node Exporter text file collector failed to scrape.
|
|
expr: node_textfile_scrape_error{job="node-exporter"} == 1
|
|
labels:
|
|
severity: {{ dig "NodeTextFileCollectorScrapeError" "severity" "warning" .Values.customRules }}
|
|
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- with .Values.defaultRules.additionalRuleLabels }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- if not (.Values.defaultRules.disabled.NodeClockSkewDetected | default false) }}
|
|
- alert: NodeClockSkewDetected
|
|
annotations:
|
|
{{- if .Values.defaultRules.additionalRuleAnnotations }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
|
|
{{- end }}
|
|
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }}
|
|
{{- end }}
|
|
description: Clock at {{`{{`}} $labels.instance {{`}}`}} is out of sync by more than 0.05s. Ensure NTP is configured correctly on this host.
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodeclockskewdetected
|
|
summary: Clock skew detected.
|
|
expr: |-
|
|
(
|
|
node_timex_offset_seconds{job="node-exporter"} > 0.05
|
|
and
|
|
deriv(node_timex_offset_seconds{job="node-exporter"}[5m]) >= 0
|
|
)
|
|
or
|
|
(
|
|
node_timex_offset_seconds{job="node-exporter"} < -0.05
|
|
and
|
|
deriv(node_timex_offset_seconds{job="node-exporter"}[5m]) <= 0
|
|
)
|
|
for: {{ dig "NodeClockSkewDetected" "for" "10m" .Values.customRules }}
|
|
{{- with .Values.defaultRules.keepFiringFor }}
|
|
keep_firing_for: "{{ . }}"
|
|
{{- end }}
|
|
labels:
|
|
severity: {{ dig "NodeClockSkewDetected" "severity" "warning" .Values.customRules }}
|
|
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- with .Values.defaultRules.additionalRuleLabels }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- if not (.Values.defaultRules.disabled.NodeClockNotSynchronising | default false) }}
|
|
- alert: NodeClockNotSynchronising
|
|
annotations:
|
|
{{- if .Values.defaultRules.additionalRuleAnnotations }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
|
|
{{- end }}
|
|
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }}
|
|
{{- end }}
|
|
description: Clock at {{`{{`}} $labels.instance {{`}}`}} is not synchronising. Ensure NTP is configured on this host.
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodeclocknotsynchronising
|
|
summary: Clock not synchronising.
|
|
expr: |-
|
|
min_over_time(node_timex_sync_status{job="node-exporter"}[5m]) == 0
|
|
and
|
|
node_timex_maxerror_seconds{job="node-exporter"} >= 16
|
|
for: {{ dig "NodeClockNotSynchronising" "for" "10m" .Values.customRules }}
|
|
{{- with .Values.defaultRules.keepFiringFor }}
|
|
keep_firing_for: "{{ . }}"
|
|
{{- end }}
|
|
labels:
|
|
severity: {{ dig "NodeClockNotSynchronising" "severity" "warning" .Values.customRules }}
|
|
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- with .Values.defaultRules.additionalRuleLabels }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- if not (.Values.defaultRules.disabled.NodeRAIDDegraded | default false) }}
|
|
- alert: NodeRAIDDegraded
|
|
annotations:
|
|
{{- if .Values.defaultRules.additionalRuleAnnotations }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
|
|
{{- end }}
|
|
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }}
|
|
{{- end }}
|
|
description: RAID array '{{`{{`}} $labels.device {{`}}`}}' at {{`{{`}} $labels.instance {{`}}`}} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically.
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/noderaiddegraded
|
|
summary: RAID Array is degraded.
|
|
expr: node_md_disks_required{job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"} - ignoring (state) (node_md_disks{state="active",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}) > 0
|
|
for: {{ dig "NodeRAIDDegraded" "for" "15m" .Values.customRules }}
|
|
{{- with .Values.defaultRules.keepFiringFor }}
|
|
keep_firing_for: "{{ . }}"
|
|
{{- end }}
|
|
labels:
|
|
severity: {{ dig "NodeRAIDDegraded" "severity" "critical" .Values.customRules }}
|
|
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- with .Values.defaultRules.additionalRuleLabels }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- if not (.Values.defaultRules.disabled.NodeRAIDDiskFailure | default false) }}
|
|
- alert: NodeRAIDDiskFailure
|
|
annotations:
|
|
{{- if .Values.defaultRules.additionalRuleAnnotations }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
|
|
{{- end }}
|
|
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }}
|
|
{{- end }}
|
|
description: At least one device in RAID array at {{`{{`}} $labels.instance {{`}}`}} failed. Array '{{`{{`}} $labels.device {{`}}`}}' needs attention and possibly a disk swap.
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/noderaiddiskfailure
|
|
summary: Failed device in RAID array.
|
|
expr: node_md_disks{state="failed",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"} > 0
|
|
labels:
|
|
severity: {{ dig "NodeRAIDDiskFailure" "severity" "warning" .Values.customRules }}
|
|
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- with .Values.defaultRules.additionalRuleLabels }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- if not (.Values.defaultRules.disabled.NodeFileDescriptorLimit | default false) }}
|
|
- alert: NodeFileDescriptorLimit
|
|
annotations:
|
|
{{- if .Values.defaultRules.additionalRuleAnnotations }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
|
|
{{- end }}
|
|
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }}
|
|
{{- end }}
|
|
description: File descriptors limit at {{`{{`}} $labels.instance {{`}}`}} is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%.
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefiledescriptorlimit
|
|
summary: Kernel is predicted to exhaust file descriptors limit soon.
|
|
expr: |-
|
|
(
|
|
node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 70
|
|
)
|
|
for: {{ dig "NodeFileDescriptorLimit" "for" "15m" .Values.customRules }}
|
|
{{- with .Values.defaultRules.keepFiringFor }}
|
|
keep_firing_for: "{{ . }}"
|
|
{{- end }}
|
|
labels:
|
|
severity: {{ dig "NodeFileDescriptorLimit" "severity" "warning" .Values.customRules }}
|
|
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- with .Values.defaultRules.additionalRuleLabels }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- if not (.Values.defaultRules.disabled.NodeFileDescriptorLimit | default false) }}
|
|
- alert: NodeFileDescriptorLimit
|
|
annotations:
|
|
{{- if .Values.defaultRules.additionalRuleAnnotations }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
|
|
{{- end }}
|
|
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }}
|
|
{{- end }}
|
|
description: File descriptors limit at {{`{{`}} $labels.instance {{`}}`}} is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%.
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefiledescriptorlimit
|
|
summary: Kernel is predicted to exhaust file descriptors limit soon.
|
|
expr: |-
|
|
(
|
|
node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 90
|
|
)
|
|
for: {{ dig "NodeFileDescriptorLimit" "for" "15m" .Values.customRules }}
|
|
{{- with .Values.defaultRules.keepFiringFor }}
|
|
keep_firing_for: "{{ . }}"
|
|
{{- end }}
|
|
labels:
|
|
severity: {{ dig "NodeFileDescriptorLimit" "severity" "critical" .Values.customRules }}
|
|
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- with .Values.defaultRules.additionalRuleLabels }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- if not (.Values.defaultRules.disabled.NodeCPUHighUsage | default false) }}
|
|
- alert: NodeCPUHighUsage
|
|
annotations:
|
|
{{- if .Values.defaultRules.additionalRuleAnnotations }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
|
|
{{- end }}
|
|
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }}
|
|
{{- end }}
|
|
description: 'CPU usage at {{`{{`}} $labels.instance {{`}}`}} has been above 90% for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%.
|
|
|
|
'
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodecpuhighusage
|
|
summary: High CPU usage.
|
|
expr: sum without(mode) (avg without (cpu) (rate(node_cpu_seconds_total{job="node-exporter", mode!~"idle|iowait"}[2m]))) * 100 > 90
|
|
for: {{ dig "NodeCPUHighUsage" "for" "15m" .Values.customRules }}
|
|
{{- with .Values.defaultRules.keepFiringFor }}
|
|
keep_firing_for: "{{ . }}"
|
|
{{- end }}
|
|
labels:
|
|
severity: {{ dig "NodeCPUHighUsage" "severity" "info" .Values.customRules }}
|
|
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- with .Values.defaultRules.additionalRuleLabels }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- if not (.Values.defaultRules.disabled.NodeSystemSaturation | default false) }}
|
|
- alert: NodeSystemSaturation
|
|
annotations:
|
|
{{- if .Values.defaultRules.additionalRuleAnnotations }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
|
|
{{- end }}
|
|
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }}
|
|
{{- end }}
|
|
description: 'System load per core at {{`{{`}} $labels.instance {{`}}`}} has been above 2 for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}.
|
|
|
|
This might indicate this instance resources saturation and can cause it becoming unresponsive.
|
|
|
|
'
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodesystemsaturation
|
|
summary: System saturated, load per core is very high.
|
|
expr: |-
|
|
node_load1{job="node-exporter"}
|
|
/ count without (cpu, mode) (node_cpu_seconds_total{job="node-exporter", mode="idle"}) > 2
|
|
for: {{ dig "NodeSystemSaturation" "for" "15m" .Values.customRules }}
|
|
{{- with .Values.defaultRules.keepFiringFor }}
|
|
keep_firing_for: "{{ . }}"
|
|
{{- end }}
|
|
labels:
|
|
severity: {{ dig "NodeSystemSaturation" "severity" "warning" .Values.customRules }}
|
|
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- with .Values.defaultRules.additionalRuleLabels }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- if not (.Values.defaultRules.disabled.NodeMemoryMajorPagesFaults | default false) }}
|
|
- alert: NodeMemoryMajorPagesFaults
|
|
annotations:
|
|
{{- if .Values.defaultRules.additionalRuleAnnotations }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
|
|
{{- end }}
|
|
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }}
|
|
{{- end }}
|
|
description: 'Memory major pages are occurring at very high rate at {{`{{`}} $labels.instance {{`}}`}}, 500 major page faults per second for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}.
|
|
|
|
Please check that there is enough memory available at this instance.
|
|
|
|
'
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodememorymajorpagesfaults
|
|
summary: Memory major page faults are occurring at very high rate.
|
|
expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[5m]) > 500
|
|
for: {{ dig "NodeMemoryMajorPagesFaults" "for" "15m" .Values.customRules }}
|
|
{{- with .Values.defaultRules.keepFiringFor }}
|
|
keep_firing_for: "{{ . }}"
|
|
{{- end }}
|
|
labels:
|
|
severity: {{ dig "NodeMemoryMajorPagesFaults" "severity" "warning" .Values.customRules }}
|
|
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- with .Values.defaultRules.additionalRuleLabels }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- if not (.Values.defaultRules.disabled.NodeMemoryHighUtilization | default false) }}
|
|
- alert: NodeMemoryHighUtilization
|
|
annotations:
|
|
{{- if .Values.defaultRules.additionalRuleAnnotations }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
|
|
{{- end }}
|
|
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }}
|
|
{{- end }}
|
|
description: 'Memory is filling up at {{`{{`}} $labels.instance {{`}}`}}, has been above 90% for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%.
|
|
|
|
'
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodememoryhighutilization
|
|
summary: Host is running out of memory.
|
|
expr: 100 - (node_memory_MemAvailable_bytes{job="node-exporter"} / node_memory_MemTotal_bytes{job="node-exporter"} * 100) > 90
|
|
for: {{ dig "NodeMemoryHighUtilization" "for" "15m" .Values.customRules }}
|
|
{{- with .Values.defaultRules.keepFiringFor }}
|
|
keep_firing_for: "{{ . }}"
|
|
{{- end }}
|
|
labels:
|
|
severity: {{ dig "NodeMemoryHighUtilization" "severity" "warning" .Values.customRules }}
|
|
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- with .Values.defaultRules.additionalRuleLabels }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- if not (.Values.defaultRules.disabled.NodeDiskIOSaturation | default false) }}
|
|
- alert: NodeDiskIOSaturation
|
|
annotations:
|
|
{{- if .Values.defaultRules.additionalRuleAnnotations }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
|
|
{{- end }}
|
|
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }}
|
|
{{- end }}
|
|
description: 'Disk IO queue (aqu-sq) is high on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}}, has been above 10 for the last 30 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}.
|
|
|
|
This symptom might indicate disk saturation.
|
|
|
|
'
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodediskiosaturation
|
|
summary: Disk IO queue is high.
|
|
expr: rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m]) > 10
|
|
for: {{ dig "NodeDiskIOSaturation" "for" "30m" .Values.customRules }}
|
|
{{- with .Values.defaultRules.keepFiringFor }}
|
|
keep_firing_for: "{{ . }}"
|
|
{{- end }}
|
|
labels:
|
|
severity: {{ dig "NodeDiskIOSaturation" "severity" "warning" .Values.customRules }}
|
|
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- with .Values.defaultRules.additionalRuleLabels }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- if not (.Values.defaultRules.disabled.NodeSystemdServiceFailed | default false) }}
|
|
- alert: NodeSystemdServiceFailed
|
|
annotations:
|
|
{{- if .Values.defaultRules.additionalRuleAnnotations }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
|
|
{{- end }}
|
|
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }}
|
|
{{- end }}
|
|
description: Systemd service {{`{{`}} $labels.name {{`}}`}} has entered failed state at {{`{{`}} $labels.instance {{`}}`}}
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodesystemdservicefailed
|
|
summary: Systemd service has entered failed state.
|
|
expr: node_systemd_unit_state{job="node-exporter", state="failed"} == 1
|
|
for: {{ dig "NodeSystemdServiceFailed" "for" "5m" .Values.customRules }}
|
|
{{- with .Values.defaultRules.keepFiringFor }}
|
|
keep_firing_for: "{{ . }}"
|
|
{{- end }}
|
|
labels:
|
|
severity: {{ dig "NodeSystemdServiceFailed" "severity" "warning" .Values.customRules }}
|
|
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- with .Values.defaultRules.additionalRuleLabels }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- if not (.Values.defaultRules.disabled.NodeSystemdServiceCrashlooping | default false) }}
|
|
- alert: NodeSystemdServiceCrashlooping
|
|
annotations:
|
|
{{- if .Values.defaultRules.additionalRuleAnnotations }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
|
|
{{- end }}
|
|
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }}
|
|
{{- end }}
|
|
description: Systemd service {{`{{`}} $labels.name {{`}}`}} has being restarted too many times at {{`{{`}} $labels.instance {{`}}`}} for the last 15 minutes. Please check if service is crash looping.
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodesystemdservicecrashlooping
|
|
summary: Systemd service keeps restaring, possibly crash looping.
|
|
expr: increase(node_systemd_service_restart_total{job="node-exporter"}[5m]) > 2
|
|
for: {{ dig "NodeSystemdServiceCrashlooping" "for" "15m" .Values.customRules }}
|
|
{{- with .Values.defaultRules.keepFiringFor }}
|
|
keep_firing_for: "{{ . }}"
|
|
{{- end }}
|
|
labels:
|
|
severity: {{ dig "NodeSystemdServiceCrashlooping" "severity" "warning" .Values.customRules }}
|
|
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- with .Values.defaultRules.additionalRuleLabels }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- if not (.Values.defaultRules.disabled.NodeBondingDegraded | default false) }}
|
|
- alert: NodeBondingDegraded
|
|
annotations:
|
|
{{- if .Values.defaultRules.additionalRuleAnnotations }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
|
|
{{- end }}
|
|
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }}
|
|
{{- end }}
|
|
description: Bonding interface {{`{{`}} $labels.master {{`}}`}} on {{`{{`}} $labels.instance {{`}}`}} is in degraded state due to one or more slave failures.
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodebondingdegraded
|
|
summary: Bonding interface is degraded
|
|
expr: (node_bonding_slaves - node_bonding_active) != 0
|
|
for: {{ dig "NodeBondingDegraded" "for" "5m" .Values.customRules }}
|
|
{{- with .Values.defaultRules.keepFiringFor }}
|
|
keep_firing_for: "{{ . }}"
|
|
{{- end }}
|
|
labels:
|
|
severity: {{ dig "NodeBondingDegraded" "severity" "warning" .Values.customRules }}
|
|
{{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- with .Values.defaultRules.additionalRuleLabels }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }}
|
|
{{- toYaml . | nindent 8 }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }} |