{{- /*
Generated from 'etcd' group from https://github.com/etcd-io/etcd.git
Do not change in-place! In order to change this file first read following link:
https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack
*/ -}}
{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }}
{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeEtcd.enabled .Values.defaultRules.rules.etcd }}
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "etcd" | trunc 63 | trimSuffix "-" }}
  namespace: {{ template "kube-prometheus-stack.namespace" . }}
  labels:
    app: {{ template "kube-prometheus-stack.name" . }}
{{ include "kube-prometheus-stack.labels" . | indent 4 }}
{{- if .Values.defaultRules.labels }}
{{ toYaml .Values.defaultRules.labels | indent 4 }}
{{- end }}
{{- if .Values.defaultRules.annotations }}
  annotations:
{{ toYaml .Values.defaultRules.annotations | indent 4 }}
{{- end }}
spec:
  groups:
  - name: etcd
    rules:
{{- if not (.Values.defaultRules.disabled.etcdMembersDown | default false) }}
    - alert: etcdMembersDown
      annotations:
{{- if .Values.defaultRules.additionalRuleAnnotations }}
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
{{- end }}
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }}
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }}
{{- end }}
        description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": members are down ({{`{{`}} $value {{`}}`}}).'
        summary: etcd cluster members are down.
      expr: |-
        max without (endpoint) (
          sum without (instance, pod) (up{job=~".*etcd.*"} == bool 0)
        or
          count without (To) (
            sum without (instance, pod) (rate(etcd_network_peer_sent_failures_total{job=~".*etcd.*"}[120s])) > 0.01
          )
        )
        > 0
      for: {{ dig "etcdMembersDown" "for" "20m" .Values.customRules }}
      {{- with .Values.defaultRules.keepFiringFor }}
      keep_firing_for: "{{ . }}"
      {{- end }}
      labels:
        severity: {{ dig "etcdMembersDown" "severity" "warning" .Values.customRules }}
      {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }}
        {{- with .Values.defaultRules.additionalRuleLabels }}
          {{- toYaml . | nindent 8 }}
        {{- end }}
        {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }}
          {{- toYaml . | nindent 8 }}
        {{- end }}
      {{- end }}
{{- end }}
{{- if not (.Values.defaultRules.disabled.etcdInsufficientMembers | default false) }}
    - alert: etcdInsufficientMembers
      annotations:
{{- if .Values.defaultRules.additionalRuleAnnotations }}
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
{{- end }}
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }}
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }}
{{- end }}
        description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": insufficient members ({{`{{`}} $value {{`}}`}}).'
        summary: etcd cluster has insufficient number of members.
      expr: sum(up{job=~".*etcd.*"} == bool 1) without (instance, pod) < ((count(up{job=~".*etcd.*"}) without (instance, pod) + 1) / 2)
      for: {{ dig "etcdInsufficientMembers" "for" "3m" .Values.customRules }}
      {{- with .Values.defaultRules.keepFiringFor }}
      keep_firing_for: "{{ . }}"
      {{- end }}
      labels:
        severity: {{ dig "etcdInsufficientMembers" "severity" "critical" .Values.customRules }}
      {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }}
        {{- with .Values.defaultRules.additionalRuleLabels }}
          {{- toYaml . | nindent 8 }}
        {{- end }}
        {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }}
          {{- toYaml . | nindent 8 }}
        {{- end }}
      {{- end }}
{{- end }}
{{- if not (.Values.defaultRules.disabled.etcdNoLeader | default false) }}
    - alert: etcdNoLeader
      annotations:
{{- if .Values.defaultRules.additionalRuleAnnotations }}
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
{{- end }}
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }}
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }}
{{- end }}
        description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": member {{`{{`}} $labels.instance {{`}}`}} has no leader.'
        summary: etcd cluster has no leader.
      expr: etcd_server_has_leader{job=~".*etcd.*"} == 0
      for: {{ dig "etcdNoLeader" "for" "1m" .Values.customRules }}
      {{- with .Values.defaultRules.keepFiringFor }}
      keep_firing_for: "{{ . }}"
      {{- end }}
      labels:
        severity: {{ dig "etcdNoLeader" "severity" "critical" .Values.customRules }}
      {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }}
        {{- with .Values.defaultRules.additionalRuleLabels }}
          {{- toYaml . | nindent 8 }}
        {{- end }}
        {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }}
          {{- toYaml . | nindent 8 }}
        {{- end }}
      {{- end }}
{{- end }}
{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfLeaderChanges | default false) }}
    - alert: etcdHighNumberOfLeaderChanges
      annotations:
{{- if .Values.defaultRules.additionalRuleAnnotations }}
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
{{- end }}
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }}
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }}
{{- end }}
        description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} leader changes within the last 15 minutes. Frequent elections may be a sign of insufficient resources, high network latency, or disruptions by other components and should be investigated.'
        summary: etcd cluster has high number of leader changes.
      expr: increase((max without (instance, pod) (etcd_server_leader_changes_seen_total{job=~".*etcd.*"}) or 0*absent(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}))[15m:1m]) >= 4
      for: {{ dig "etcdHighNumberOfLeaderChanges" "for" "5m" .Values.customRules }}
      {{- with .Values.defaultRules.keepFiringFor }}
      keep_firing_for: "{{ . }}"
      {{- end }}
      labels:
        severity: {{ dig "etcdHighNumberOfLeaderChanges" "severity" "warning" .Values.customRules }}
      {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }}
        {{- with .Values.defaultRules.additionalRuleLabels }}
          {{- toYaml . | nindent 8 }}
        {{- end }}
        {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }}
          {{- toYaml . | nindent 8 }}
        {{- end }}
      {{- end }}
{{- end }}
{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfFailedGRPCRequests | default false) }}
    - alert: etcdHighNumberOfFailedGRPCRequests
      annotations:
{{- if .Values.defaultRules.additionalRuleAnnotations }}
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
{{- end }}
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }}
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }}
{{- end }}
        description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.grpc_method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}.'
        summary: etcd cluster has high number of failed grpc requests.
      expr: |-
        100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code)
          /
        sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code)
          > 1
      for: {{ dig "etcdHighNumberOfFailedGRPCRequests" "for" "10m" .Values.customRules }}
      {{- with .Values.defaultRules.keepFiringFor }}
      keep_firing_for: "{{ . }}"
      {{- end }}
      labels:
        severity: {{ dig "etcdHighNumberOfFailedGRPCRequests" "severity" "warning" .Values.customRules }}
      {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }}
        {{- with .Values.defaultRules.additionalRuleLabels }}
          {{- toYaml . | nindent 8 }}
        {{- end }}
        {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }}
          {{- toYaml . | nindent 8 }}
        {{- end }}
      {{- end }}
{{- end }}
{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfFailedGRPCRequests | default false) }}
    - alert: etcdHighNumberOfFailedGRPCRequests
      annotations:
{{- if .Values.defaultRules.additionalRuleAnnotations }}
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
{{- end }}
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }}
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }}
{{- end }}
        description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.grpc_method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}.'
        summary: etcd cluster has high number of failed grpc requests.
      expr: |-
        100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code)
          /
        sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code)
          > 5
      for: {{ dig "etcdHighNumberOfFailedGRPCRequests" "for" "5m" .Values.customRules }}
      {{- with .Values.defaultRules.keepFiringFor }}
      keep_firing_for: "{{ . }}"
      {{- end }}
      labels:
        severity: {{ dig "etcdHighNumberOfFailedGRPCRequests" "severity" "critical" .Values.customRules }}
      {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }}
        {{- with .Values.defaultRules.additionalRuleLabels }}
          {{- toYaml . | nindent 8 }}
        {{- end }}
        {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }}
          {{- toYaml . | nindent 8 }}
        {{- end }}
      {{- end }}
{{- end }}
{{- if not (.Values.defaultRules.disabled.etcdGRPCRequestsSlow | default false) }}
    - alert: etcdGRPCRequestsSlow
      annotations:
{{- if .Values.defaultRules.additionalRuleAnnotations }}
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
{{- end }}
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }}
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }}
{{- end }}
        description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile of gRPC requests is {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}} for {{`{{`}} $labels.grpc_method {{`}}`}} method.'
        summary: etcd grpc requests are slow
      expr: |-
        histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_method!="Defragment", grpc_type="unary"}[5m])) without(grpc_type))
        > 0.15
      for: {{ dig "etcdGRPCRequestsSlow" "for" "10m" .Values.customRules }}
      {{- with .Values.defaultRules.keepFiringFor }}
      keep_firing_for: "{{ . }}"
      {{- end }}
      labels:
        severity: {{ dig "etcdGRPCRequestsSlow" "severity" "critical" .Values.customRules }}
      {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }}
        {{- with .Values.defaultRules.additionalRuleLabels }}
          {{- toYaml . | nindent 8 }}
        {{- end }}
        {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }}
          {{- toYaml . | nindent 8 }}
        {{- end }}
      {{- end }}
{{- end }}
{{- if not (.Values.defaultRules.disabled.etcdMemberCommunicationSlow | default false) }}
    - alert: etcdMemberCommunicationSlow
      annotations:
{{- if .Values.defaultRules.additionalRuleAnnotations }}
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
{{- end }}
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }}
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }}
{{- end }}
        description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": member communication with {{`{{`}} $labels.To {{`}}`}} is taking {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.'
        summary: etcd cluster member communication is slow.
      expr: |-
        histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m]))
        > 0.15
      for: {{ dig "etcdMemberCommunicationSlow" "for" "10m" .Values.customRules }}
      {{- with .Values.defaultRules.keepFiringFor }}
      keep_firing_for: "{{ . }}"
      {{- end }}
      labels:
        severity: {{ dig "etcdMemberCommunicationSlow" "severity" "warning" .Values.customRules }}
      {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }}
        {{- with .Values.defaultRules.additionalRuleLabels }}
          {{- toYaml . | nindent 8 }}
        {{- end }}
        {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }}
          {{- toYaml . | nindent 8 }}
        {{- end }}
      {{- end }}
{{- end }}
{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfFailedProposals | default false) }}
    - alert: etcdHighNumberOfFailedProposals
      annotations:
{{- if .Values.defaultRules.additionalRuleAnnotations }}
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
{{- end }}
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }}
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }}
{{- end }}
        description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} proposal failures within the last 30 minutes on etcd instance {{`{{`}} $labels.instance {{`}}`}}.'
        summary: etcd cluster has high number of proposal failures.
      expr: rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5
      for: {{ dig "etcdHighNumberOfFailedProposals" "for" "15m" .Values.customRules }}
      {{- with .Values.defaultRules.keepFiringFor }}
      keep_firing_for: "{{ . }}"
      {{- end }}
      labels:
        severity: {{ dig "etcdHighNumberOfFailedProposals" "severity" "warning" .Values.customRules }}
      {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }}
        {{- with .Values.defaultRules.additionalRuleLabels }}
          {{- toYaml . | nindent 8 }}
        {{- end }}
        {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }}
          {{- toYaml . | nindent 8 }}
        {{- end }}
      {{- end }}
{{- end }}
{{- if not (.Values.defaultRules.disabled.etcdHighFsyncDurations | default false) }}
    - alert: etcdHighFsyncDurations
      annotations:
{{- if .Values.defaultRules.additionalRuleAnnotations }}
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
{{- end }}
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }}
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }}
{{- end }}
        description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile fsync durations are {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.'
        summary: etcd cluster 99th percentile fsync durations are too high.
      expr: |-
        histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
        > 0.5
      for: {{ dig "etcdHighFsyncDurations" "for" "10m" .Values.customRules }}
      {{- with .Values.defaultRules.keepFiringFor }}
      keep_firing_for: "{{ . }}"
      {{- end }}
      labels:
        severity: {{ dig "etcdHighFsyncDurations" "severity" "warning" .Values.customRules }}
      {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }}
        {{- with .Values.defaultRules.additionalRuleLabels }}
          {{- toYaml . | nindent 8 }}
        {{- end }}
        {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }}
          {{- toYaml . | nindent 8 }}
        {{- end }}
      {{- end }}
{{- end }}
{{- if not (.Values.defaultRules.disabled.etcdHighFsyncDurations | default false) }}
    - alert: etcdHighFsyncDurations
      annotations:
{{- if .Values.defaultRules.additionalRuleAnnotations }}
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
{{- end }}
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }}
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }}
{{- end }}
        description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile fsync durations are {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.'
        summary: etcd cluster 99th percentile fsync durations are too high.
      expr: |-
        histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
        > 1
      for: {{ dig "etcdHighFsyncDurations" "for" "10m" .Values.customRules }}
      {{- with .Values.defaultRules.keepFiringFor }}
      keep_firing_for: "{{ . }}"
      {{- end }}
      labels:
        severity: {{ dig "etcdHighFsyncDurations" "severity" "critical" .Values.customRules }}
      {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }}
        {{- with .Values.defaultRules.additionalRuleLabels }}
          {{- toYaml . | nindent 8 }}
        {{- end }}
        {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }}
          {{- toYaml . | nindent 8 }}
        {{- end }}
      {{- end }}
{{- end }}
{{- if not (.Values.defaultRules.disabled.etcdHighCommitDurations | default false) }}
    - alert: etcdHighCommitDurations
      annotations:
{{- if .Values.defaultRules.additionalRuleAnnotations }}
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
{{- end }}
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }}
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }}
{{- end }}
        description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile commit durations {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.'
        summary: etcd cluster 99th percentile commit durations are too high.
      expr: |-
        histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
        > 0.25
      for: {{ dig "etcdHighCommitDurations" "for" "10m" .Values.customRules }}
      {{- with .Values.defaultRules.keepFiringFor }}
      keep_firing_for: "{{ . }}"
      {{- end }}
      labels:
        severity: {{ dig "etcdHighCommitDurations" "severity" "warning" .Values.customRules }}
      {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }}
        {{- with .Values.defaultRules.additionalRuleLabels }}
          {{- toYaml . | nindent 8 }}
        {{- end }}
        {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }}
          {{- toYaml . | nindent 8 }}
        {{- end }}
      {{- end }}
{{- end }}
{{- if not (.Values.defaultRules.disabled.etcdDatabaseQuotaLowSpace | default false) }}
    - alert: etcdDatabaseQuotaLowSpace
      annotations:
{{- if .Values.defaultRules.additionalRuleAnnotations }}
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
{{- end }}
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }}
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }}
{{- end }}
        description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": database size exceeds the defined quota on etcd instance {{`{{`}} $labels.instance {{`}}`}}, please defrag or increase the quota as the writes to etcd will be disabled when it is full.'
        summary: etcd cluster database is running full.
      expr: (last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m]) / last_over_time(etcd_server_quota_backend_bytes{job=~".*etcd.*"}[5m]))*100 > 95
      for: {{ dig "etcdDatabaseQuotaLowSpace" "for" "10m" .Values.customRules }}
      {{- with .Values.defaultRules.keepFiringFor }}
      keep_firing_for: "{{ . }}"
      {{- end }}
      labels:
        severity: {{ dig "etcdDatabaseQuotaLowSpace" "severity" "critical" .Values.customRules }}
      {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }}
        {{- with .Values.defaultRules.additionalRuleLabels }}
          {{- toYaml . | nindent 8 }}
        {{- end }}
        {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }}
          {{- toYaml . | nindent 8 }}
        {{- end }}
      {{- end }}
{{- end }}
{{- if not (.Values.defaultRules.disabled.etcdExcessiveDatabaseGrowth | default false) }}
    - alert: etcdExcessiveDatabaseGrowth
      annotations:
{{- if .Values.defaultRules.additionalRuleAnnotations }}
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
{{- end }}
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }}
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }}
{{- end }}
        description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": Predicting running out of disk space in the next four hours, based on write observations within the past four hours on etcd instance {{`{{`}} $labels.instance {{`}}`}}, please check as it might be disruptive.'
        summary: etcd cluster database growing very fast.
      expr: predict_linear(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[4h], 4*60*60) > etcd_server_quota_backend_bytes{job=~".*etcd.*"}
      for: {{ dig "etcdExcessiveDatabaseGrowth" "for" "10m" .Values.customRules }}
      {{- with .Values.defaultRules.keepFiringFor }}
      keep_firing_for: "{{ . }}"
      {{- end }}
      labels:
        severity: {{ dig "etcdExcessiveDatabaseGrowth" "severity" "warning" .Values.customRules }}
      {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }}
        {{- with .Values.defaultRules.additionalRuleLabels }}
          {{- toYaml . | nindent 8 }}
        {{- end }}
        {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }}
          {{- toYaml . | nindent 8 }}
        {{- end }}
      {{- end }}
{{- end }}
{{- if not (.Values.defaultRules.disabled.etcdDatabaseHighFragmentationRatio | default false) }}
    - alert: etcdDatabaseHighFragmentationRatio
      annotations:
{{- if .Values.defaultRules.additionalRuleAnnotations }}
{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}
{{- end }}
{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }}
{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }}
{{- end }}
        description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": database size in use on instance {{`{{`}} $labels.instance {{`}}`}} is {{`{{`}} $value | humanizePercentage {{`}}`}} of the actual allocated disk space, please run defragmentation (e.g. etcdctl defrag) to retrieve the unused fragmented disk space.'
        runbook_url: https://etcd.io/docs/v3.5/op-guide/maintenance/#defragmentation
        summary: etcd database size in use is less than 50% of the actual allocated storage.
      expr: (last_over_time(etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"}[5m]) / last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m])) < 0.5 and etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"} > 104857600
      for: {{ dig "etcdDatabaseHighFragmentationRatio" "for" "10m" .Values.customRules }}
      {{- with .Values.defaultRules.keepFiringFor }}
      keep_firing_for: "{{ . }}"
      {{- end }}
      labels:
        severity: {{ dig "etcdDatabaseHighFragmentationRatio" "severity" "warning" .Values.customRules }}
      {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }}
        {{- with .Values.defaultRules.additionalRuleLabels }}
          {{- toYaml . | nindent 8 }}
        {{- end }}
        {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }}
          {{- toYaml . | nindent 8 }}
        {{- end }}
      {{- end }}
{{- end }}
{{- end }}