kube-state-metrics

alert: KubeStateMetricsListErrors
annotations:
  description: kube-state-metrics is experiencing errors at an elevated rate in list
    operations. This is likely causing it to not be able to expose metrics about Kubernetes
    objects correctly or at all.
  summary: kube-state-metrics is experiencing errors in list operations.
expr: |
  (sum(rate(kube_state_metrics_list_total{job="kube-state-metrics",result="error"}[5m])) by (cluster)
    /
  sum(rate(kube_state_metrics_list_total{job="kube-state-metrics"}[5m])) by (cluster))
  > 0.01
for: 15m
labels:
  severity: critical

alert: KubeStateMetricsWatchErrors
annotations:
  description: kube-state-metrics is experiencing errors at an elevated rate in watch
    operations. This is likely causing it to not be able to expose metrics about Kubernetes
    objects correctly or at all.
  summary: kube-state-metrics is experiencing errors in watch operations.
expr: |
  (sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics",result="error"}[5m])) by (cluster)
    /
  sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics"}[5m])) by (cluster))
  > 0.01
for: 15m
labels:
  severity: critical

alert: KubeStateMetricsShardingMismatch
annotations:
  description: kube-state-metrics pods are running with different --total-shards configuration,
    some Kubernetes objects may be exposed multiple times or not exposed at all.
  summary: kube-state-metrics sharding is misconfigured.
expr: |
  stdvar (kube_state_metrics_total_shards{job="kube-state-metrics"}) by (cluster) != 0
for: 15m
labels:
  severity: critical

alert: KubeStateMetricsShardsMissing
annotations:
  description: kube-state-metrics shards are missing, some Kubernetes objects are
    not being exposed.
  summary: kube-state-metrics shards are missing.
expr: |
  2^max(kube_state_metrics_total_shards{job="kube-state-metrics"}) by (cluster) - 1
    -
  sum( 2 ^ max by (cluster, shard_ordinal) (kube_state_metrics_shard_ordinal{job="kube-state-metrics"}) ) by (cluster)
  != 0
for: 15m
labels:
  severity: critical