jaeger

alert: JaegerAgentUDPPacketsBeingDropped
annotations:
  message: |
    {{ $labels.job }} {{ $labels.instance }} is dropping {{ printf "%.2f" $value }} UDP packets per second.
expr: rate(jaeger_agent_thrift_udp_server_packets_dropped_total[1m]) > 1
for: 15m
labels:
  severity: warning

alert: JaegerAgentHTTPServerErrs
annotations:
  message: |
    {{ $labels.job }} {{ $labels.instance }} is experiencing {{ printf "%.2f" $value }}% HTTP errors.
expr: 100 * sum(rate(jaeger_agent_http_server_errors_total[1m])) by (instance, job,
  namespace) / sum(rate(jaeger_agent_http_server_total[1m])) by (instance, job, namespace)>
  1
for: 15m
labels:
  severity: warning

alert: JaegerClientSpansDropped
annotations:
  message: |
    service {{ $labels.job }} {{ $labels.instance }} is dropping {{ printf "%.2f" $value }}% spans.
expr: 100 * sum(rate(jaeger_reporter_spans{result=~"dropped|err"}[1m])) by (instance,
  job, namespace) / sum(rate(jaeger_reporter_spans[1m])) by (instance, job, namespace)>
  1
for: 15m
labels:
  severity: warning

alert: JaegerAgentSpansDropped
annotations:
  message: |
    agent {{ $labels.job }} {{ $labels.instance }} is dropping {{ printf "%.2f" $value }}% spans.
expr: 100 * sum(rate(jaeger_agent_reporter_batches_failures_total[1m])) by (instance,
  job, namespace) / sum(rate(jaeger_agent_reporter_batches_submitted_total[1m])) by
  (instance, job, namespace)> 1
for: 15m
labels:
  severity: warning

alert: JaegerCollectorQueueNotDraining
annotations:
  message: |
    collector {{ $labels.job }} {{ $labels.instance }} is not able to drain the queue.
expr: avg_over_time(jaeger_collector_queue_length[10m]) > 1000
for: 15m
labels:
  severity: warning

alert: JaegerCollectorDroppingSpans
annotations:
  message: |
    collector {{ $labels.job }} {{ $labels.instance }} is dropping {{ printf "%.2f" $value }}% spans.
expr: 100 * sum(rate(jaeger_collector_spans_dropped_total[1m])) by (instance, job,
  namespace) / sum(rate(jaeger_collector_spans_received_total[1m])) by (instance,
  job, namespace)> 1
for: 15m
labels:
  severity: warning

alert: JaegerSamplingUpdateFailing
annotations:
  message: |
    {{ $labels.job }} {{ $labels.instance }} is failing {{ printf "%.2f" $value }}% in updating sampling policies.
expr: 100 * sum(rate(jaeger_sampler_queries{result="err"}[1m])) by (instance, job,
  namespace) / sum(rate(jaeger_sampler_queries[1m])) by (instance, job, namespace)>
  1
for: 15m
labels:
  severity: warning

alert: JaegerCollectorPersistenceSlow
annotations:
  message: |
    {{ $labels.job }} {{ $labels.instance }} is slow at persisting spans.
expr: histogram_quantile(0.99, sum by (le) (rate(jaeger_collector_save_latency_bucket[1m])))
  > 0.5
for: 15m
labels:
  severity: warning

alert: JaegerThrottlingUpdateFailing
annotations:
  message: |
    {{ $labels.job }} {{ $labels.instance }} is failing {{ printf "%.2f" $value }}% in updating throttling policies.
expr: 100 * sum(rate(jaeger_throttler_updates{result="err"}[1m])) by (instance, job,
  namespace) / sum(rate(jaeger_throttler_updates[1m])) by (instance, job, namespace)>
  1
for: 15m
labels:
  severity: warning

alert: JaegerQueryReqsFailing
annotations:
  message: |
    {{ $labels.job }} {{ $labels.instance }} is seeing {{ printf "%.2f" $value }}% query errors on {{ $labels.operation }}.
expr: 100 * sum(rate(jaeger_query_requests_total{result="err"}[1m])) by (instance,
  job, namespace) / sum(rate(jaeger_query_requests_total[1m])) by (instance, job,
  namespace)> 1
for: 15m
labels:
  severity: warning

alert: JaegerCassandraWritesFailing
annotations:
  message: |
    {{ $labels.job }} {{ $labels.instance }} is seeing {{ printf "%.2f" $value }}% query errors on {{ $labels.operation }}.
expr: 100 * sum(rate(jaeger_cassandra_errors_total[1m])) by (instance, job, namespace)
  / sum(rate(jaeger_cassandra_attempts_total[1m])) by (instance, job, namespace)>
  1
for: 15m
labels:
  severity: warning

alert: JaegerCassandraReadsFailing
annotations:
  message: |
    {{ $labels.job }} {{ $labels.instance }} is seeing {{ printf "%.2f" $value }}% query errors on {{ $labels.operation }}.
expr: 100 * sum(rate(jaeger_cassandra_read_errors_total[1m])) by (instance, job, namespace)
  / sum(rate(jaeger_cassandra_read_attempts_total[1m])) by (instance, job, namespace)>
  1
for: 15m
labels:
  severity: warning