Skip to content
Snippets Groups Projects
Commit bd299319 authored by Ian Baum's avatar Ian Baum
Browse files

Merge branch 'bjk/4246_alert_sensitivity' into 'master'

Update WorkhorseHighErrorRate alert

See merge request gitlab-org/omnibus-gitlab!3183
parents eaf77abd c3e7924b
No related branches found
No related tags found
No related merge requests found
---
title: Update WorkhorseHighErrorRate alert
merge_request: 3183
author:
type: fixed
......@@ -73,4 +73,5 @@
job_route_method_code:gitlab_workhorse_http_request_duration_seconds_count:rate5m = sum by (job, route, method, code) (rate(gitlab_workhorse_http_request_duration_seconds_count[5m]))
# Low-traffic - < 10 QPS (600 RPM)
ALERT WorkhorseHighErrorRate
......@@ -76,5 +77,12 @@
ALERT WorkhorseHighErrorRate
IF sum without (job, code) (job_route_method_code:gitlab_workhorse_http_request_duration_seconds_count:rate5m{code=~"5.."}) / sum without (job,code) (job_route_method_code:gitlab_workhorse_http_request_duration_seconds_count:rate5m) * 100 > 1
IF (
sum without (job, code) (
job_route_method_code:gitlab_workhorse_http_request_duration_seconds_count:rate5m{code=~"5.."}
) /
sum without (job,code) (
job_route_method_code:gitlab_workhorse_http_request_duration_seconds_count:rate5m
) < 10
) * 100 > 50
FOR 60m
ANNOTATIONS {
summary = 'Workhorse has high error rates',
......@@ -78,5 +86,5 @@
FOR 60m
ANNOTATIONS {
summary = 'Workhorse has high error rates',
description = 'Workhorse route {{ $labels.route }} method {{ $labels.method }} has more than 1% errors ({{ $value | printf "%.1f" }}%) for the last 60 minutes.',
description = 'Workhorse route {{ $labels.route }} method {{ $labels.method }} has more than 50% errors ({{ $value | printf "%.1f" }}%) for the last 60 minutes.',
}
......@@ -82,1 +90,17 @@
}
# High-traffic - >= 10 QPS (600 RPM)
ALERT WorkhorseHighErrorRate
IF (
sum without (job, code) (
job_route_method_code:gitlab_workhorse_http_request_duration_seconds_count:rate5m{code=~"5.."}
) /
sum without (job,code) (
job_route_method_code:gitlab_workhorse_http_request_duration_seconds_count:rate5m
) >= 10
) * 100 > 10
FOR 60m
ANNOTATIONS {
summary = 'Workhorse has high error rates',
description = 'Workhorse route {{ $labels.route }} method {{ $labels.method }} has more than 10% errors ({{ $value | printf "%.1f" }}%) for the last 60 minutes.',
}
......@@ -78,5 +78,6 @@
annotations:
summary: Postgres database has queries canceled due to deadlocks
description: Postgres database {{ $labels.instance }} had {{ $value | printf "%d" }} queries canceled due to deadlocks in the last 5 minutes.
# Low-traffic - < 10 QPS (600 RPM)
- alert: WorkhorseHighErrorRate
expr: >
......@@ -81,10 +82,12 @@
- alert: WorkhorseHighErrorRate
expr: >
sum without (job, code) (
job_route_method_code:gitlab_workhorse_http_request_duration_seconds_count:rate5m{code=~"5.."}
) /
sum without (job,code) (
job_route_method_code:gitlab_workhorse_http_request_duration_seconds_count:rate5m
) * 100 > 1
(
sum without (job, code) (
job_route_method_code:gitlab_workhorse_http_request_duration_seconds_count:rate5m{code=~"5.."}
) /
sum without (job,code) (
job_route_method_code:gitlab_workhorse_http_request_duration_seconds_count:rate5m
) < 10
) * 100 > 50
annotations:
summary: Workhorse has high error rates
......@@ -89,3 +92,17 @@
annotations:
summary: Workhorse has high error rates
description: Workhorse route {{ $labels.route }} method {{ $labels.method }} has more than 1% errors ({{ $value | printf "%.1f" }}%) for the last 60 minutes.
description: Workhorse route {{ $labels.route }} method {{ $labels.method }} has more than 50% errors ({{ $value | printf "%.1f" }}%) for the last 60 minutes.
# High-traffic - >= 10 QPS (600 RPM)
- alert: WorkhorseHighErrorRate
expr: >
(
sum without (job, code) (
job_route_method_code:gitlab_workhorse_http_request_duration_seconds_count:rate5m{code=~"5.."}
) /
sum without (job,code) (
job_route_method_code:gitlab_workhorse_http_request_duration_seconds_count:rate5m
) > 10
) * 100 > 10
annotations:
summary: Workhorse has high error rates
description: Workhorse route {{ $labels.route }} method {{ $labels.method }} has more than 10% errors ({{ $value | printf "%.1f" }}%) for the last 60 minutes.
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment