diff --git a/changelogs/unreleased/bjk-4246_alert_sensitivity.yml b/changelogs/unreleased/bjk-4246_alert_sensitivity.yml new file mode 100644 index 0000000000000000000000000000000000000000..c3e7924beb540813713c38f14ad99abda13678e7_Y2hhbmdlbG9ncy91bnJlbGVhc2VkL2Jqay00MjQ2X2FsZXJ0X3NlbnNpdGl2aXR5LnltbA== --- /dev/null +++ b/changelogs/unreleased/bjk-4246_alert_sensitivity.yml @@ -0,0 +1,5 @@ +--- +title: Update WorkhorseHighErrorRate alert +merge_request: 3183 +author: +type: fixed diff --git a/files/gitlab-cookbooks/gitlab/templates/default/prometheus/rules/gitlab.rules.v1 b/files/gitlab-cookbooks/gitlab/templates/default/prometheus/rules/gitlab.rules.v1 index 622ee2ca3703e7445d14080cc017f9cf6dad4a2a_ZmlsZXMvZ2l0bGFiLWNvb2tib29rcy9naXRsYWIvdGVtcGxhdGVzL2RlZmF1bHQvcHJvbWV0aGV1cy9ydWxlcy9naXRsYWIucnVsZXMudjE=..c3e7924beb540813713c38f14ad99abda13678e7_ZmlsZXMvZ2l0bGFiLWNvb2tib29rcy9naXRsYWIvdGVtcGxhdGVzL2RlZmF1bHQvcHJvbWV0aGV1cy9ydWxlcy9naXRsYWIucnVsZXMudjE= 100644 --- a/files/gitlab-cookbooks/gitlab/templates/default/prometheus/rules/gitlab.rules.v1 +++ b/files/gitlab-cookbooks/gitlab/templates/default/prometheus/rules/gitlab.rules.v1 @@ -73,4 +73,5 @@ job_route_method_code:gitlab_workhorse_http_request_duration_seconds_count:rate5m = sum by (job, route, method, code) (rate(gitlab_workhorse_http_request_duration_seconds_count[5m])) +# Low-traffic - < 10 QPS (600 RPM) ALERT WorkhorseHighErrorRate @@ -76,5 +77,12 @@ ALERT WorkhorseHighErrorRate - IF sum without (job, code) (job_route_method_code:gitlab_workhorse_http_request_duration_seconds_count:rate5m{code=~"5.."}) / sum without (job,code) (job_route_method_code:gitlab_workhorse_http_request_duration_seconds_count:rate5m) * 100 > 1 + IF ( + sum without (job, code) ( + job_route_method_code:gitlab_workhorse_http_request_duration_seconds_count:rate5m{code=~"5.."} + ) / + sum without (job,code) ( + job_route_method_code:gitlab_workhorse_http_request_duration_seconds_count:rate5m + ) < 10 + ) * 100 > 50 FOR 60m ANNOTATIONS { summary = 'Workhorse has high error rates', @@ -78,5 +86,5 @@ FOR 60m ANNOTATIONS { summary = 'Workhorse has high error rates', - description = 'Workhorse route {{ $labels.route }} method {{ $labels.method }} has more than 1% errors ({{ $value | printf "%.1f" }}%) for the last 60 minutes.', + description = 'Workhorse route {{ $labels.route }} method {{ $labels.method }} has more than 50% errors ({{ $value | printf "%.1f" }}%) for the last 60 minutes.', } @@ -82,1 +90,17 @@ } + +# High-traffic - >= 10 QPS (600 RPM) +ALERT WorkhorseHighErrorRate + IF ( + sum without (job, code) ( + job_route_method_code:gitlab_workhorse_http_request_duration_seconds_count:rate5m{code=~"5.."} + ) / + sum without (job,code) ( + job_route_method_code:gitlab_workhorse_http_request_duration_seconds_count:rate5m + ) >= 10 + ) * 100 > 10 + FOR 60m + ANNOTATIONS { + summary = 'Workhorse has high error rates', + description = 'Workhorse route {{ $labels.route }} method {{ $labels.method }} has more than 10% errors ({{ $value | printf "%.1f" }}%) for the last 60 minutes.', + } diff --git a/files/gitlab-cookbooks/gitlab/templates/default/prometheus/rules/gitlab.rules.v2 b/files/gitlab-cookbooks/gitlab/templates/default/prometheus/rules/gitlab.rules.v2 index 622ee2ca3703e7445d14080cc017f9cf6dad4a2a_ZmlsZXMvZ2l0bGFiLWNvb2tib29rcy9naXRsYWIvdGVtcGxhdGVzL2RlZmF1bHQvcHJvbWV0aGV1cy9ydWxlcy9naXRsYWIucnVsZXMudjI=..c3e7924beb540813713c38f14ad99abda13678e7_ZmlsZXMvZ2l0bGFiLWNvb2tib29rcy9naXRsYWIvdGVtcGxhdGVzL2RlZmF1bHQvcHJvbWV0aGV1cy9ydWxlcy9naXRsYWIucnVsZXMudjI= 100644 --- a/files/gitlab-cookbooks/gitlab/templates/default/prometheus/rules/gitlab.rules.v2 +++ b/files/gitlab-cookbooks/gitlab/templates/default/prometheus/rules/gitlab.rules.v2 @@ -78,5 +78,6 @@ annotations: summary: Postgres database has queries canceled due to deadlocks description: Postgres database {{ $labels.instance }} had {{ $value | printf "%d" }} queries canceled due to deadlocks in the last 5 minutes. + # Low-traffic - < 10 QPS (600 RPM) - alert: WorkhorseHighErrorRate expr: > @@ -81,10 +82,12 @@ - alert: WorkhorseHighErrorRate expr: > - sum without (job, code) ( - job_route_method_code:gitlab_workhorse_http_request_duration_seconds_count:rate5m{code=~"5.."} - ) / - sum without (job,code) ( - job_route_method_code:gitlab_workhorse_http_request_duration_seconds_count:rate5m - ) * 100 > 1 + ( + sum without (job, code) ( + job_route_method_code:gitlab_workhorse_http_request_duration_seconds_count:rate5m{code=~"5.."} + ) / + sum without (job,code) ( + job_route_method_code:gitlab_workhorse_http_request_duration_seconds_count:rate5m + ) < 10 + ) * 100 > 50 annotations: summary: Workhorse has high error rates @@ -89,3 +92,17 @@ annotations: summary: Workhorse has high error rates - description: Workhorse route {{ $labels.route }} method {{ $labels.method }} has more than 1% errors ({{ $value | printf "%.1f" }}%) for the last 60 minutes. + description: Workhorse route {{ $labels.route }} method {{ $labels.method }} has more than 50% errors ({{ $value | printf "%.1f" }}%) for the last 60 minutes. + # High-traffic - >= 10 QPS (600 RPM) + - alert: WorkhorseHighErrorRate + expr: > + ( + sum without (job, code) ( + job_route_method_code:gitlab_workhorse_http_request_duration_seconds_count:rate5m{code=~"5.."} + ) / + sum without (job,code) ( + job_route_method_code:gitlab_workhorse_http_request_duration_seconds_count:rate5m + ) > 10 + ) * 100 > 10 + annotations: + summary: Workhorse has high error rates + description: Workhorse route {{ $labels.route }} method {{ $labels.method }} has more than 10% errors ({{ $value | printf "%.1f" }}%) for the last 60 minutes.