additional_metrics.yml 4.99 KB
Newer Older
1
2
3
4
5
6
- group: Response metrics (NGINX Ingress)
  priority: 10
  metrics:
  - title: "Throughput"
    y_label: "Requests / Sec"
    required_metrics:
7
      - nginx_upstream_responses_total
8
9
    weight: 1
    queries:
Joshua Lambert's avatar
Joshua Lambert committed
10
    - query_range: 'sum(rate(nginx_upstream_responses_total{upstream=~"%{kube_namespace}-%{ci_environment_slug}-.*"}[2m])) by (status_code)'
11
      unit: req / sec
Joshua Lambert's avatar
Joshua Lambert committed
12
      label: Status Code
13
14
15
16
17
18
      series:
        - label: status_code
          when:
            - value: 2xx
              color: green
            - value: 4xx
Joshua Lambert's avatar
Joshua Lambert committed
19
              color: orange
20
21
            - value: 5xx
              color: red
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
  - title: "Latency"
    y_label: "Latency (ms)"
    required_metrics:
      - nginx_upstream_response_msecs_avg
    weight: 1
    queries:
    - query_range: 'avg(nginx_upstream_response_msecs_avg{upstream=~"%{kube_namespace}-%{ci_environment_slug}-.*"})'
      label: Average
      unit: ms
  - title: "HTTP Error Rate"
    y_label: "HTTP 500 Errors / Sec"
    required_metrics:
      - nginx_upstream_responses_total
    weight: 1
    queries:
    - query_range: 'sum(rate(nginx_upstream_responses_total{status_code="5xx", upstream=~"%{kube_namespace}-%{ci_environment_slug}-.*"}[2m]))'
      label: HTTP Errors
      unit: "errors / sec"
Joshua Lambert's avatar
Joshua Lambert committed
40
- group: Response metrics (HA Proxy)
41
42
43
44
45
46
47
48
  priority: 10
  metrics:
  - title: "Throughput"
    y_label: "Requests / Sec"
    required_metrics:
      - haproxy_frontend_http_requests_total
    weight: 1
    queries:
49
    - query_range: 'sum(rate(haproxy_frontend_http_requests_total{%{environment_filter}}[2m])) by (code)'
50
      unit: req / sec
51
52
53
54
55
56
57
58
59
      series:
        - label: code
          when:
            - value: 2xx
              color: green
            - value: 4xx
              color: yellow
            - value: 5xx
              color: red
60
61
62
63
64
65
  - title: "HTTP Error Rate"
    y_label: "Error Rate (%)"
    required_metrics:
      - haproxy_frontend_http_responses_total
    weight: 1
    queries:
Joshua Lambert's avatar
Joshua Lambert committed
66
    - query_range: 'sum(rate(haproxy_frontend_http_responses_total{code="5xx",%{environment_filter}}[2m])) / sum(rate(haproxy_frontend_http_responses_total{%{environment_filter}}[2m]))'
67
68
      label: HTTP Errors
      unit: "%"
Joshua Lambert's avatar
Joshua Lambert committed
69
- group: Response metrics (AWS ELB)
Joshua Lambert's avatar
Joshua Lambert committed
70
71
  priority: 10
  metrics:
72
73
  - title: "Throughput"
    y_label: "Requests / Sec"
Joshua Lambert's avatar
Joshua Lambert committed
74
75
76
77
    required_metrics:
      - aws_elb_request_count_sum
    weight: 1
    queries:
78
    - query_range: 'sum(aws_elb_request_count_sum{%{environment_filter}}) / 60'
79
80
      label: Total
      unit: req / sec
Joshua Lambert's avatar
Joshua Lambert committed
81
  - title: "Latency"
82
    y_label: "Latency (ms)"
Joshua Lambert's avatar
Joshua Lambert committed
83
    required_metrics:
84
      - aws_elb_latency_average
Joshua Lambert's avatar
Joshua Lambert committed
85
86
    weight: 1
    queries:
87
88
89
90
91
    - query_range: 'avg(aws_elb_latency_average{%{environment_filter}}) * 1000'
      label: Average
      unit: ms
  - title: "HTTP Error Rate"
    y_label: "Error Rate (%)"
Joshua Lambert's avatar
Joshua Lambert committed
92
    required_metrics:
93
94
      - aws_elb_request_count_sum
      - aws_elb_httpcode_backend_5_xx_sum
Joshua Lambert's avatar
Joshua Lambert committed
95
96
97
    weight: 1
    queries:
    - query_range: 'sum(aws_elb_httpcode_backend_5_xx_sum{%{environment_filter}}) / sum(aws_elb_request_count_sum{%{environment_filter}})'
98
99
      label: HTTP Errors
      unit: "%"
Joshua Lambert's avatar
Joshua Lambert committed
100
- group: Response metrics (NGINX)
101
  priority: 10
102
  metrics:
103
104
  - title: "Throughput"
    y_label: "Requests / Sec"
Pawel Chojnacki's avatar
Pawel Chojnacki committed
105
    required_metrics:
106
      - nginx_responses_total
107
108
    weight: 1
    queries:
109
    - query_range: 'sum(rate(nginx_responses_total{server_zone!="*", server_zone!="_", %{environment_filter}}[2m])) by (status_code)'
110
      unit: req / sec
111
112
113
114
115
116
117
118
119
120
      label: Status Code
      series:
        - label: status_code
          when:
            - value: 2xx
              color: green
            - value: 4xx
              color: orange
            - value: 5xx
              color: red
121
122
  - title: "Latency"
    y_label: "Latency (ms)"
Pawel Chojnacki's avatar
Pawel Chojnacki committed
123
    required_metrics:
124
      - nginx_upstream_response_msecs_avg
125
126
    weight: 1
    queries:
127
    - query_range: 'avg(nginx_upstream_response_msecs_avg{%{environment_filter}})'
128
129
130
      label: Upstream
      unit: ms
  - title: "HTTP Error Rate"
131
    y_label: "HTTP 500 Errors / Sec"
Pawel Chojnacki's avatar
Pawel Chojnacki committed
132
    required_metrics:
133
134
135
      - nginx_responses_total
    weight: 1
    queries:
136
    - query_range: 'sum(rate(nginx_responses_total{status_code="5xx", %{environment_filter}}[2m]))'
137
      label: HTTP Errors
138
      unit: "errors / sec"
Joshua Lambert's avatar
Joshua Lambert committed
139
- group: System metrics (Kubernetes)
140
141
142
143
144
145
  priority: 5
  metrics:
  - title: "Memory Usage"
    y_label: "Memory Usage (MB)"
    required_metrics:
      - container_memory_usage_bytes
146
147
    weight: 1
    queries:
Joshua Lambert's avatar
Joshua Lambert committed
148
    - query_range: '(sum(avg(container_memory_usage_bytes{container_name!="POD",environment="%{ci_environment_slug}"}) without (job))) / count(avg(container_memory_usage_bytes{container_name!="POD",environment="%{ci_environment_slug}"}) without (job)) /1024/1024'
149
150
151
152
      label: Average
      unit: MB
  - title: "CPU Utilization"
    y_label: "CPU Utilization (%)"
Pawel Chojnacki's avatar
Pawel Chojnacki committed
153
154
    required_metrics:
     - container_cpu_usage_seconds_total
155
156
    weight: 1
    queries:
Joshua Lambert's avatar
Joshua Lambert committed
157
    - query_range: 'sum(avg(rate(container_cpu_usage_seconds_total{container_name!="POD",environment="%{ci_environment_slug}"}[2m])) without (job)) * 100'
158
159
      label: Average
      unit: "%"