Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
heptapod
heptapod
Commits
50a118708e49
Commit
746f0ec3
authored
Aug 07, 2017
by
Paweł Chojnacki
Committed by
Rémy Coutable
Aug 07, 2017
Browse files
Add sidekiq metrics endpoint and add http server to sidekiq
parent
7e29e2e722f2
Changes
10
Hide whitespace changes
Inline
Side-by-side
changelogs/unreleased/pawel-add-sidekiq-metrics-endpoint-32145.yml
0 → 100644
View file @
50a11870
---
title
:
Add Prometheus metrics exporter to Sidekiq
merge_request
:
13082
author
:
config/gitlab.yml.example
View file @
50a11870
...
...
@@ -590,6 +590,12 @@ production: &base
ip_whitelist:
- 127.0.0.0/8
# Sidekiq exporter is webserver built in to Sidekiq to expose Prometheus metrics
sidekiq_exporter:
# enabled: true
# address: localhost
# port: 3807
#
# 5. Extra customization
# ==========================
...
...
config/initializers/1_settings.rb
View file @
50a11870
...
...
@@ -524,6 +524,10 @@ def cron_random_weekly_time
Settings
[
'monitoring'
]
||=
Settingslogic
.
new
({})
Settings
.
monitoring
[
'ip_whitelist'
]
||=
[
'127.0.0.1/8'
]
Settings
.
monitoring
[
'unicorn_sampler_interval'
]
||=
10
Settings
.
monitoring
[
'sidekiq_exporter'
]
||=
Settingslogic
.
new
({})
Settings
.
monitoring
.
sidekiq_exporter
[
'enabled'
]
||=
false
Settings
.
monitoring
.
sidekiq_exporter
[
'address'
]
||=
'localhost'
Settings
.
monitoring
.
sidekiq_exporter
[
'port'
]
||=
3807
#
# Testing settings
...
...
config/initializers/7_prometheus_metrics.rb
View file @
50a11870
...
...
@@ -10,3 +10,9 @@
config
.
multiprocess_files_dir
||=
Rails
.
root
.
join
(
'tmp/prometheus_multiproc_dir'
)
end
end
Sidekiq
.
configure_server
do
|
config
|
config
.
on
(
:startup
)
do
Gitlab
::
Metrics
::
SidekiqMetricsExporter
.
instance
.
start
end
end
lib/gitlab/daemon.rb
0 → 100644
View file @
50a11870
module
Gitlab
class
Daemon
def
self
.
initialize_instance
(
*
args
)
raise
"
#{
name
}
singleton instance already initialized"
if
@instance
@instance
=
new
(
*
args
)
Kernel
.
at_exit
(
&
@instance
.
method
(
:stop
))
@instance
end
def
self
.
instance
@instance
||=
initialize_instance
end
attr_reader
:thread
def
thread?
!
thread
.
nil?
end
def
initialize
@mutex
=
Mutex
.
new
end
def
enabled?
true
end
def
start
return
unless
enabled?
@mutex
.
synchronize
do
return
thread
if
thread?
@thread
=
Thread
.
new
{
start_working
}
end
end
def
stop
@mutex
.
synchronize
do
return
unless
thread?
stop_working
if
thread
thread
.
wakeup
if
thread
.
alive?
thread
.
join
@thread
=
nil
end
end
end
private
def
start_working
raise
NotImplementedError
end
def
stop_working
# no-ops
end
end
end
lib/gitlab/metrics/base_sampler.rb
View file @
50a11870
require
'logger'
module
Gitlab
module
Metrics
class
BaseSampler
def
self
.
initialize_instance
(
*
args
)
raise
"
#{
name
}
singleton instance already initialized"
if
@instance
@instance
=
new
(
*
args
)
at_exit
(
&
@instance
.
method
(
:stop
))
@instance
end
def
self
.
instance
@instance
end
attr_reader
:running
class
BaseSampler
<
Daemon
# interval - The sampling interval in seconds.
def
initialize
(
interval
)
interval_half
=
interval
.
to_f
/
2
...
...
@@ -22,44 +9,7 @@ def initialize(interval)
@interval
=
interval
@interval_steps
=
(
-
interval_half
..
interval_half
).
step
(
0.1
).
to_a
@mutex
=
Mutex
.
new
end
def
enabled?
true
end
def
start
return
unless
enabled?
@mutex
.
synchronize
do
return
if
running
@running
=
true
@thread
=
Thread
.
new
do
sleep
(
sleep_interval
)
while
running
safe_sample
sleep
(
sleep_interval
)
end
end
end
end
def
stop
@mutex
.
synchronize
do
return
unless
running
@running
=
false
if
@thread
@thread
.
wakeup
if
@thread
.
alive?
@thread
.
join
@thread
=
nil
end
end
super
()
end
def
safe_sample
...
...
@@ -81,7 +31,7 @@ def sample
# potentially missing anything that happens in between samples).
# 2. Don't sample data at the same interval two times in a row.
def
sleep_interval
while
step
=
@interval_steps
.
sample
while
(
step
=
@interval_steps
.
sample
)
if
step
!=
@last_step
@last_step
=
step
...
...
@@ -89,6 +39,25 @@ def sleep_interval
end
end
end
private
attr_reader
:running
def
start_working
@running
=
true
sleep
(
sleep_interval
)
while
running
safe_sample
sleep
(
sleep_interval
)
end
end
def
stop_working
@running
=
false
end
end
end
end
lib/gitlab/metrics/sidekiq_metrics_exporter.rb
0 → 100644
View file @
50a11870
require
'webrick'
require
'prometheus/client/rack/exporter'
module
Gitlab
module
Metrics
class
SidekiqMetricsExporter
<
Daemon
def
enabled?
Gitlab
::
Metrics
.
metrics_folder_present?
&&
settings
.
enabled
end
def
settings
Settings
.
monitoring
.
sidekiq_exporter
end
private
attr_reader
:server
def
start_working
@server
=
::
WEBrick
::
HTTPServer
.
new
(
Port
:
settings
.
port
,
BindAddress
:
settings
.
address
)
server
.
mount
"/"
,
Rack
::
Handler
::
WEBrick
,
rack_app
server
.
start
end
def
stop_working
server
.
shutdown
@server
=
nil
end
def
rack_app
Rack
::
Builder
.
app
do
use
Rack
::
Deflater
use
::
Prometheus
::
Client
::
Rack
::
Exporter
run
->
(
env
)
{
[
404
,
{},
[
''
]]
}
end
end
end
end
end
spec/lib/gitlab/daemon_spec.rb
0 → 100644
View file @
50a11870
require
'spec_helper'
describe
Gitlab
::
Daemon
do
subject
{
described_class
.
new
}
before
do
allow
(
subject
).
to
receive
(
:start_working
)
allow
(
subject
).
to
receive
(
:stop_working
)
end
describe
'.instance'
do
before
do
allow
(
Kernel
).
to
receive
(
:at_exit
)
end
after
(
:each
)
do
described_class
.
instance_variable_set
(
:@instance
,
nil
)
end
it
'provides instance of Daemon'
do
expect
(
described_class
.
instance
).
to
be_instance_of
(
described_class
)
end
it
'subsequent invocations provide the same instance'
do
expect
(
described_class
.
instance
).
to
eq
(
described_class
.
instance
)
end
it
'creates at_exit hook when instance is created'
do
expect
(
described_class
.
instance
).
not_to
be_nil
expect
(
Kernel
).
to
have_received
(
:at_exit
)
end
end
describe
'when Daemon is enabled'
do
before
do
allow
(
subject
).
to
receive
(
:enabled?
).
and_return
(
true
)
end
describe
'when Daemon is stopped'
do
describe
'#start'
do
it
'starts the Daemon'
do
expect
{
subject
.
start
.
join
}.
to
change
{
subject
.
thread?
}.
from
(
false
).
to
(
true
)
expect
(
subject
).
to
have_received
(
:start_working
)
end
end
describe
'#stop'
do
it
"doesn't shutdown stopped Daemon"
do
expect
{
subject
.
stop
}.
not_to
change
{
subject
.
thread?
}
expect
(
subject
).
not_to
have_received
(
:start_working
)
end
end
end
describe
'when Daemon is running'
do
before
do
subject
.
start
.
join
end
describe
'#start'
do
it
"doesn't start running Daemon"
do
expect
{
subject
.
start
.
join
}.
not_to
change
{
subject
.
thread?
}
expect
(
subject
).
to
have_received
(
:start_working
).
once
end
end
describe
'#stop'
do
it
'shutdowns Daemon'
do
expect
{
subject
.
stop
}.
to
change
{
subject
.
thread?
}.
from
(
true
).
to
(
false
)
expect
(
subject
).
to
have_received
(
:stop_working
)
end
end
end
end
describe
'when Daemon is disabled'
do
before
do
allow
(
subject
).
to
receive
(
:enabled?
).
and_return
(
false
)
end
describe
'#start'
do
it
"doesn't start working"
do
expect
(
subject
.
start
).
to
be_nil
expect
{
subject
.
start
}.
not_to
change
{
subject
.
thread?
}
expect
(
subject
).
not_to
have_received
(
:start_working
)
end
end
describe
'#stop'
do
it
"doesn't stop working"
do
expect
{
subject
.
stop
}.
not_to
change
{
subject
.
thread?
}
expect
(
subject
).
not_to
have_received
(
:stop_working
)
end
end
end
end
spec/lib/gitlab/metrics/influx_sampler_spec.rb
View file @
50a11870
...
...
@@ -11,7 +11,7 @@
it
'runs once and gathers a sample at a given interval'
do
expect
(
sampler
).
to
receive
(
:sleep
).
with
(
a_kind_of
(
Numeric
)).
twice
expect
(
sampler
).
to
receive
(
:sample
).
once
expect
(
sampler
).
to
receive
(
:running
).
and_return
(
false
,
true
,
false
)
expect
(
sampler
).
to
receive
(
:running
).
and_return
(
true
,
false
)
sampler
.
start
.
join
end
...
...
spec/lib/gitlab/metrics/sidekiq_metrics_exporter_spec.rb
0 → 100644
View file @
50a11870
require
'spec_helper'
describe
Gitlab
::
Metrics
::
SidekiqMetricsExporter
do
let
(
:exporter
)
{
described_class
.
new
}
let
(
:server
)
{
double
(
'server'
)
}
before
do
allow
(
::
WEBrick
::
HTTPServer
).
to
receive
(
:new
).
and_return
(
server
)
allow
(
server
).
to
receive
(
:mount
)
allow
(
server
).
to
receive
(
:start
)
allow
(
server
).
to
receive
(
:shutdown
)
end
describe
'when exporter is enabled'
do
before
do
allow
(
Settings
.
monitoring
.
sidekiq_exporter
).
to
receive
(
:enabled
).
and_return
(
true
)
end
describe
'when exporter is stopped'
do
describe
'#start'
do
it
'starts the exporter'
do
expect
{
exporter
.
start
.
join
}.
to
change
{
exporter
.
thread?
}.
from
(
false
).
to
(
true
)
expect
(
server
).
to
have_received
(
:start
)
end
describe
'with custom settings'
do
let
(
:port
)
{
99999
}
let
(
:address
)
{
'sidekiq_exporter_address'
}
before
do
allow
(
Settings
.
monitoring
.
sidekiq_exporter
).
to
receive
(
:port
).
and_return
(
port
)
allow
(
Settings
.
monitoring
.
sidekiq_exporter
).
to
receive
(
:address
).
and_return
(
address
)
end
it
'starts server with port and address from settings'
do
exporter
.
start
.
join
expect
(
::
WEBrick
::
HTTPServer
).
to
have_received
(
:new
).
with
(
Port
:
port
,
BindAddress
:
address
)
end
end
end
describe
'#stop'
do
it
"doesn't shutdown stopped server"
do
expect
{
exporter
.
stop
}.
not_to
change
{
exporter
.
thread?
}
expect
(
server
).
not_to
have_received
(
:shutdown
)
end
end
end
describe
'when exporter is running'
do
before
do
exporter
.
start
.
join
end
describe
'#start'
do
it
"doesn't start running server"
do
expect
{
exporter
.
start
.
join
}.
not_to
change
{
exporter
.
thread?
}
expect
(
server
).
to
have_received
(
:start
).
once
end
end
describe
'#stop'
do
it
'shutdowns server'
do
expect
{
exporter
.
stop
}.
to
change
{
exporter
.
thread?
}.
from
(
true
).
to
(
false
)
expect
(
server
).
to
have_received
(
:shutdown
)
end
end
end
end
describe
'when exporter is disabled'
do
before
do
allow
(
Settings
.
monitoring
.
sidekiq_exporter
).
to
receive
(
:enabled
).
and_return
(
false
)
end
describe
'#start'
do
it
"doesn't start"
do
expect
(
exporter
.
start
).
to
be_nil
expect
{
exporter
.
start
}.
not_to
change
{
exporter
.
thread?
}
expect
(
server
).
not_to
have_received
(
:start
)
end
end
describe
'#stop'
do
it
"doesn't shutdown"
do
expect
{
exporter
.
stop
}.
not_to
change
{
exporter
.
thread?
}
expect
(
server
).
not_to
have_received
(
:shutdown
)
end
end
end
end
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment