Commit 7dbd8e2b authored by Grzegorz Bizon's avatar Grzegorz Bizon
Browse files

Merge branch '41771-reduce-cardinality-of-metrics' into 'master'

Reduce the cardinality of GitLab metrics

Closes #41771

See merge request gitlab-org/gitlab-ce!16443
---
title: Reduce the number of Prometheus metrics
merge_request: 16443
author:
type: performance
......@@ -6,6 +6,7 @@
module Gitlab
module GitalyClient
include Gitlab::Metrics::Methods
module MigrationStatus
DISABLED = 1
OPT_IN = 2
......@@ -33,8 +34,6 @@ def initialize(call_site, invocation_count, max_call_stack, most_invoked_stack)
CLIENT_NAME = (Sidekiq.server? ? 'gitlab-sidekiq' : 'gitlab-web').freeze
MUTEX = Mutex.new
METRICS_MUTEX = Mutex.new
private_constant :MUTEX, :METRICS_MUTEX
class << self
attr_accessor :query_time
......@@ -42,28 +41,14 @@ class << self
self.query_time = 0
def self.migrate_histogram
@migrate_histogram ||=
METRICS_MUTEX.synchronize do
# If a thread was blocked on the mutex, the value was set already
return @migrate_histogram if @migrate_histogram
Gitlab::Metrics.histogram(:gitaly_migrate_call_duration_seconds,
"Gitaly migration call execution timings",
gitaly_enabled: nil, feature: nil)
end
define_histogram :gitaly_migrate_call_duration_seconds do
docstring "Gitaly migration call execution timings"
base_labels gitaly_enabled: nil, feature: nil
end
def self.gitaly_call_histogram
@gitaly_call_histogram ||=
METRICS_MUTEX.synchronize do
# If a thread was blocked on the mutex, the value was set already
return @gitaly_call_histogram if @gitaly_call_histogram
Gitlab::Metrics.histogram(:gitaly_controller_action_duration_seconds,
"Gitaly endpoint histogram by controller and action combination",
Gitlab::Metrics::Transaction::BASE_LABELS.merge(gitaly_service: nil, rpc: nil))
end
define_histogram :gitaly_controller_action_duration_seconds do
docstring "Gitaly endpoint histogram by controller and action combination"
base_labels Gitlab::Metrics::Transaction::BASE_LABELS.merge(gitaly_service: nil, rpc: nil)
end
def self.stub(name, storage)
......@@ -145,7 +130,7 @@ def self.call(storage, service, rpc, request, remote_storage: nil, timeout: nil)
# Keep track, seperately, for the performance bar
self.query_time += duration
gitaly_call_histogram.observe(
gitaly_controller_action_duration_seconds.observe(
current_transaction_labels.merge(gitaly_service: service.to_s, rpc: rpc.to_s),
duration)
end
......@@ -247,7 +232,7 @@ def self.migrate(feature, status: MigrationStatus::OPT_IN)
yield is_enabled
ensure
total_time = Gitlab::Metrics::System.monotonic_time - start
migrate_histogram.observe({ gitaly_enabled: is_enabled, feature: feature }, total_time)
gitaly_migrate_call_duration_seconds.observe({ gitaly_enabled: is_enabled, feature: feature }, total_time)
feature_stack.shift
Thread.current[:gitaly_feature_stack] = nil if feature_stack.empty?
end
......
module Gitlab
module Metrics
extend Gitlab::Metrics::InfluxDb
extend Gitlab::Metrics::Prometheus
include Gitlab::Metrics::InfluxDb
include Gitlab::Metrics::Prometheus
def self.enabled?
influx_metrics_enabled? || prometheus_metrics_enabled?
......
module Gitlab
module Metrics
module InfluxDb
include Gitlab::CurrentSettings
extend self
extend ActiveSupport::Concern
include Gitlab::Metrics::Methods
EXECUTION_MEASUREMENT_BUCKETS = [0.001, 0.01, 0.1, 1].freeze
MUTEX = Mutex.new
private_constant :MUTEX
def influx_metrics_enabled?
settings[:enabled] || false
end
class_methods do
def influx_metrics_enabled?
settings[:enabled] || false
end
# Prometheus histogram buckets used for arbitrary code measurements
EXECUTION_MEASUREMENT_BUCKETS = [0.001, 0.002, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1].freeze
RAILS_ROOT = Rails.root.to_s
METRICS_ROOT = Rails.root.join('lib', 'gitlab', 'metrics').to_s
PATH_REGEX = /^#{RAILS_ROOT}\/?/
def settings
@settings ||= {
enabled: current_application_settings[:metrics_enabled],
pool_size: current_application_settings[:metrics_pool_size],
timeout: current_application_settings[:metrics_timeout],
method_call_threshold: current_application_settings[:metrics_method_call_threshold],
host: current_application_settings[:metrics_host],
port: current_application_settings[:metrics_port],
sample_interval: current_application_settings[:metrics_sample_interval] || 15,
packet_size: current_application_settings[:metrics_packet_size] || 1
}
end
# Prometheus histogram buckets used for arbitrary code measurements
def settings
@settings ||= begin
current_settings = Gitlab::CurrentSettings.current_application_settings
{
enabled: current_settings[:metrics_enabled],
pool_size: current_settings[:metrics_pool_size],
timeout: current_settings[:metrics_timeout],
method_call_threshold: current_settings[:metrics_method_call_threshold],
host: current_settings[:metrics_host],
port: current_settings[:metrics_port],
sample_interval: current_settings[:metrics_sample_interval] || 15,
packet_size: current_settings[:metrics_packet_size] || 1
}
end
end
def mri?
RUBY_ENGINE == 'ruby'
end
def mri?
RUBY_ENGINE == 'ruby'
end
def method_call_threshold
# This is memoized since this method is called for every instrumented
# method. Loading data from an external cache on every method call slows
# things down too much.
# in milliseconds
@method_call_threshold ||= settings[:method_call_threshold]
end
def method_call_threshold
# This is memoized since this method is called for every instrumented
# method. Loading data from an external cache on every method call slows
# things down too much.
# in milliseconds
@method_call_threshold ||= settings[:method_call_threshold]
end
def submit_metrics(metrics)
prepared = prepare_metrics(metrics)
def submit_metrics(metrics)
prepared = prepare_metrics(metrics)
pool&.with do |connection|
prepared.each_slice(settings[:packet_size]) do |slice|
begin
connection.write_points(slice)
rescue StandardError
pool&.with do |connection|
prepared.each_slice(settings[:packet_size]) do |slice|
begin
connection.write_points(slice)
rescue StandardError
end
end
end
rescue Errno::EADDRNOTAVAIL, SocketError => ex
Gitlab::EnvironmentLogger.error('Cannot resolve InfluxDB address. GitLab Performance Monitoring will not work.')
Gitlab::EnvironmentLogger.error(ex)
end
rescue Errno::EADDRNOTAVAIL, SocketError => ex
Gitlab::EnvironmentLogger.error('Cannot resolve InfluxDB address. GitLab Performance Monitoring will not work.')
Gitlab::EnvironmentLogger.error(ex)
end
def prepare_metrics(metrics)
metrics.map do |hash|
new_hash = hash.symbolize_keys
def prepare_metrics(metrics)
metrics.map do |hash|
new_hash = hash.symbolize_keys
new_hash[:tags].each do |key, value|
if value.blank?
new_hash[:tags].delete(key)
else
new_hash[:tags][key] = escape_value(value)
new_hash[:tags].each do |key, value|
if value.blank?
new_hash[:tags].delete(key)
else
new_hash[:tags][key] = escape_value(value)
end
end
new_hash
end
end
new_hash
def escape_value(value)
value.to_s.gsub('=', '\\=')
end
end
def escape_value(value)
value.to_s.gsub('=', '\\=')
end
# Measures the execution time of a block.
#
# Example:
#
# Gitlab::Metrics.measure(:find_by_username_duration) do
# User.find_by_username(some_username)
# end
#
# name - The name of the field to store the execution time in.
#
# Returns the value yielded by the supplied block.
def measure(name)
trans = current_transaction
return yield unless trans
real_start = Time.now.to_f
cpu_start = System.cpu_time
retval = yield
cpu_stop = System.cpu_time
real_stop = Time.now.to_f
real_time = (real_stop - real_start)
cpu_time = cpu_stop - cpu_start
real_duration_seconds = fetch_histogram("gitlab_#{name}_real_duration_seconds".to_sym) do
docstring "Measure #{name}"
base_labels Transaction::BASE_LABELS
buckets EXECUTION_MEASUREMENT_BUCKETS
end
# Measures the execution time of a block.
#
# Example:
#
# Gitlab::Metrics.measure(:find_by_username_duration) do
# User.find_by_username(some_username)
# end
#
# name - The name of the field to store the execution time in.
#
# Returns the value yielded by the supplied block.
def measure(name)
trans = current_transaction
return yield unless trans
real_start = Time.now.to_f
cpu_start = System.cpu_time
retval = yield
cpu_stop = System.cpu_time
real_stop = Time.now.to_f
real_time = (real_stop - real_start)
cpu_time = cpu_stop - cpu_start
Gitlab::Metrics.histogram("gitlab_#{name}_real_duration_seconds".to_sym,
"Measure #{name}",
Transaction::BASE_LABELS,
EXECUTION_MEASUREMENT_BUCKETS)
.observe(trans.labels, real_time)
Gitlab::Metrics.histogram("gitlab_#{name}_cpu_duration_seconds".to_sym,
"Measure #{name}",
Transaction::BASE_LABELS,
EXECUTION_MEASUREMENT_BUCKETS)
.observe(trans.labels, cpu_time / 1000.0)
# InfluxDB stores the _real_time time values as milliseconds
trans.increment("#{name}_real_time", real_time * 1000, false)
trans.increment("#{name}_cpu_time", cpu_time, false)
trans.increment("#{name}_call_count", 1, false)
retval
end
real_duration_seconds.observe(trans.labels, real_time)
# Sets the action of the current transaction (if any)
#
# action - The name of the action.
def action=(action)
trans = current_transaction
cpu_duration_seconds = fetch_histogram("gitlab_#{name}_cpu_duration_seconds".to_sym) do
docstring "Measure #{name}"
base_labels Transaction::BASE_LABELS
buckets EXECUTION_MEASUREMENT_BUCKETS
with_feature "prometheus_metrics_measure_#{name}_cpu_duration"
end
cpu_duration_seconds.observe(trans.labels, cpu_time)
trans&.action = action
end
# InfluxDB stores the _real_time and _cpu_time time values as milliseconds
trans.increment("#{name}_real_time", real_time.in_milliseconds, false)
trans.increment("#{name}_cpu_time", cpu_time.in_milliseconds, false)
trans.increment("#{name}_call_count", 1, false)
# Tracks an event.
#
# See `Gitlab::Metrics::Transaction#add_event` for more details.
def add_event(*args)
trans = current_transaction
retval
end
trans&.add_event(*args)
end
# Sets the action of the current transaction (if any)
#
# action - The name of the action.
def action=(action)
trans = current_transaction
# Returns the prefix to use for the name of a series.
def series_prefix
@series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_'
end
trans&.action = action
end
# Allow access from other metrics related middlewares
def current_transaction
Transaction.current
end
# Tracks an event.
#
# See `Gitlab::Metrics::Transaction#add_event` for more details.
def add_event(*args)
trans = current_transaction
# When enabled this should be set before being used as the usual pattern
# "@foo ||= bar" is _not_ thread-safe.
# rubocop:disable Gitlab/ModuleWithInstanceVariables
def pool
if influx_metrics_enabled?
if @pool.nil?
MUTEX.synchronize do
@pool ||= ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do
host = settings[:host]
port = settings[:port]
InfluxDB::Client
.new(udp: { host: host, port: port })
trans&.add_event(*args)
end
# Returns the prefix to use for the name of a series.
def series_prefix
@series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_'
end
# Allow access from other metrics related middlewares
def current_transaction
Transaction.current
end
# When enabled this should be set before being used as the usual pattern
# "@foo ||= bar" is _not_ thread-safe.
# rubocop:disable Gitlab/ModuleWithInstanceVariables
def pool
if influx_metrics_enabled?
if @pool.nil?
MUTEX.synchronize do
@pool ||= ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do
host = settings[:host]
port = settings[:port]
InfluxDB::Client
.new(udp: { host: host, port: port })
end
end
end
end
@pool
@pool
end
end
# rubocop:enable Gitlab/ModuleWithInstanceVariables
end
# rubocop:enable Gitlab/ModuleWithInstanceVariables
end
end
end
......@@ -4,26 +4,15 @@ module Gitlab
module Metrics
# Class for tracking timing information about method calls
class MethodCall
@@measurement_enabled_cache = Concurrent::AtomicBoolean.new(false)
@@measurement_enabled_cache_expires_at = Concurrent::AtomicReference.new(Time.now.to_i)
MUTEX = Mutex.new
include Gitlab::Metrics::Methods
BASE_LABELS = { module: nil, method: nil }.freeze
attr_reader :real_time, :cpu_time, :call_count, :labels
def self.call_duration_histogram
return @call_duration_histogram if @call_duration_histogram
MUTEX.synchronize do
@call_duration_histogram ||= Gitlab::Metrics.histogram(
:gitlab_method_call_duration_seconds,
'Method calls real duration',
Transaction::BASE_LABELS.merge(BASE_LABELS),
[0.01, 0.05, 0.1, 0.5, 1])
end
end
def self.measurement_enabled_cache_expires_at
@@measurement_enabled_cache_expires_at
define_histogram :gitlab_method_call_duration_seconds do
docstring 'Method calls real duration'
base_labels Transaction::BASE_LABELS.merge(BASE_LABELS)
buckets [0.01, 0.05, 0.1, 0.5, 1]
with_feature :prometheus_metrics_method_instrumentation
end
# name - The full name of the method (including namespace) such as
......@@ -53,8 +42,8 @@ def measure
@cpu_time += cpu_time
@call_count += 1
if call_measurement_enabled? && above_threshold?
self.class.call_duration_histogram.observe(@transaction.labels.merge(labels), real_time)
if above_threshold?
self.class.gitlab_method_call_duration_seconds.observe(@transaction.labels.merge(labels), real_time)
end
retval
......@@ -78,17 +67,6 @@ def to_metric
def above_threshold?
real_time.in_milliseconds >= Metrics.method_call_threshold
end
def call_measurement_enabled?
expires_at = @@measurement_enabled_cache_expires_at.value
if expires_at < Time.now.to_i
if @@measurement_enabled_cache_expires_at.compare_and_set(expires_at, 1.minute.from_now.to_i)
@@measurement_enabled_cache.value = Feature.get(:prometheus_metrics_method_instrumentation).enabled?
end
end
@@measurement_enabled_cache.value
end
end
end
end
# rubocop:disable Style/ClassVars
module Gitlab
module Metrics
module Methods
extend ActiveSupport::Concern
included do
@@_metric_provider_mutex ||= Mutex.new
@@_metrics_provider_cache = {}
end
class_methods do
def reload_metric!(name)
@@_metrics_provider_cache.delete(name)
end
private
def define_metric(type, name, opts = {}, &block)
if respond_to?(name)
raise ArgumentError, "method #{name} already exists"
end
define_singleton_method(name) do
# inlining fetch_metric method to avoid method call overhead when instrumenting hot spots
@@_metrics_provider_cache[name] || init_metric(type, name, opts, &block)
end
end
def fetch_metric(type, name, opts = {}, &block)
@@_metrics_provider_cache[name] || init_metric(type, name, opts, &block)
end
def init_metric(type, name, opts = {}, &block)
options = MetricOptions.new(opts)
options.evaluate(&block)
if disabled_by_feature(options)
synchronized_cache_fill(name) { NullMetric.instance }
else
synchronized_cache_fill(name) { build_metric!(type, name, options) }
end
end
def synchronized_cache_fill(key)
@@_metric_provider_mutex.synchronize do
@@_metrics_provider_cache[key] ||= yield
end
end
def disabled_by_feature(options)
options.with_feature && !Feature.get(options.with_feature).enabled?
end
def build_metric!(type, name, options)
case type
when :gauge
Gitlab::Metrics.gauge(name, options.docstring, options.base_labels, options.multiprocess_mode)
when :counter
Gitlab::Metrics.counter(name, options.docstring, options.base_labels)
when :histogram
Gitlab::Metrics.histogram(name, options.docstring, options.base_labels, options.buckets)
when :summary
raise NotImplementedError, "summary metrics are not currently supported"
else
raise ArgumentError, "uknown metric type #{type}"
end
end
# Fetch and/or initialize counter metric
# @param [Symbol] name
# @param [Hash] opts
def fetch_counter(name, opts = {}, &block)
fetch_metric(:counter, name, opts, &block)
end
# Fetch and/or initialize gauge metric
# @param [Symbol] name
# @param [Hash] opts
def fetch_gauge(name, opts = {}, &block)
fetch_metric(:gauge, name, opts, &block)
end
# Fetch and/or initialize histogram metric
# @param [Symbol] name
# @param [Hash] opts
def fetch_histogram(name, opts = {}, &block)
fetch_metric(:histogram, name, opts, &block)
end
# Fetch and/or initialize summary metric
# @param [Symbol] name
# @param [Hash] opts
def fetch_summary(name, opts = {}, &block)
fetch_metric(:summary, name, opts, &block)
end
# Define metric accessor method for a Counter
# @param [Symbol] name
# @param [Hash] opts
def define_counter(name, opts = {}, &block)
define_metric(:counter, name, opts, &block)
end
# Define metric accessor method for a Gauge
# @param [Symbol] name
# @param [Hash] opts
def define_gauge(name, opts = {}, &block)
define_metric(:gauge, name, opts, &block)
end
# Define metric accessor method for a Histogram
# @param [Symbol] name
# @param [Hash] opts
def define_histogram(name, opts = {}, &block)
define_metric(:histogram, name, opts, &block)
end
# Define metric accessor method for a Summary
# @param [Symbol] name
# @param [Hash] opts
def define_summary(name, opts = {}, &block)
define_metric(:summary, name, opts, &block)
end
end
end
end
end