diff --git a/changelogs/unreleased/gitaly-zero-downtime-deployment.yml b/changelogs/unreleased/gitaly-zero-downtime-deployment.yml new file mode 100644 index 0000000000000000000000000000000000000000..8d1c74a4871a68ac40503f97247def4f4d297eab_Y2hhbmdlbG9ncy91bnJlbGVhc2VkL2dpdGFseS16ZXJvLWRvd250aW1lLWRlcGxveW1lbnQueW1s --- /dev/null +++ b/changelogs/unreleased/gitaly-zero-downtime-deployment.yml @@ -0,0 +1,5 @@ +--- +title: Add gitaly graceful restart wrapper +merge_request: 3116 +author: +type: added diff --git a/files/gitlab-config-template/gitlab.rb.template b/files/gitlab-config-template/gitlab.rb.template index bd29931932ffd18283c5ffc34a403355f8d987b0_ZmlsZXMvZ2l0bGFiLWNvbmZpZy10ZW1wbGF0ZS9naXRsYWIucmIudGVtcGxhdGU=..8d1c74a4871a68ac40503f97247def4f4d297eab_ZmlsZXMvZ2l0bGFiLWNvbmZpZy10ZW1wbGF0ZS9naXRsYWIucmIudGVtcGxhdGU= 100644 --- a/files/gitlab-config-template/gitlab.rb.template +++ b/files/gitlab-config-template/gitlab.rb.template @@ -1628,6 +1628,7 @@ # gitaly['prometheus_grpc_latency_buckets'] = "[0.001, 0.005, 0.025, 0.1, 0.5, 1.0, 10.0, 30.0, 60.0, 300.0, 1500.0]" # gitaly['auth_token'] = '<secret>' # gitaly['auth_transitioning'] = false # When true, auth is logged to Prometheus but NOT enforced +# gitaly['graceful_restart_timeout'] = '1m' # Grace time for a gitaly process to finish ongoing requests # gitaly['ruby_max_rss'] = 300000000 # RSS threshold in bytes for triggering a gitaly-ruby restart # gitaly['ruby_graceful_restart_timeout'] = '10m' # Grace time for a gitaly-ruby process to finish ongoing requests # gitaly['ruby_restart_delay'] = '5m' # Period of sustained high RSS that needs to be observed before restarting gitaly-ruby diff --git a/files/gitlab-cookbooks/gitaly/attributes/default.rb b/files/gitlab-cookbooks/gitaly/attributes/default.rb index bd29931932ffd18283c5ffc34a403355f8d987b0_ZmlsZXMvZ2l0bGFiLWNvb2tib29rcy9naXRhbHkvYXR0cmlidXRlcy9kZWZhdWx0LnJi..8d1c74a4871a68ac40503f97247def4f4d297eab_ZmlsZXMvZ2l0bGFiLWNvb2tib29rcy9naXRhbHkvYXR0cmlidXRlcy9kZWZhdWx0LnJi 100644 --- a/files/gitlab-cookbooks/gitaly/attributes/default.rb +++ b/files/gitlab-cookbooks/gitaly/attributes/default.rb @@ -3,6 +3,7 @@ default['gitaly']['dir'] = "/var/opt/gitlab/gitaly" default['gitaly']['log_directory'] = "/var/log/gitlab/gitaly" default['gitaly']['env_directory'] = "/opt/gitlab/etc/gitaly/env" +default['gitaly']['graceful_restart_timeout'] = nil # default['gitaly']['env'] is set in ../recipes/enable.rb default['gitaly']['bin_path'] = "/opt/gitlab/embedded/bin/gitaly" default['gitaly']['socket_path'] = "#{node['gitaly']['dir']}/gitaly.socket" diff --git a/files/gitlab-cookbooks/gitaly/recipes/enable.rb b/files/gitlab-cookbooks/gitaly/recipes/enable.rb index bd29931932ffd18283c5ffc34a403355f8d987b0_ZmlsZXMvZ2l0bGFiLWNvb2tib29rcy9naXRhbHkvcmVjaXBlcy9lbmFibGUucmI=..8d1c74a4871a68ac40503f97247def4f4d297eab_ZmlsZXMvZ2l0bGFiLWNvb2tib29rcy9naXRhbHkvcmVjaXBlcy9lbmFibGUucmI= 100644 --- a/files/gitlab-cookbooks/gitaly/recipes/enable.rb +++ b/files/gitlab-cookbooks/gitaly/recipes/enable.rb @@ -20,6 +20,10 @@ log_directory = node['gitaly']['log_directory'] env_directory = node['gitaly']['env_directory'] config_path = File.join(working_dir, "config.toml") +gitaly_path = node['gitaly']['bin_path'] +wrapper_path = "#{gitaly_path}-wrapper" +pid_file = File.join(working_dir, "gitaly.pid") +json_logging = node['gitaly']['logging_format'].eql?('json') directory working_dir do owner account_helper.gitlab_user @@ -47,7 +51,10 @@ # Charlock Holmes and libicu will report U_FILE_ACCESS_ERROR if this is not set to the right path # See https://gitlab.com/gitlab-org/gitlab-ce/issues/17415#note_13868167 'ICU_DATA' => "#{node['package']['install-dir']}/embedded/share/icu/current", - 'SSL_CERT_DIR' => "#{node['package']['install-dir']}/embedded/ssl/certs/" + 'SSL_CERT_DIR' => "#{node['package']['install-dir']}/embedded/ssl/certs/", + # wrapper script parameters + 'GITALY_PID_FILE' => pid_file, + 'WRAPPER_JSON_LOGGING' => json_logging.to_s } env_dir env_directory do @@ -73,7 +80,7 @@ group account_helper.gitlab_group mode "0640" variables node['gitaly'].to_hash - notifies :restart, "service[gitaly]" + notifies :hup, "runit_service[gitaly]" end runit_service 'gitaly' do @@ -83,5 +90,6 @@ groupname: account_helper.gitlab_group, working_dir: working_dir, env_dir: env_directory, - bin_path: node['gitaly']['bin_path'], + bin_path: gitaly_path, + wrapper_path: wrapper_path, config_path: config_path, @@ -87,5 +95,6 @@ config_path: config_path, - log_directory: log_directory + log_directory: log_directory, + json_logging: json_logging }.merge(params)) log_options node['gitlab']['logging'].to_hash.merge(node['gitaly'].to_hash) end @@ -98,5 +107,5 @@ file File.join(working_dir, "VERSION") do content VersionHelper.version("/opt/gitlab/embedded/bin/gitaly --version") - notifies :restart, "service[gitaly]" + notifies :hup, "runit_service[gitaly]" end diff --git a/files/gitlab-cookbooks/gitaly/templates/default/gitaly-config.toml.erb b/files/gitlab-cookbooks/gitaly/templates/default/gitaly-config.toml.erb index bd29931932ffd18283c5ffc34a403355f8d987b0_ZmlsZXMvZ2l0bGFiLWNvb2tib29rcy9naXRhbHkvdGVtcGxhdGVzL2RlZmF1bHQvZ2l0YWx5LWNvbmZpZy50b21sLmVyYg==..8d1c74a4871a68ac40503f97247def4f4d297eab_ZmlsZXMvZ2l0bGFiLWNvb2tib29rcy9naXRhbHkvdGVtcGxhdGVzL2RlZmF1bHQvZ2l0YWx5LWNvbmZpZy50b21sLmVyYg== 100644 --- a/files/gitlab-cookbooks/gitaly/templates/default/gitaly-config.toml.erb +++ b/files/gitlab-cookbooks/gitaly/templates/default/gitaly-config.toml.erb @@ -6,6 +6,9 @@ socket_path = '<%= @socket_path %>' bin_dir = '/opt/gitlab/embedded/bin' +<% if @graceful_restart_timeout %> +graceful_restart_timeout = '<%= @graceful_restart_timeout %>' +<% end %> <% if @listen_addr %> # Optional: listen on a TCP socket. This is insecure (no authentication) diff --git a/files/gitlab-cookbooks/gitaly/templates/default/sv-gitaly-log-run.erb b/files/gitlab-cookbooks/gitaly/templates/default/sv-gitaly-log-run.erb index bd29931932ffd18283c5ffc34a403355f8d987b0_ZmlsZXMvZ2l0bGFiLWNvb2tib29rcy9naXRhbHkvdGVtcGxhdGVzL2RlZmF1bHQvc3YtZ2l0YWx5LWxvZy1ydW4uZXJi..8d1c74a4871a68ac40503f97247def4f4d297eab_ZmlsZXMvZ2l0bGFiLWNvb2tib29rcy9naXRhbHkvdGVtcGxhdGVzL2RlZmF1bHQvc3YtZ2l0YWx5LWxvZy1ydW4uZXJi 100644 --- a/files/gitlab-cookbooks/gitaly/templates/default/sv-gitaly-log-run.erb +++ b/files/gitlab-cookbooks/gitaly/templates/default/sv-gitaly-log-run.erb @@ -1,2 +1,2 @@ #!/bin/sh -exec svlogd <% unless node['gitaly']['logging_format'].eql?('json') %>-tt <% end %><%= @options[:log_directory] %> +exec svlogd <% unless @options[:json_logging] %>-tt <% end %><%= @options[:log_directory] %> diff --git a/files/gitlab-cookbooks/gitaly/templates/default/sv-gitaly-run.erb b/files/gitlab-cookbooks/gitaly/templates/default/sv-gitaly-run.erb index bd29931932ffd18283c5ffc34a403355f8d987b0_ZmlsZXMvZ2l0bGFiLWNvb2tib29rcy9naXRhbHkvdGVtcGxhdGVzL2RlZmF1bHQvc3YtZ2l0YWx5LXJ1bi5lcmI=..8d1c74a4871a68ac40503f97247def4f4d297eab_ZmlsZXMvZ2l0bGFiLWNvb2tib29rcy9naXRhbHkvdGVtcGxhdGVzL2RlZmF1bHQvc3YtZ2l0YWx5LXJ1bi5lcmI= 100644 --- a/files/gitlab-cookbooks/gitaly/templates/default/sv-gitaly-run.erb +++ b/files/gitlab-cookbooks/gitaly/templates/default/sv-gitaly-run.erb @@ -11,4 +11,4 @@ exec chpst -e <%= @options[:env_dir] %> -P \ -U <%= @options[:user] %>:<%= @options[:groupname] %> \ -u <%= @options[:user] %>:<%= @options[:groupname] %> \ - <%= @options[:bin_path] %> <%= @options[:config_path] %> + <%= @options[:wrapper_path] %> <%= @options[:bin_path] %> <%= @options[:config_path] %> diff --git a/lib/gitlab/license/analyzer.rb b/lib/gitlab/license/analyzer.rb index bd29931932ffd18283c5ffc34a403355f8d987b0_bGliL2dpdGxhYi9saWNlbnNlL2FuYWx5emVyLnJi..8d1c74a4871a68ac40503f97247def4f4d297eab_bGliL2dpdGxhYi9saWNlbnNlL2FuYWx5emVyLnJi 100644 --- a/lib/gitlab/license/analyzer.rb +++ b/lib/gitlab/license/analyzer.rb @@ -34,6 +34,7 @@ 'object-component', # MIT Licensed - https://github.com/component/object/blob/master/LICENSE 'select2', # MIT Licensed - https://github.com/select2/select2/blob/master/LICENSE.md 'exiftool', # License similar to Perl, which is under either GPL v1 or Artistic license - https://www.sno.phy.queensu.ca/~phil/exiftool/#license + 'github.com/cloudflare/tableflip', # BSD-3-Clause Licensed - https://github.com/cloudflare/tableflip/blob/master/LICENSE ] # readline is GPL licensed and its use was not mere aggregation. Hence it is # blacklisted. diff --git a/spec/chef/recipes/gitaly_spec.rb b/spec/chef/recipes/gitaly_spec.rb index bd29931932ffd18283c5ffc34a403355f8d987b0_c3BlYy9jaGVmL3JlY2lwZXMvZ2l0YWx5X3NwZWMucmI=..8d1c74a4871a68ac40503f97247def4f4d297eab_c3BlYy9jaGVmL3JlY2lwZXMvZ2l0YWx5X3NwZWMucmI= 100644 --- a/spec/chef/recipes/gitaly_spec.rb +++ b/spec/chef/recipes/gitaly_spec.rb @@ -20,6 +20,7 @@ let(:auth_token) { '123secret456' } let(:auth_transitioning) { true } let(:ruby_max_rss) { 1000000 } + let(:graceful_restart_timeout) { '20m' } let(:ruby_graceful_restart_timeout) { '30m' } let(:ruby_restart_delay) { '10m' } let(:ruby_num_workers) { 5 } @@ -31,6 +32,8 @@ 'PATH' => '/opt/gitlab/bin:/opt/gitlab/embedded/bin:/bin:/usr/bin', 'ICU_DATA' => '/opt/gitlab/embedded/share/icu/current', 'PYTHONPATH' => '/opt/gitlab/embedded/lib/python3.4/site-packages', + 'WRAPPER_JSON_LOGGING' => 'false', + "GITALY_PID_FILE" => '/var/opt/gitlab/gitaly/gitaly.pid', } end @@ -118,6 +121,7 @@ prometheus_grpc_latency_buckets: prometheus_grpc_latency_buckets, auth_token: auth_token, auth_transitioning: auth_transitioning, + graceful_restart_timeout: graceful_restart_timeout, ruby_max_rss: ruby_max_rss, ruby_graceful_restart_timeout: ruby_graceful_restart_timeout, ruby_restart_delay: ruby_restart_delay, @@ -140,6 +144,8 @@ expect(chef_run).to render_file(config_path) .with_content("listen_addr = 'localhost:7777'") expect(chef_run).to render_file(config_path) + .with_content("graceful_restart_timeout = '#{graceful_restart_timeout}'") + expect(chef_run).to render_file(config_path) .with_content { |content| expect(content).to include("tls_listen_addr = 'localhost:8888'") expect(content).to include("certificate_path = '/path/to/cert.pem'")