diff --git a/files/gitlab-config-template/gitlab.rb.template b/files/gitlab-config-template/gitlab.rb.template index 5f38c5accda923aeb49572e10cc6a26e49aa0899_ZmlsZXMvZ2l0bGFiLWNvbmZpZy10ZW1wbGF0ZS9naXRsYWIucmIudGVtcGxhdGU=..f96de8ed698b5aa108e6756efe8fd2965e978950_ZmlsZXMvZ2l0bGFiLWNvbmZpZy10ZW1wbGF0ZS9naXRsYWIucmIudGVtcGxhdGU= 100644 --- a/files/gitlab-config-template/gitlab.rb.template +++ b/files/gitlab-config-template/gitlab.rb.template @@ -3024,12 +3024,7 @@ # } # } # } -# consul['watchers'] = { -# 'postgresql' => { -# enable: false, -# handler: 'failover_pgbouncer' -# } -# } +# consul['watchers'] = [] # # consul['custom_config_dir'] = '/path/to/service/configs/directory' # diff --git a/files/gitlab-cookbooks/consul/attributes/watchers.rb b/files/gitlab-cookbooks/consul/attributes/watchers.rb index 5f38c5accda923aeb49572e10cc6a26e49aa0899_ZmlsZXMvZ2l0bGFiLWNvb2tib29rcy9jb25zdWwvYXR0cmlidXRlcy93YXRjaGVycy5yYg==..f96de8ed698b5aa108e6756efe8fd2965e978950_ZmlsZXMvZ2l0bGFiLWNvb2tib29rcy9jb25zdWwvYXR0cmlidXRlcy93YXRjaGVycy5yYg== 100644 --- a/files/gitlab-cookbooks/consul/attributes/watchers.rb +++ b/files/gitlab-cookbooks/consul/attributes/watchers.rb @@ -1,6 +1,1 @@ default['consul']['watchers'] = [] -default['consul']['watcher_config'] = { - 'postgresql' => { - 'handler' => 'failover_pgbouncer' - } -} diff --git a/files/gitlab-cookbooks/consul/libraries/watch_helper.rb b/files/gitlab-cookbooks/consul/libraries/watch_helper.rb index 5f38c5accda923aeb49572e10cc6a26e49aa0899_ZmlsZXMvZ2l0bGFiLWNvb2tib29rcy9jb25zdWwvbGlicmFyaWVzL3dhdGNoX2hlbHBlci5yYg==..f96de8ed698b5aa108e6756efe8fd2965e978950_ZmlsZXMvZ2l0bGFiLWNvb2tib29rcy9jb25zdWwvbGlicmFyaWVzL3dhdGNoX2hlbHBlci5yYg== 100644 --- a/files/gitlab-cookbooks/consul/libraries/watch_helper.rb +++ b/files/gitlab-cookbooks/consul/libraries/watch_helper.rb @@ -1,3 +1,6 @@ -class WatchHelper - attr_reader :node +require 'json' + +module WatchHelper + class Watcher + attr_reader :name, :handler_script, :handler_template, :type, :consul_config_file, :template_variables, :service_name @@ -3,5 +6,24 @@ - def initialize(node) - @node = node + def initialize(name = nil, handler_script = nil, handler_template = nil, type = nil, consul_watch_config_directory = nil, template_variables = {}) + @name = name + @handler_script = handler_script + @handler_template = handler_template + @type = type + @consul_config_file = "#{consul_watch_config_directory}/watcher_#{name}.json" + @service_name = "service:#{name}" + @template_variables = template_variables.merge({ "watcher_service_name" => @service_name }) + end + + def consul_config + { + watches: [ + { + type: @type, + service: @name, + args: [handler_script] + } + ] + }.to_json + end end @@ -6,11 +28,23 @@ end - def watcher_config(watcher) - { - watches: [ - { - type: 'service', - service: watcher, - args: ["#{@node['consul']['script_directory']}/#{watcher_handler(watcher)}"] - } + class WatcherConfig + attr_reader :node, :enabled_watchers, :standard_watchers + + def initialize(node) + @node = node + + # user configuration + @enabled_watchers = @node['consul']['watchers'] + @handler_directory = @node['consul']['script_directory'] + @consul_config_directory = @node['consul']['config_dir'] + + # library standards + @standard_watchers = [ + Watcher.new(name = node['consul']['internal']['postgresql_service_name'], + handler_script = "#{@handler_directory}/failover_postgresql_in_pgbouncer", + handler_template = 'failover_pgbouncer.erb', + type = 'service', + consul_config = @consul_config_directory, + template_variables = node['consul'].to_hash.merge({ 'database_name' => node['gitlab']['gitlab-rails']['db_database'] }) + ) ] @@ -16,4 +50,6 @@ ] - } - end + + # Backward compatibility if someone had actually made a customer + # watcher, even though it was never documented or supported + @user_watcher_configs = @node['consul']['watcher_config'] || {} @@ -19,5 +55,22 @@ - def watcher_handler(watcher) - node['consul']['watcher_config'][watcher]['handler'] + @user_watchers = [] + @user_watcher_configs.each do |watcher, config| + handler = config['handler'] + @user_watchers.push(Watcher.new(name = watcher, + handler_script = "#{@handler_directory}/#{handler}", + handler_template = "#{handler}.erb", + type = 'service', + consul_config = @consul_config_directory, + template_variables = node['consul'].to_hash + ) + ) + end + + @all_watchers = @standard_watchers + @user_watchers + end + + def watchers + @all_watchers.select { |watcher| @enabled_watchers.include? watcher.name } + end end end diff --git a/files/gitlab-cookbooks/consul/recipes/watchers.rb b/files/gitlab-cookbooks/consul/recipes/watchers.rb index 5f38c5accda923aeb49572e10cc6a26e49aa0899_ZmlsZXMvZ2l0bGFiLWNvb2tib29rcy9jb25zdWwvcmVjaXBlcy93YXRjaGVycy5yYg==..f96de8ed698b5aa108e6756efe8fd2965e978950_ZmlsZXMvZ2l0bGFiLWNvb2tib29rcy9jb25zdWwvcmVjaXBlcy93YXRjaGVycy5yYg== 100644 --- a/files/gitlab-cookbooks/consul/recipes/watchers.rb +++ b/files/gitlab-cookbooks/consul/recipes/watchers.rb @@ -14,5 +14,5 @@ # limitations under the License. # account_helper = AccountHelper.new(node) -watch_helper = WatchHelper.new(node) +watch_helper = WatchHelper::WatcherConfig.new(node) @@ -18,9 +18,7 @@ -node['consul']['watchers'].each do |watcher| - config = watch_helper.watcher_config(watcher) - - file "#{node['consul']['config_dir']}/watcher_#{watcher}.json" do - content config.to_json +watch_helper.watchers.each do |watcher| + file watcher.consul_config_file do + content watcher.consul_config owner account_helper.postgresql_user end @@ -24,12 +22,10 @@ owner account_helper.postgresql_user end - config[:watches].each do |watch| - template "#{node['consul']['script_directory']}/#{watch_helper.watcher_handler(watch[:service])}" do - source "watcher_scripts/#{node['consul']['watcher_config'][watch[:service]][:handler]}.erb" - variables node['consul'].to_hash - mode 0555 - end + template watcher.handler_script do + source "watcher_scripts/#{watcher.handler_template}" + variables watcher.template_variables + mode 0555 end end diff --git a/files/gitlab-cookbooks/consul/templates/default/watcher_scripts/failover_pgbouncer.erb b/files/gitlab-cookbooks/consul/templates/default/watcher_scripts/failover_pgbouncer.erb index 5f38c5accda923aeb49572e10cc6a26e49aa0899_ZmlsZXMvZ2l0bGFiLWNvb2tib29rcy9jb25zdWwvdGVtcGxhdGVzL2RlZmF1bHQvd2F0Y2hlcl9zY3JpcHRzL2ZhaWxvdmVyX3BnYm91bmNlci5lcmI=..f96de8ed698b5aa108e6756efe8fd2965e978950_ZmlsZXMvZ2l0bGFiLWNvb2tib29rcy9jb25zdWwvdGVtcGxhdGVzL2RlZmF1bHQvd2F0Y2hlcl9zY3JpcHRzL2ZhaWxvdmVyX3BnYm91bmNlci5lcmI= 100644 --- a/files/gitlab-cookbooks/consul/templates/default/watcher_scripts/failover_pgbouncer.erb +++ b/files/gitlab-cookbooks/consul/templates/default/watcher_scripts/failover_pgbouncer.erb @@ -27,7 +27,7 @@ end service_data = FailoverHelper::ServiceData.new -service_data.service_name = "service:postgresql" +service_data.service_name = "<%= @watcher_service_name %>" service_data.check_field = "Status" service_data.leader_value = "passing" @@ -36,7 +36,7 @@ new_primary = failover.primary_node_address @log.info("Found primary: #{new_primary}") - Kernel.exit run_command("gitlab-ctl pgb-notify --newhost #{new_primary} --user pgbouncer --hostuser gitlab-consul") + Kernel.exit run_command("gitlab-ctl pgb-notify --pg-database <%= @database_name %> --newhost #{new_primary} --user pgbouncer --hostuser gitlab-consul") rescue JSON::ParserError => jparser_error @log.error(jparser_error.message) Kernel.exit 2 @@ -46,7 +46,7 @@ @log.error(" Node: #{primary.name}") end @log.error('Stopping pgbouncer to prevent issues. Once the error is cleared, consul will reload pgbouncer') - results = run_command('gitlab-ctl pgb-kill --pg-database gitlabhq_production --user pgbouncer --hostuser gitlab-consul') + results = run_command('gitlab-ctl pgb-kill --pg-database <%= @database_name %> --user pgbouncer --hostuser gitlab-consul') Kernel.exit 3 + results rescue FailoverHelper::PrimaryMissing => no_primary_error @log.error(no_primary_error.message) diff --git a/spec/chef/cookbooks/consul/libraries/watch_helper_spec.rb b/spec/chef/cookbooks/consul/libraries/watch_helper_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..f96de8ed698b5aa108e6756efe8fd2965e978950_c3BlYy9jaGVmL2Nvb2tib29rcy9jb25zdWwvbGlicmFyaWVzL3dhdGNoX2hlbHBlcl9zcGVjLnJi --- /dev/null +++ b/spec/chef/cookbooks/consul/libraries/watch_helper_spec.rb @@ -0,0 +1,15 @@ +require 'chef_helper' + +RSpec.describe WatchHelper::WatcherConfig do + cached(:chef_run) { converge_config } + let(:node) { chef_run.node } + subject { described_class.new(node) } + + let(:standard_watcher_names) { ['postgresql'] } + + context 'when initialized' do + it 'should have standard watchers defined' do + expect(subject.standard_watchers.map(&:name)).to match_array(standard_watcher_names) + end + end +end diff --git a/spec/chef/cookbooks/consul/recipes/consul_watcher_spec.rb b/spec/chef/cookbooks/consul/recipes/consul_watcher_spec.rb index 5f38c5accda923aeb49572e10cc6a26e49aa0899_c3BlYy9jaGVmL2Nvb2tib29rcy9jb25zdWwvcmVjaXBlcy9jb25zdWxfd2F0Y2hlcl9zcGVjLnJi..f96de8ed698b5aa108e6756efe8fd2965e978950_c3BlYy9jaGVmL2Nvb2tib29rcy9jb25zdWwvcmVjaXBlcy9jb25zdWxfd2F0Y2hlcl9zcGVjLnJi 100644 --- a/spec/chef/cookbooks/consul/recipes/consul_watcher_spec.rb +++ b/spec/chef/cookbooks/consul/recipes/consul_watcher_spec.rb @@ -3,7 +3,7 @@ RSpec.describe 'consul::watchers' do let(:chef_run) { ChefSpec::SoloRunner.converge('gitlab-ee::default') } let(:watcher_conf) { '/var/opt/gitlab/consul/config.d/watcher_postgresql.json' } - let(:watcher_check) { '/var/opt/gitlab/consul/scripts/failover_pgbouncer' } + let(:watcher_check) { '/var/opt/gitlab/consul/scripts/failover_postgresql_in_pgbouncer' } before do allow(Gitlab).to receive(:[]).and_call_original