diff --git a/config/software/consul.rb b/config/software/consul.rb index 94f38026430014c3bb13ca98be388c572e2264a0_Y29uZmlnL3NvZnR3YXJlL2NvbnN1bC5yYg==..abf9d8a85f8b3b0db2fb5a59a5b7e53668d0c8fc_Y29uZmlnL3NvZnR3YXJlL2NvbnN1bC5yYg== 100644 --- a/config/software/consul.rb +++ b/config/software/consul.rb @@ -16,6 +16,8 @@ name 'consul' +# This version should be keep in sync with consul versions in +# software/consul.rb and consul_helper.rb. version = Gitlab::Version.new('consul', 'v1.16.3') default_version version.print(false) diff --git a/deps.yml b/deps.yml index 94f38026430014c3bb13ca98be388c572e2264a0_ZGVwcy55bWw=..abf9d8a85f8b3b0db2fb5a59a5b7e53668d0c8fc_ZGVwcy55bWw= 100644 --- a/deps.yml +++ b/deps.yml @@ -79,6 +79,13 @@ - filename: config/software/consul.rb pattern: "version = Gitlab::Version.new\\('consul', 'v(\\S+)'\\)" range: '< 1.17.0' + - filename: files/gitlab-ctl-commands-ee/lib/consul_download.rb + pattern: "DEFAULT_VERSION = '(\\S+)'" + range: '< 1.17.0' + - filename: files/gitlab-cookbooks/consul/libraries/consul_helper.rb + pattern: "SUPPORTED_MINOR = '(\\S+)'" + tag_filter: + output_as: '$1.$2' # go-crond https://github.com/webdevops/go-crond.git: replace_in_files: diff --git a/files/gitlab-config-template/gitlab.rb.template b/files/gitlab-config-template/gitlab.rb.template index 94f38026430014c3bb13ca98be388c572e2264a0_ZmlsZXMvZ2l0bGFiLWNvbmZpZy10ZW1wbGF0ZS9naXRsYWIucmIudGVtcGxhdGU=..abf9d8a85f8b3b0db2fb5a59a5b7e53668d0c8fc_ZmlsZXMvZ2l0bGFiLWNvbmZpZy10ZW1wbGF0ZS9naXRsYWIucmIudGVtcGxhdGU= 100644 --- a/files/gitlab-config-template/gitlab.rb.template +++ b/files/gitlab-config-template/gitlab.rb.template @@ -3233,6 +3233,7 @@ # Consul (EE only) ################################################################################ # consul['enable'] = false +# consul['binary_path'] = '/opt/gitlab/embedded/bin/consul' # consul['dir'] = '/var/opt/gitlab/consul' # consul['username'] = 'gitlab-consul' # consul['group'] = 'gitlab-consul' diff --git a/files/gitlab-cookbooks/consul/attributes/default.rb b/files/gitlab-cookbooks/consul/attributes/default.rb index 94f38026430014c3bb13ca98be388c572e2264a0_ZmlsZXMvZ2l0bGFiLWNvb2tib29rcy9jb25zdWwvYXR0cmlidXRlcy9kZWZhdWx0LnJi..abf9d8a85f8b3b0db2fb5a59a5b7e53668d0c8fc_ZmlsZXMvZ2l0bGFiLWNvb2tib29rcy9jb25zdWwvYXR0cmlidXRlcy9kZWZhdWx0LnJi 100644 --- a/files/gitlab-cookbooks/consul/attributes/default.rb +++ b/files/gitlab-cookbooks/consul/attributes/default.rb @@ -1,4 +1,5 @@ default['consul']['enable'] = false +default['consul']['binary_path'] = '/opt/gitlab/embedded/bin/consul' default['consul']['dir'] = '/var/opt/gitlab/consul' default['consul']['username'] = 'gitlab-consul' default['consul']['group'] = 'gitlab-consul' diff --git a/files/gitlab-cookbooks/consul/libraries/consul_helper.rb b/files/gitlab-cookbooks/consul/libraries/consul_helper.rb index 94f38026430014c3bb13ca98be388c572e2264a0_ZmlsZXMvZ2l0bGFiLWNvb2tib29rcy9jb25zdWwvbGlicmFyaWVzL2NvbnN1bF9oZWxwZXIucmI=..abf9d8a85f8b3b0db2fb5a59a5b7e53668d0c8fc_ZmlsZXMvZ2l0bGFiLWNvb2tib29rcy9jb25zdWwvbGlicmFyaWVzL2NvbnN1bF9oZWxwZXIucmI= 100644 --- a/files/gitlab-cookbooks/consul/libraries/consul_helper.rb +++ b/files/gitlab-cookbooks/consul/libraries/consul_helper.rb @@ -9,6 +9,10 @@ # provide an `enable_service_#{service_name}` and `disable_service_#{service_name}` recipe SERVICES = %w(postgresql).freeze + # This version should be keep in sync with consul versions in + # software/consul.rb and consul_helper.rb. + SUPPORTED_MINOR = '1.16'.freeze + def initialize(node) @node = node @default_configuration = { @@ -176,7 +180,7 @@ def installed_version return unless OmnibusHelper.new(@node).service_up?('consul') - command = '/opt/gitlab/embedded/bin/consul version' + command = "#{@node['consul']['binary_path']} version" command_output = VersionHelper.version(command) raise "Execution of the command `#{command}` failed" unless command_output @@ -195,6 +199,13 @@ info[:Config][:Version] unless info.empty? end + def installed_is_supported? + major_installed, minor_installed = installed_version.split('.')[0..1] + major_supported, minor_supported = SUPPORTED_MINOR.split('.') + + major_installed == major_supported && minor_installed == minor_supported + end + private def verify_incoming? diff --git a/files/gitlab-cookbooks/consul/recipes/enable_daemon.rb b/files/gitlab-cookbooks/consul/recipes/enable_daemon.rb index 94f38026430014c3bb13ca98be388c572e2264a0_ZmlsZXMvZ2l0bGFiLWNvb2tib29rcy9jb25zdWwvcmVjaXBlcy9lbmFibGVfZGFlbW9uLnJi..abf9d8a85f8b3b0db2fb5a59a5b7e53668d0c8fc_ZmlsZXMvZ2l0bGFiLWNvb2tib29rcy9jb25zdWwvcmVjaXBlcy9lbmFibGVfZGFlbW9uLnJi 100644 --- a/files/gitlab-cookbooks/consul/recipes/enable_daemon.rb +++ b/files/gitlab-cookbooks/consul/recipes/enable_daemon.rb @@ -20,6 +20,7 @@ runit_service 'consul' do options({ + binary_path: node['consul']['binary_path'], config_dir: node['consul']['config_dir'], custom_config_dir: node['consul']['custom_config_dir'], config_file: node['consul']['config_file'], @@ -57,3 +58,17 @@ end only_if { consul_helper.running_version != consul_helper.installed_version } end + +ruby_block 'warn consul version mismatch' do + block do + message = <<~MESSAGE + The version of the installed consul service is different than what is expected. + Please run `gitlab-ctl consul-download --force` and restart consul to start + the new version: + + https://docs.gitlab.com/ee/administration/consul.html#restart-consul + MESSAGE + LoggingHelper.warning(message) + end + only_if { consul_helper.installed_is_supported? } +end diff --git a/files/gitlab-cookbooks/consul/templates/default/sv-consul-run.erb b/files/gitlab-cookbooks/consul/templates/default/sv-consul-run.erb index 94f38026430014c3bb13ca98be388c572e2264a0_ZmlsZXMvZ2l0bGFiLWNvb2tib29rcy9jb25zdWwvdGVtcGxhdGVzL2RlZmF1bHQvc3YtY29uc3VsLXJ1bi5lcmI=..abf9d8a85f8b3b0db2fb5a59a5b7e53668d0c8fc_ZmlsZXMvZ2l0bGFiLWNvb2tib29rcy9jb25zdWwvdGVtcGxhdGVzL2RlZmF1bHQvc3YtY29uc3VsLXJ1bi5lcmI= 100644 --- a/files/gitlab-cookbooks/consul/templates/default/sv-consul-run.erb +++ b/files/gitlab-cookbooks/consul/templates/default/sv-consul-run.erb @@ -9,7 +9,9 @@ exec chpst -P -e <%= @options[:env_dir] %> \ -U <%= @options[:user] %>:<%= @options[:groupname] %> \ -u <%= @options[:user] %>:<%= @options[:groupname] %> \ -/opt/gitlab/embedded/bin/consul agent -config-file <%= @options[:config_file] %> \ +<%= @options[:binary_path] %> \ + agent \ + -config-file <%= @options[:config_file] %> \ -config-dir <%= @options[:config_dir] %> \ <% unless @options[:custom_config_dir].nil? -%> -config-dir <%= @options[:custom_config_dir] %> \ diff --git a/files/gitlab-ctl-commands-ee/consul.rb b/files/gitlab-ctl-commands-ee/consul.rb index 94f38026430014c3bb13ca98be388c572e2264a0_ZmlsZXMvZ2l0bGFiLWN0bC1jb21tYW5kcy1lZS9jb25zdWwucmI=..abf9d8a85f8b3b0db2fb5a59a5b7e53668d0c8fc_ZmlsZXMvZ2l0bGFiLWN0bC1jb21tYW5kcy1lZS9jb25zdWwucmI= 100644 --- a/files/gitlab-ctl-commands-ee/consul.rb +++ b/files/gitlab-ctl-commands-ee/consul.rb @@ -15,8 +15,9 @@ # require "#{base_path}/embedded/service/omnibus-ctl-ee/lib/consul" +require "#{base_path}/embedded/service/omnibus-ctl-ee/lib/consul_download" add_command_under_category('consul', 'consul', 'Interact with the gitlab-consul cluster', 2) do consul = ConsulHandler.new(ARGV, $stdin.gets) consul.execute end @@ -18,5 +19,9 @@ add_command_under_category('consul', 'consul', 'Interact with the gitlab-consul cluster', 2) do consul = ConsulHandler.new(ARGV, $stdin.gets) consul.execute end + +add_command_under_category('consul-download', 'consul', 'Download consul for the gitlab-consul cluster', 2) do + ConsulDownloadCommand.new(ARGV).run +end diff --git a/files/gitlab-ctl-commands-ee/lib/consul.rb b/files/gitlab-ctl-commands-ee/lib/consul.rb index 94f38026430014c3bb13ca98be388c572e2264a0_ZmlsZXMvZ2l0bGFiLWN0bC1jb21tYW5kcy1lZS9saWIvY29uc3VsLnJi..abf9d8a85f8b3b0db2fb5a59a5b7e53668d0c8fc_ZmlsZXMvZ2l0bGFiLWN0bC1jb21tYW5kcy1lZS9saWIvY29uc3VsLnJi 100644 --- a/files/gitlab-ctl-commands-ee/lib/consul.rb +++ b/files/gitlab-ctl-commands-ee/lib/consul.rb @@ -1,4 +1,10 @@ require 'mixlib/shellout' +# For testing purposes, if the first path cannot be found load the second +begin + require_relative '../../omnibus-ctl/lib/gitlab_ctl' +rescue LoadError + require_relative '../../gitlab-ctl-commands/lib/gitlab_ctl' +end class ConsulHandler WatcherError = Class.new(StandardError) @@ -17,7 +23,7 @@ class << self def run_consul(cmd) - command = Mixlib::ShellOut.new("/opt/gitlab/embedded/bin/consul #{cmd}") + command = Mixlib::ShellOut.new("#{consul_binary} #{cmd}") command.run_command begin command.error! @@ -28,6 +34,12 @@ end command.stdout end + + private + + def consul_binary + GitlabCtl::Util.get_node_attributes.dig('consul', 'binary_path') + end end class Kv diff --git a/files/gitlab-ctl-commands-ee/lib/consul_download.rb b/files/gitlab-ctl-commands-ee/lib/consul_download.rb new file mode 100644 index 0000000000000000000000000000000000000000..abf9d8a85f8b3b0db2fb5a59a5b7e53668d0c8fc_ZmlsZXMvZ2l0bGFiLWN0bC1jb21tYW5kcy1lZS9saWIvY29uc3VsX2Rvd25sb2FkLnJi --- /dev/null +++ b/files/gitlab-ctl-commands-ee/lib/consul_download.rb @@ -0,0 +1,84 @@ +require 'net/http' +require 'zip' + +class ConsulDownloadCommand + # This version should be keep in sync with consul versions in + # software/consul.rb and consul_helper.rb. + DEFAULT_VERSION = '1.16.3'.freeze + + def initialize(args) + @args = args + @options = { + version: DEFAULT_VERSION, + architecture: default_architecture, + output: '/usr/local/bin/consul', + force: false, + } + + parse_options! + end + + def run + if !@options[:force] && File.exist?(@options[:output]) + warn("Consul binary already exists! Use --force to overwrite #{@options[:output]}.") + return + end + + response = Net::HTTP.get_response(download_url) + + if response.code != '200' + warn("Error downloading consul from #{download_url}: #{resp.code}") + return + end + + Zip::File.open_buffer(StringIO.new(response.body)) do |zipio| + zipio.each do |entry| + next unless entry.file? && entry.name == 'consul' + + File.write(@options[:output], entry.get_input_stream.read) + File.chmod(0755, @options[:output]) + puts "Successfully downloaded Consul #{@options[:version]} to #{@options[:output]}." + puts "Start using the external Consul by setting consul['binary_path'] = '#{@options[:output]}' in your `gitlab.rb` and run `gitlab-ctl reconfigure`" + end + end + end + + private + + def parse_options! + opts_parser = OptionParser.new do |opts| + opts.banner = 'Usage: gitlab-ctl consul download [options]' + + opts.on('-v', '--version=VERSION', 'Version of the consul binary to be downloaded') do |v| + @options[:version] = v + end + + opts.on('-a', '--arch=ARCHITECTURE', 'Architecture of the consul binary to be downloaded') do |v| + @options[:architecture] = v + end + + opts.on('-o', '--output=PATH', 'Path to write the consul binary to') do |v| + @options[:output] = v + end + + opts.on('-f', '--force', 'Overwrite existing consul binary') do |v| + @options[:force] = v + end + + opts.on_tail('-h', '--help', 'Show this message') do + puts opts + exit + end + end + + opts_parser.parse!(@args.dup) + end + + def download_url + URI.parse("https://releases.hashicorp.com/consul/#{@options[:version]}/consul_#{@options[:version]}_linux_#{@options[:architecture]}.zip") + end + + def default_architecture + GitlabCtl::Util.get_node_attributes.dig('kernel', 'machine') || 'amd64' + end +end diff --git a/files/gitlab-ctl-commands/pg-upgrade.rb b/files/gitlab-ctl-commands/pg-upgrade.rb index 94f38026430014c3bb13ca98be388c572e2264a0_ZmlsZXMvZ2l0bGFiLWN0bC1jb21tYW5kcy9wZy11cGdyYWRlLnJi..abf9d8a85f8b3b0db2fb5a59a5b7e53668d0c8fc_ZmlsZXMvZ2l0bGFiLWN0bC1jb21tYW5kcy9wZy11cGdyYWRlLnJi 100644 --- a/files/gitlab-ctl-commands/pg-upgrade.rb +++ b/files/gitlab-ctl-commands/pg-upgrade.rb @@ -647,6 +647,7 @@ node = Patroni::Client.new scope = @attributes.dig(:patroni, :scope) node_name = @attributes.dig(:patroni, :name) + consul_binary = @attributes.dig(:consul, :binary_path) if node.up? @instance_type = :patroni_leader if node.leader? @@ -655,7 +656,7 @@ failure_cause = :patroni_running_on_replica if @instance_type == :patroni_replica @instance_type == :patroni_leader || @instance_type == :patroni_standby_leader else - leader_name = GitlabCtl::Util.get_command_output("#{base_path}/embedded/bin/consul kv get /service/#{scope}/leader").strip + leader_name = GitlabCtl::Util.get_command_output("#{consul_binary} kv get /service/#{scope}/leader").strip @instance_type = node_name == leader_name ? :patroni_leader : :patroni_replica unless leader_name.nil? || leader_name.empty? failure_cause = :patroni_stopped_on_leader if @instance_type == :patroni_leader @instance_type == :patroni_replica @@ -708,4 +709,5 @@ def remove_patroni_cluster_state scope = @attributes.dig(:patroni, :scope) || '' + consul_binary = @attributes.dig(:consul, :binary_path) unless !scope.empty? && GitlabCtl::Util.progress_message('Wiping Patroni cluster state') do @@ -711,5 +713,5 @@ unless !scope.empty? && GitlabCtl::Util.progress_message('Wiping Patroni cluster state') do - run_command("#{base_path}/embedded/bin/consul kv delete -recurse /service/#{scope}/") + run_command("#{consul_binary} kv delete -recurse /service/#{scope}/") end die 'Unable to wipe the cluster state' end diff --git a/spec/chef/gitlab-ctl-commands-ee/lib/consul_spec.rb b/spec/chef/gitlab-ctl-commands-ee/lib/consul_spec.rb index 94f38026430014c3bb13ca98be388c572e2264a0_c3BlYy9jaGVmL2dpdGxhYi1jdGwtY29tbWFuZHMtZWUvbGliL2NvbnN1bF9zcGVjLnJi..abf9d8a85f8b3b0db2fb5a59a5b7e53668d0c8fc_c3BlYy9jaGVmL2dpdGxhYi1jdGwtY29tbWFuZHMtZWUvbGliL2NvbnN1bF9zcGVjLnJi 100644 --- a/spec/chef/gitlab-ctl-commands-ee/lib/consul_spec.rb +++ b/spec/chef/gitlab-ctl-commands-ee/lib/consul_spec.rb @@ -1,5 +1,2 @@ require 'spec_helper' - -$LOAD_PATH << File.join(__dir__, '../../../files/gitlab-ctl-commands-ee/lib') - require 'consul' @@ -5,4 +2,5 @@ require 'consul' +require 'consul_download' RSpec.describe ConsulHandler do describe '#initialize' do @@ -26,6 +24,10 @@ let(:consul_cmd) { '/opt/gitlab/embedded/bin/consul' } describe ConsulHandler::Kv do + before(:each) do + allow(GitlabCtl::Util).to receive(:get_node_attributes).and_return('consul' => { 'binary_path' => consul_cmd }) + end + it 'allows nil values' do results = double('results', run_command: [], error!: nil, stdout: '') expect(Mixlib::ShellOut).to receive(:new).with("#{consul_cmd} kv put foo ").and_return(results) @@ -59,6 +61,10 @@ end describe ConsulHandler::Encrypt do + before(:each) do + allow(GitlabCtl::Util).to receive(:get_node_attributes).and_return('consul' => { 'binary_path' => consul_cmd }) + end + describe '#keygen' do it 'returns the generated key' do generated_key = 'BYX6EPaUiUI0TDdm6gAMmmLnpJSwePJ33Xwh6rjCYbg=' @@ -78,3 +84,41 @@ end end end + +RSpec.describe ConsulDownloadCommand do + let(:command) do + described_class.new([ + '', '', + '--arch', 'amd64', + '--output', '/tmp/consul', + '--force', true, + '--version', '1.16.1', + ]) + end + + it 'downloads consul' do + consul_zip = Zip::OutputStream.write_buffer do |zio| + zio.put_next_entry("consul") + zio.write "binary" + zio.put_next_entry("foo") + zio.write "bar" + end + mock_response = Net::HTTPSuccess.new('1.0', '200', '') + expect(mock_response).to receive(:body).and_return(consul_zip.string) + + expect(GitlabCtl::Util) + .to receive(:get_node_attributes) + .and_return({ 'kernel' => { 'machine' => 'amd64' } }) + expect(Net::HTTP).to receive(:get_response) + .with(URI.parse('https://releases.hashicorp.com/consul/1.16.1/consul_1.16.1_linux_amd64.zip')) + .and_return(mock_response) + expect(File) + .to receive(:write) + .with('/tmp/consul', 'binary') + expect(File) + .to receive(:chmod) + .with(0755, '/tmp/consul') + + command.run + end +end diff --git a/spec/chef/gitlab-ctl-commands-ee/lib/geo/pitr_file_spec.rb b/spec/chef/gitlab-ctl-commands-ee/lib/geo/pitr_file_spec.rb index 94f38026430014c3bb13ca98be388c572e2264a0_c3BlYy9jaGVmL2dpdGxhYi1jdGwtY29tbWFuZHMtZWUvbGliL2dlby9waXRyX2ZpbGVfc3BlYy5yYg==..abf9d8a85f8b3b0db2fb5a59a5b7e53668d0c8fc_c3BlYy9jaGVmL2dpdGxhYi1jdGwtY29tbWFuZHMtZWUvbGliL2dlby9waXRyX2ZpbGVfc3BlYy5yYg== 100644 --- a/spec/chef/gitlab-ctl-commands-ee/lib/geo/pitr_file_spec.rb +++ b/spec/chef/gitlab-ctl-commands-ee/lib/geo/pitr_file_spec.rb @@ -58,6 +58,8 @@ context 'when consul is enabled' do before do allow(Dir).to receive(:exist?).with('/opt/gitlab/service/consul').and_return(true) + allow(GitlabCtl::Util).to receive(:get_node_attributes) + .and_return({ 'consul' => { 'binary_path' => '/opt/gitlab/embedded/bin/consul' } }) end it "uses consul" do