diff --git a/changelogs/unreleased/ag-add-preflight-checks-to-promote-script.yml b/changelogs/unreleased/ag-add-preflight-checks-to-promote-script.yml new file mode 100644 index 0000000000000000000000000000000000000000..8067b581c3bf02c87449c1ccdc048fd82540c958_Y2hhbmdlbG9ncy91bnJlbGVhc2VkL2FnLWFkZC1wcmVmbGlnaHQtY2hlY2tzLXRvLXByb21vdGUtc2NyaXB0LnltbA== --- /dev/null +++ b/changelogs/unreleased/ag-add-preflight-checks-to-promote-script.yml @@ -0,0 +1,5 @@ +--- +title: Add (optional) list of manual checks to promote_to_primary script. +merge_request: 4231 +author: +type: other diff --git a/files/gitlab-ctl-commands-ee/lib/geo/promote_to_primary.rb b/files/gitlab-ctl-commands-ee/lib/geo/promote_to_primary.rb index 05ce52e709cdfda22e47415e30d438fcf8a75d68_ZmlsZXMvZ2l0bGFiLWN0bC1jb21tYW5kcy1lZS9saWIvZ2VvL3Byb21vdGVfdG9fcHJpbWFyeS5yYg==..8067b581c3bf02c87449c1ccdc048fd82540c958_ZmlsZXMvZ2l0bGFiLWN0bC1jb21tYW5kcy1lZS9saWIvZ2VvL3Byb21vdGVfdG9fcHJpbWFyeS5yYg== 100644 --- a/files/gitlab-ctl-commands-ee/lib/geo/promote_to_primary.rb +++ b/files/gitlab-ctl-commands-ee/lib/geo/promote_to_primary.rb @@ -9,6 +9,8 @@ end def execute + run_preflight_checks + make_sure_primary_is_down promote_postgresql_to_primary @@ -22,6 +24,31 @@ private + def run_preflight_checks + return true if @options[:skip_preflight_checks] + + puts + puts 'Ensure you have completed the following manual '\ + 'preflight checks:' + puts '- Check if you need to migrate to Object Storage' + puts '- Review configuration of each secondary node' + puts '- Run system checks' + puts '- Check that secrets match between nodes' + puts '- Ensure Geo replication is up-to-date' + puts '- Verify the integrity of replicated data' + puts '- Notify users of scheduled maintenance' + puts 'Please read https://docs.gitlab.com/ee/administration/geo/'\ + 'disaster_recovery/planned_failover.html#preflight-checks' + puts + puts 'Did you perform all manual preflight checks (y/n)?'.color(:green) + + return if STDIN.gets.chomp.casecmp('y').zero? + + raise 'ERROR: Manual preflight checks were not performed! '\ + 'Please read https://docs.gitlab.com/ee/administration/geo/'\ + 'disaster_recovery/planned_failover.html#preflight-checks'.color(:red) + end + def make_sure_primary_is_down return true if @options[:confirm_primary_is_down] diff --git a/files/gitlab-ctl-commands-ee/promote_to_primary_node.rb b/files/gitlab-ctl-commands-ee/promote_to_primary_node.rb index 05ce52e709cdfda22e47415e30d438fcf8a75d68_ZmlsZXMvZ2l0bGFiLWN0bC1jb21tYW5kcy1lZS9wcm9tb3RlX3RvX3ByaW1hcnlfbm9kZS5yYg==..8067b581c3bf02c87449c1ccdc048fd82540c958_ZmlsZXMvZ2l0bGFiLWN0bC1jb21tYW5kcy1lZS9wcm9tb3RlX3RvX3ByaW1hcnlfbm9kZS5yYg== 100644 --- a/files/gitlab-ctl-commands-ee/promote_to_primary_node.rb +++ b/files/gitlab-ctl-commands-ee/promote_to_primary_node.rb @@ -26,7 +26,7 @@ OptionParser.new do |opts| opts.banner = "Usage: gitlab-ctl promote-to-primary-node [options]" - opts.on('-p', '--[no-]confirm-primary-is-down', 'Do not ask a confirmation that primary is down') do |p| + opts.on('-p', '--[no-]confirm-primary-is-down', 'Do not ask for confirmation that primary is down') do |p| options[:confirm_primary_is_down] = p end @@ -30,6 +30,6 @@ options[:confirm_primary_is_down] = p end - opts.on('-c', '--[no-]confirm-removing-keys', 'Do not ask a confirmation about removing keys') do |c| + opts.on('-c', '--[no-]confirm-removing-keys', 'Do not ask for confirmation about removing keys') do |c| options[:confirm_removing_keys] = c end @@ -34,5 +34,9 @@ options[:confirm_removing_keys] = c end + + opts.on('-m', '--skip-preflight-checks', 'Do not ask for confirmation if manual checks ran') do |c| + options[:skip_preflight_checks] = m + end end.parse!(ARGV.dup) options diff --git a/spec/gitlab-ctl-commands-ee/lib/geo/promote_to_primary_spec.rb b/spec/gitlab-ctl-commands-ee/lib/geo/promote_to_primary_spec.rb index 05ce52e709cdfda22e47415e30d438fcf8a75d68_c3BlYy9naXRsYWItY3RsLWNvbW1hbmRzLWVlL2xpYi9nZW8vcHJvbW90ZV90b19wcmltYXJ5X3NwZWMucmI=..8067b581c3bf02c87449c1ccdc048fd82540c958_c3BlYy9naXRsYWItY3RsLWNvbW1hbmRzLWVlL2xpYi9nZW8vcHJvbW90ZV90b19wcmltYXJ5X3NwZWMucmI= 100644 --- a/spec/gitlab-ctl-commands-ee/lib/geo/promote_to_primary_spec.rb +++ b/spec/gitlab-ctl-commands-ee/lib/geo/promote_to_primary_spec.rb @@ -5,7 +5,6 @@ require 'fileutils' require 'geo/promote_to_primary' -require 'fileutils' require 'gitlab_ctl/util' describe Geo::PromoteToPrimary, '#execute' do @@ -9,7 +8,7 @@ require 'gitlab_ctl/util' describe Geo::PromoteToPrimary, '#execute' do - subject(:command) { described_class.new(nil, {}) } + subject(:command) { described_class.new(nil, { skip_preflight_checks: true }) } let(:temp_directory) { Dir.mktmpdir } let(:gitlab_config_path) { File.join(temp_directory, 'gitlab.rb') } @@ -23,6 +22,54 @@ FileUtils.rm_rf(temp_directory) end + describe '#run_preflight_checks' do + subject(:run_preflight_checks) do + described_class.new(nil, options).send(:run_preflight_checks) + end + + let(:confirmation) { 'y' } + + before do + allow(STDIN).to receive(:gets).and_return(confirmation) + end + + context 'when `--skip-preflight-checks` is passed' do + let(:options) { { skip_preflight_checks: true } } + let(:confirmation) { 'n' } + + it 'does not raise error' do + expect { run_preflight_checks }.not_to raise_error + end + end + + context 'when `--skip-preflight-checks` is not passed' do + let(:options) { {} } + + it 'prints preflight check instructions' do + expect { run_preflight_checks }.to output( + /Ensure you have completed the following manual preflight checks/) + .to_stdout + end + + context 'when confirmation is accepted' do + it 'does not raise an error' do + expect { run_preflight_checks }.to_not raise_error + end + end + + context 'when confirmation is not accepted' do + let(:confirmation) { 'n' } + + it 'raises error' do + expect { run_preflight_checks }.to raise_error( + RuntimeError, + /ERROR: Manual preflight checks were not performed/ + ) + end + end + end + end + context 'when confirmation is accepted' do before do allow(STDIN).to receive(:gets).and_return('y')