# HG changeset patch # User Jaime Martinez <jmartinez@gitlab.com> # Date 1700042919 0 # Wed Nov 15 10:08:39 2023 +0000 # Node ID 09b423c6b0920f3c998895987b67797a2e7fe312 # Parent d296cc0f14bfef4acb57e25db6f4d50b149fccfc Add registry-database migrate command to gitlab-ctl The command will allow gitlab-ctl to interact with the configured registry metadata database. It adds the subcommand `migrate` to manage schema migrations for the database. The command interacts with a given database but does not do anything about provisioning the database. For more details see https://gitlab.com/gitlab-org/container-registry/-/issues/1108. Changelog: added diff --git a/files/gitlab-ctl-commands/lib/registry/migrate.rb b/files/gitlab-ctl-commands/lib/registry/migrate.rb new file mode 100644 --- /dev/null +++ b/files/gitlab-ctl-commands/lib/registry/migrate.rb @@ -0,0 +1,206 @@ +require 'optparse' + +module Migrate + CMD_NAME = 'migrate'.freeze + SUMMARY_WIDTH = 40 + DESC_INDENT = 45 + + def self.indent(str, len) + str.gsub(/%%/, ' ' * len) + end + + USAGE ||= <<~EOS.freeze + Manage migrations + + Usage: + gitlab-ctl registry-database migrate SUBCOMMAND [options] + + Subcommands: + down Apply down migrations + status Show migration status + up Apply up migrations + version Show current migration version + + Options: + -h, --help help for migrate + EOS + + UP_USAGE ||= <<~EOS.freeze + Apply up migrations + + Usage: + gitlab-ctl registry-database migrate up [options] + + Options: + -d, --dry-run do not commit changes to the database + -h, --help help for up + -l, --limit int limit the number of migrations (all by default) + -s, --skip-post-deployment do not apply post deployment migrations + EOS + + DOWN_USAGE ||= <<~EOS.freeze + Apply down migrations + + Usage: + gitlab-ctl registry-database migrate down [options] + + Options: + -d, --dry-run do not commit changes to the database + -f, --force no confirmation message + -h, --help help for down + -l, --limit int limit the number of migrations (all by default) + EOS + + STATUS_USAGE ||= <<~EOS.freeze + Show migration status + + Usage: + gitlab-ctl registry-database migrate status [options] + + Options: + -h, --help help for status + -s, --skip-post-deployment ignore post deployment migrations + -u, --up-to-date check if all known migrations are applied + + EOS + + VERSION_USAGE ||= <<~EOS.freeze + Show current migration version + + Usage: + gitlab-ctl registry-database migrate version [options] + + Flags: + -h, --help help for version + EOS + + def self.parse_options!(args, options) + return unless args.include? CMD_NAME + + loop do + break if args.shift == CMD_NAME + end + + OptionParser.new do |opts| + opts.on('-h', '--help', 'Usage help') do + Kernel.puts USAGE + Kernel.exit 0 + end + end.order! args + + subcommands = populate_subcommands(options) + subcommand = args.shift + + raise OptionParser::ParseError, "migrate subcommand is not specified." \ + if subcommand.nil? || subcommand.empty? + + raise OptionParser::ParseError, "Unknown migrate subcommand: #{subcommand}" \ + unless subcommands.key?(subcommand) + + subcommands[subcommand].parse!(args) + options[:subcommand] = subcommand + needs_stop!(options) + + options + end + + def self.populate_subcommands(options) + database_docs_url = 'https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/database-migrations.md?ref_type=heads#administration' + + { + 'up' => OptionParser.new do |opts| + opts.banner = "Usage gitlab-ctl registry-database migrate up [options]. See documentation at #{database_docs_url}" + parse_common_options!(opts) + parse_up_down_common_options!(options, opts) + parse_up_options!(options, opts) + end, + 'down' => OptionParser.new do |opts| + opts.banner = "Usage gitlab-ctl registry-database migrate down [options]. See documentation at #{database_docs_url}" + parse_common_options!(opts) + parse_up_down_common_options!(options, opts) + parse_down_options!(options, opts) + end, + 'status' => OptionParser.new do |opts| + opts.banner = "Usage gitlab-ctl registry-database migrate status [options]. See documentation at #{database_docs_url}" + parse_common_options!(opts) + parse_status_options!(options, opts) + end, + 'version' => OptionParser.new do |opts| + opts.banner = "Usage gitlab-ctl registry-database migrate version [options]. See documentation at #{database_docs_url}" + opts.on('-h', '--help', 'Usage help') do + Kernel.puts VERSION_USAGE + Kernel.exit 0 + end + + parse_common_options!(opts) + end, + } + end + + def self.parse_common_options!(option_parser) + option_parser.on("-h", "--help", "Prints this help") do + option_parser.set_summary_width(SUMMARY_WIDTH) + Kernel.puts USAGE + Kernel.exit 0 + end + end + + def self.parse_up_down_common_options!(options, option_parser) + option_parser.on('-d', '--dry-run', indent('do not commit changes to the database', DESC_INDENT)) do + options[:dry_run] = '-d' + end + + option_parser.on('-l limit', '--limit LIMIT', indent('limit the number of migrations (all by default)', DESC_INDENT)) do |limit| + raise OptionParser::ParseError, "--limit option must be a positive number" \ + if limit.nil? || limit.to_i <= 0 + + options[:limit] = limit + end + end + + def self.parse_up_options!(options, option_parser) + option_parser.on('-h', '--help', 'Usage help') do + Kernel.puts UP_USAGE + Kernel.exit 0 + end + + option_parser.on('-s', '--skip-post-deployment', indent('do not apply post deployment migration', DESC_INDENT)) do + options[:skip_post_deploy] = '-s' + end + end + + def self.parse_down_options!(options, option_parser) + option_parser.on('-h', '--help', 'Usage help') do + Kernel.puts DOWN_USAGE + Kernel.exit 0 + end + + option_parser.on('-f', '--force', indent('no confirmation message', DESC_INDENT)) do + options[:force] = '-f' + end + end + + def self.parse_status_options!(options, option_parser) + option_parser.on('-h', '--help', 'Usage help') do + Kernel.puts STATUS_USAGE + Kernel.exit 0 + end + + option_parser.on('-u', '--up-to-date', indent('do not commit changes to the database', DESC_INDENT)) do + options[:up_to_date] = '-u' + end + + option_parser.on('-s', '--skip-post-deployment', indent('do not apply post deployment migration', DESC_INDENT)) do + options[:skip_post_deploy] = '-s' + end + end + + def self.needs_stop!(options) + case options[:subcommand] + when 'up', 'down' + options[:needs_stop] = true unless options.has_key? :dry_run + else + options[:needs_stop] = false + end + end +end diff --git a/files/gitlab-ctl-commands/lib/registry/registry_database.rb b/files/gitlab-ctl-commands/lib/registry/registry_database.rb new file mode 100644 --- /dev/null +++ b/files/gitlab-ctl-commands/lib/registry/registry_database.rb @@ -0,0 +1,164 @@ +require 'fileutils' +require 'optparse' + +require_relative './migrate' + +module RegistryDatabase + EXEC_PATH = '/opt/gitlab/embedded/bin/registry'.freeze + CONFIG_PATH = '/var/opt/gitlab/registry/config.yml'.freeze + + USAGE ||= <<~EOS.freeze + Usage: + gitlab-ctl registry-database command subcommand [options] + + GLOBAL OPTIONS: + -h, --help Usage help + + COMMANDS: + migrate Manage schema migrations + EOS + + def self.parse_options!(ctl, args) + @ctl = ctl + + loop do + break if args.shift == 'registry-database' + end + + global = OptionParser.new do |opts| + opts.on('-h', '--help', 'Usage help') do + Kernel.puts USAGE + Kernel.exit 0 + end + end + + global.order!(args) + + # the command is needed by the dependencies in populate_commands + command = args[0] + raise OptionParser::ParseError, "registry-database command is not specified." \ + if command.nil? || command.empty? + + options = {} + commands = populate_commands(options) + + raise OptionParser::ParseError, "Unknown registry-database command: #{command}" \ + unless commands.key?(command) + + commands[command].parse!(args) + options[:command] = command + + options + end + + def self.populate_commands(options) + database_docs_url = 'https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/database-migrations.md?ref_type=heads#administration' + + { + 'migrate' => OptionParser.new do |opts| + opts.banner = "Usage: gitlab-ctl registry-database migrate SUBCOMMAND [options]. See documentation at #{database_docs_url}" + begin + Migrate.parse_options!(ARGV, options) + rescue OptionParser::ParseError => e + warn "#{e}\n\n#{Migrate::USAGE}" + exit 128 + end + end, + } + end + + def self.usage + USAGE + end + + def self.execute(options) + unless enabled? + log "Container registry is not enabled, exiting..." + return + end + + [EXEC_PATH, CONFIG_PATH].each do |path| + next if File.exist?(path) + + Kernel.abort "Could not find '#{path}' file. Is this command being run on a Container Registry node?" + end + + command = set_command(options) + + begin + status = Kernel.system(*command) + Kernel.exit!(1) unless status + ensure + start! + end + end + + def self.set_command(options) + command = [EXEC_PATH, "database", options[:command], options[:subcommand]] + + options.delete(:command) + options.delete(:subcommand) + needs_stop = options[:needs_stop] + options.delete(:needs_stop) + + continue?(needs_stop) + + command += ["-n", options[:limit]] unless options[:limit].nil? + options.delete(:limit) + + options.each do |_, opt| + command.append(opt) + end + + # always set the config file at the end + command += [CONFIG_PATH] + + command + end + + def self.log(msg) + @ctl.log(msg) + end + + def self.running? + !GitlabCtl::Util.run_command("gitlab-ctl status #{service_name}").error? + end + + def self.start! + puts "Starting service #{service_name}" + + @ctl.run_sv_command_for_service('start', service_name) + end + + def self.stop! + puts "Stopping service #{service_name}" + + @ctl.run_sv_command_for_service('stop', service_name) + end + + def self.enabled? + @ctl.service_enabled?(service_name) + end + + def self.config? + File.exist?(@path) + end + + def self.service_name + "registry" + end + + def self.continue?(needs_stop) + return unless needs_stop && running? + + puts 'WARNING: Migrations cannot be applied while the container registry is running. '\ + 'Stop the registry before proceeding? (y/n)'.color(:yellow) + + if $stdin.gets.chomp.casecmp('y').zero? + stop! + else + puts "Exiting..." + exit 1 + end + end +end diff --git a/files/gitlab-ctl-commands/registry_database.rb b/files/gitlab-ctl-commands/registry_database.rb new file mode 100644 --- /dev/null +++ b/files/gitlab-ctl-commands/registry_database.rb @@ -0,0 +1,33 @@ +# +# Copyright:: Copyright (c) 2016 GitLab Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +require 'optparse' + +require "#{base_path}/embedded/service/omnibus-ctl/lib/gitlab_ctl" +require "#{base_path}/embedded/service/omnibus-ctl/lib/registry/registry_database" + +add_command_under_category('registry-database', 'container-registry', 'Manage Container Registry database.', 2) do + begin + options = RegistryDatabase.parse_options!(self, ARGV) + rescue OptionParser::ParseError => e + warn "#{e}\n\n#{RegistryDatabase::USAGE}" + exit 128 + end + + puts "Running #{options[:command]} #{options[:subcommand]}" + RegistryDatabase.execute(options) + exit 0 +end diff --git a/spec/chef/gitlab-ctl-commands/lib/registry/migrate_spec.rb b/spec/chef/gitlab-ctl-commands/lib/registry/migrate_spec.rb new file mode 100644 --- /dev/null +++ b/spec/chef/gitlab-ctl-commands/lib/registry/migrate_spec.rb @@ -0,0 +1,121 @@ +require 'optparse' + +require_relative('../../../../../files/gitlab-ctl-commands/lib/registry/migrate') + +RSpec.describe Migrate do + describe '.parse_options!' do + before do + allow(Kernel).to receive(:exit) { |code| raise "Kernel.exit(#{code})" } + end + + shared_examples 'unknown option is specified' do + it 'throws an error' do + expect { Migrate.parse_options!(%W(migrate #{command} --unknown), {}) }.to raise_error(OptionParser::InvalidOption, /unknown/) + end + end + + it 'throws an error when subcommand is not specified' do + expect { Migrate.parse_options!(%w(migrate), {}) }.to raise_error(OptionParser::ParseError, /migrate subcommand is not specified./) + end + + it 'throws an error when unknown subcommand is specified' do + expect { Migrate.parse_options!(%w(migrate unknown-subcommand), {}) }.to raise_error(OptionParser::ParseError, /Unknown migrate subcommand: unknown-subcommand/) + end + + shared_examples 'parses subcommand options' do + it 'throws an error when an unknown option is specified' do + expect { Migrate.parse_options!(%W(migrate #{command} --unknown), {}) }.to raise_error(OptionParser::InvalidOption, /unknown/) + end + end + + shared_examples 'parses limit option' do + it 'throws an error when --limit is not a number' do + expect { Migrate.parse_options!(%W(migrate #{command} --limit not-a-number), {}) }.to raise_error(OptionParser::ParseError, /--limit option must be a positive number/) + end + + it 'throws an error when --limit is a negative number' do + expect { Migrate.parse_options!(%W(migrate #{command} --limit -5), {}) }.to raise_error(OptionParser::ParseError, /--limit option must be a positive number/) + end + + it 'throws an error when --limit is zero' do + expect { Migrate.parse_options!(%W(migrate #{command} --limit 0), {}) }.to raise_error(OptionParser::ParseError, /--limit option must be a positive number/) + end + end + + shared_examples 'parses dry_run option' do + it 'parses dry-run correctly' do + expected_options = { subcommand: command, dry_run: '-d' } + + expect(Migrate.parse_options!(%W(migrate #{command} -d), {})).to eq(expected_options) + end + end + + context 'when subcommand is up' do + let(:command) { 'up' } + + it_behaves_like 'unknown option is specified' + it_behaves_like 'parses subcommand options' + it_behaves_like 'parses limit option' + it_behaves_like 'parses dry_run option' + + it 'parses subcommand correctly' do + expected_options = { subcommand: 'up', limit: '5', skip_post_deploy: '-s', needs_stop: true } + + expect(Migrate.parse_options!(%W(migrate #{command} -s -l 5), {})).to eq(expected_options) + end + end + + context 'when subcommand is down' do + let(:command) { 'down' } + + it_behaves_like 'parses subcommand options' + it_behaves_like 'unknown option is specified' + it_behaves_like 'parses limit option' + it_behaves_like 'parses dry_run option' + + it 'parses subcommand correctly' do + expected_options = { subcommand: 'down', needs_stop: true } + + expect(Migrate.parse_options!(%W(migrate #{command}), {})).to eq(expected_options) + end + + it 'parses subcommand correctly with options' do + expected_options = { subcommand: 'down', force: '-f', limit: '10', needs_stop: true } + + expect(Migrate.parse_options!(%W(migrate #{command} -f -l 10), {})).to eq(expected_options) + end + end + + context 'when subcommand is status' do + let(:command) { 'status' } + + it_behaves_like 'parses subcommand options' + it_behaves_like 'unknown option is specified' + + it 'parses subcommand correctly' do + expected_options = { subcommand: 'status', needs_stop: false } + + expect(Migrate.parse_options!(%W(migrate #{command}), {})).to eq(expected_options) + end + + it 'parses subcommand correctly with options' do + expected_options = { subcommand: 'status', skip_post_deploy: '-s', up_to_date: '-u', needs_stop: false } + + expect(Migrate.parse_options!(%W(migrate #{command} -s -u), {})).to eq(expected_options) + end + end + + context 'when subcommand is version' do + let(:command) { 'version' } + + it_behaves_like 'parses subcommand options' + it_behaves_like 'unknown option is specified' + + it 'parses subcommand correctly' do + expected_options = { subcommand: 'version', needs_stop: false } + + expect(Migrate.parse_options!(%W(migrate #{command}), {})).to eq(expected_options) + end + end + end +end diff --git a/spec/chef/gitlab-ctl-commands/lib/registry/registry_database_spec.rb b/spec/chef/gitlab-ctl-commands/lib/registry/registry_database_spec.rb new file mode 100644 --- /dev/null +++ b/spec/chef/gitlab-ctl-commands/lib/registry/registry_database_spec.rb @@ -0,0 +1,46 @@ +require 'optparse' +require_relative('../../../../../files/gitlab-ctl-commands/lib/registry/registry_database') + +RSpec.describe RegistryDatabase do + describe '.parse_options!' do + let(:migrate_options) { { subcommand: 'up' } } + let(:ctl) {} + + before do + allow(Migrate).to receive(:parse_options!).and_return(:migrate_options) + allow(Kernel).to receive(:exit) { |code| raise "Kernel.exit(#{code})" } + end + + shared_examples 'unknown option is specified' do + it 'throws an error' do + expect { RegistryDatabase.parse_options!(ctl, %W(registry-database #{command} --unknown)) }.to raise_error(OptionParser::InvalidOption, /unknown/) + end + end + + it 'throws an error when command is not specified' do + expect { RegistryDatabase.parse_options!(ctl, %w(registry-database)) }.to raise_error(OptionParser::ParseError, /registry-database command is not specified./) + end + + it 'throws an error when unknown command is specified' do + expect { RegistryDatabase.parse_options!(ctl, %w(registry-database unknown-subcommand)) }.to raise_error(OptionParser::ParseError, /Unknown registry-database command: unknown-subcommand/) + end + + shared_examples 'parses command options' do + it 'throws an error when an unknown option is specified' do + expect { RegistryDatabase.parse_options!(ctl, %W(registry-database #{command} --unknown)) }.to raise_error(OptionParser::InvalidOption, /unknown/) + end + end + + context 'when command is migrate' do + let(:command) { 'migrate' } + + it_behaves_like 'unknown option is specified' + it_behaves_like 'parses command options' + + it 'parses subcommand correctly' do + received = RegistryDatabase.parse_options!(ctl, %W(registry-database #{command})) + expect(received).to have_key(:command) + end + end + end +end