# HG changeset patch # User João Pereira <jpereira@gitlab.com> # Date 1701774717 0 # Tue Dec 05 11:11:57 2023 +0000 # Node ID 30d714be376266d8d48a8d15847b89e1658839fc # Parent c2d1c9f5b20dea7e89d47e284e173effafddd44b Add registry-database import command to gitlab-ctl The command will allow gitlab-ctl to interact with the configured registry metadata database. It adds the subcommand `import` to to the `registry-database` to allow users to import existing filestystem metadata into the metadata database. For more details see https://gitlab.com/gitlab-org/container-registry/-/issues/1160. Changelog: added diff --git a/files/gitlab-ctl-commands/lib/registry/import.rb b/files/gitlab-ctl-commands/lib/registry/import.rb new file mode 100644 --- /dev/null +++ b/files/gitlab-ctl-commands/lib/registry/import.rb @@ -0,0 +1,88 @@ +require 'optparse' + +module Import + CMD_NAME = 'import'.freeze + SUMMARY_WIDTH = 40 + DESC_INDENT = 45 + + def self.indent(str, len) + str.gsub(/%%/, ' ' * len) + end + + USAGE = <<~EOS.freeze + Import filesystem metadata into the database + + See documentation at https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/database-import-tool.md + + Usage: + gitlab-ctl registry-database import [options] + + Options: + -B, --common-blobs Import all blob metadata from common storage + -c, --row-count Count and log number of rows across relevant database tables on (pre)import completion + -d, --dry-run Do not commit changes to the database + -e, --require-empty-database Abort import if the database is not empty + -p, --pre-import Import immutable repository-scoped data to speed up a following import + -r, --all-repositories Import all repository-scoped data + -h, --help Help for import + -1, --step-one pre-import Perform step one of a multi-step import: alias for pre-import + -2, --step-two all-repositories Perform step two of a multi-step import: alias for all-repositories + -3, --step-three common-blobs Perform step three of a multi-step import: alias for common-blobs + EOS + + def self.parse_options!(args, parser, options) + return unless args.include? CMD_NAME + + loop do + break if args.shift == CMD_NAME + end + + parser.on('-h', '--help', 'Usage help') do + parser.set_summary_width(SUMMARY_WIDTH) + Kernel.puts USAGE + Kernel.exit 0 + end + + parser.on('-B', '--common-blobs', indent('import all blob metadata from common storage', DESC_INDENT)) do + options[:common_blobs] = '--common-blobs' + end + + parser.on('-c', '--row-count', indent('count and log number of rows across relevant database tables on (pre)import completion', DESC_INDENT)) do + options[:row_count] = '--row-count' + end + + parser.on('-d', '--dry-run', indent('do not commit changes to the database', DESC_INDENT)) do + options[:dry_run] = '--dry-run' + end + + parser.on('-e', '--require-empty-database', indent('abort import if the database is not empty', DESC_INDENT)) do + options[:empty] = '--require-empty-database' + end + + parser.on('-p', '--pre-import', indent('import immutable repository-scoped data to speed up a following import', DESC_INDENT)) do + options[:pre_import] = '--pre-import' + end + + parser.on('-r', '--all-repositories', indent('import all repository-scoped data', DESC_INDENT)) do + options[:all_repositories] = '--all-repositories' + options[:needs_read_only] = true + end + + parser.on('-1', '--step-one', indent('perform step one of a multi-step import: alias for pre-import', DESC_INDENT)) do + options[:step_one] = '--step-one' + end + + parser.on('-2', '--step-two', indent('perform step two of a multi-step import: alias for all-repositories', DESC_INDENT)) do + options[:step_two] = '--step-two' + options[:needs_read_only] = true + end + + parser.on('-3', '--step-three', indent('perform step three of a multi-step import: alias for common-blobs', DESC_INDENT)) do + options[:step_three] = '--step-three' + end + + parser.order!(args) + + options + end +end diff --git a/files/gitlab-ctl-commands/lib/registry/migrate.rb b/files/gitlab-ctl-commands/lib/registry/migrate.rb --- a/files/gitlab-ctl-commands/lib/registry/migrate.rb +++ b/files/gitlab-ctl-commands/lib/registry/migrate.rb @@ -105,7 +105,7 @@ end def self.populate_subcommands(options) - database_docs_url = 'https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/database-migrations.md?ref_type=heads#administration' + database_docs_url = 'https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/database-migrations.md#administration' { 'up' => OptionParser.new do |opts| diff --git a/files/gitlab-ctl-commands/lib/registry/registry_database.rb b/files/gitlab-ctl-commands/lib/registry/registry_database.rb --- a/files/gitlab-ctl-commands/lib/registry/registry_database.rb +++ b/files/gitlab-ctl-commands/lib/registry/registry_database.rb @@ -2,12 +2,13 @@ require 'optparse' require_relative './migrate' +require_relative './import' module RegistryDatabase EXEC_PATH = '/opt/gitlab/embedded/bin/registry'.freeze CONFIG_PATH = '/var/opt/gitlab/registry/config.yml'.freeze - USAGE ||= <<~EOS.freeze + USAGE = <<~EOS.freeze Usage: gitlab-ctl registry-database command subcommand [options] @@ -16,6 +17,7 @@ COMMANDS: migrate Manage schema migrations + import Import filesystem metadata into the database EOS def self.parse_options!(ctl, args) @@ -52,7 +54,7 @@ end def self.populate_commands(options) - database_docs_url = 'https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/database-migrations.md?ref_type=heads#administration' + database_docs_url = 'https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/database-migrations.md#administration' { 'migrate' => OptionParser.new do |opts| @@ -64,6 +66,13 @@ exit 128 end end, + + 'import' => OptionParser.new do |parser| + Import.parse_options!(ARGV, parser, options) + rescue OptionParser::ParseError => e + warn "#{e}\n\n#{Import::USAGE}" + exit 128 + end, } end @@ -85,23 +94,30 @@ command = set_command(options) + puts "Executing command:\n#{command.join(' ')}\n" + begin status = Kernel.system(*command) Kernel.exit!(1) unless status ensure - start! + start! unless running? end end def self.set_command(options) - command = [EXEC_PATH, "database", options[:command], options[:subcommand]] + command = [EXEC_PATH, "database", options[:command]] + options.delete(:command) - options.delete(:command) + command += [options[:subcommand]] unless options[:subcommand].nil? options.delete(:subcommand) + needs_stop = options[:needs_stop] options.delete(:needs_stop) - continue?(needs_stop) + needs_read_only = options[:needs_read_only] + options.delete(:needs_read_only) + + continue?(needs_stop, needs_read_only) command += ["-n", options[:limit]] unless options[:limit].nil? options.delete(:limit) @@ -124,6 +140,11 @@ !GitlabCtl::Util.run_command("gitlab-ctl status #{service_name}").error? end + def self.read_only? + config = YAML.load_file(CONFIG_PATH) + config.dig('storage', 'maintenance', 'readonly', 'enabled') == true + end + def self.start! puts "Starting service #{service_name}" @@ -148,17 +169,22 @@ "registry" end - def self.continue?(needs_stop) - return unless needs_stop && running? - - puts 'WARNING: Migrations cannot be applied while the container registry is running. '\ - 'Stop the registry before proceeding? (y/n)'.color(:yellow) + def self.continue?(needs_stop, needs_read_only) + if needs_stop && running? + puts 'WARNING: Command cannot run while the container registry is running. ' \ + 'Stop the registry before proceeding? (y/n)'.color(:yellow) - if $stdin.gets.chomp.casecmp('y').zero? - stop! - else - puts "Exiting..." - exit 1 + if $stdin.gets.chomp.casecmp('y').zero? + stop! + else + puts "Exiting..." + exit 1 + end end + + return unless running? && needs_read_only && !read_only? + + puts 'Command requires the container registry to be in read-only mode. Exiting...' + exit 1 end end diff --git a/spec/chef/gitlab-ctl-commands/lib/registry/import_spec.rb b/spec/chef/gitlab-ctl-commands/lib/registry/import_spec.rb new file mode 100644 --- /dev/null +++ b/spec/chef/gitlab-ctl-commands/lib/registry/import_spec.rb @@ -0,0 +1,33 @@ +require 'optparse' + +require_relative('../../../../../files/gitlab-ctl-commands/lib/registry/import') + +RSpec.describe Import do + describe '.parse_options!' do + before do + allow(Kernel).to receive(:exit) { |code| raise "Kernel.exit(#{code})" } + end + + options_data = [ + [:common_blobs, 'common-blobs', false], + [:row_count, 'row-count', false], + [:dry_run, 'dry-run', false], + [:empty, 'require-empty-database', false], + [:pre_import, 'pre-import', false], + [:all_repositories, 'all-repositories', true], + [:step_one, 'step-one', false], + [:step_two, 'step-two', true], + [:step_three, 'step-three', false] + ] + + options_data.each do |option, option_name, read_only| + it "correctly parses the #{option_name} option#{' with read-only mode' if read_only}" do + expected_options = { option => "--#{option_name}" } + expected_options[:needs_read_only] = true if read_only + + result = Import.parse_options!(%W[import --#{option_name}], OptionParser.new, {}) + expect(result).to eq(expected_options) + end + end + end +end diff --git a/spec/chef/gitlab-ctl-commands/lib/registry/registry_database_spec.rb b/spec/chef/gitlab-ctl-commands/lib/registry/registry_database_spec.rb --- a/spec/chef/gitlab-ctl-commands/lib/registry/registry_database_spec.rb +++ b/spec/chef/gitlab-ctl-commands/lib/registry/registry_database_spec.rb @@ -42,5 +42,12 @@ expect(received).to have_key(:command) end end + + context 'when command is import' do + let(:command) { 'import' } + + it_behaves_like 'unknown option is specified' + it_behaves_like 'parses command options' + end end end