Skip to content
Snippets Groups Projects
Commit 30d714be3762 authored by João Pereira's avatar João Pereira
Browse files

Add registry-database import command to gitlab-ctl

The command will allow gitlab-ctl to interact with the configured
registry metadata database. It adds the subcommand  `import` to to the `registry-database`
to allow users to import existing filestystem metadata into the
metadata database.

For more details see
https://gitlab.com/gitlab-org/container-registry/-/issues/1160.

Changelog: added
parent c2d1c9f5b20d
3 merge requests!119heptapod#1448: making Heptapod 1.1 the new oldstable,!116heptapod#1395: make 1.1 the new stable,!115Merged upstream 16.7 stable branches first commit
require 'optparse'
module Import
CMD_NAME = 'import'.freeze
SUMMARY_WIDTH = 40
DESC_INDENT = 45
def self.indent(str, len)
str.gsub(/%%/, ' ' * len)
end
USAGE = <<~EOS.freeze
Import filesystem metadata into the database
See documentation at https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/database-import-tool.md
Usage:
gitlab-ctl registry-database import [options]
Options:
-B, --common-blobs Import all blob metadata from common storage
-c, --row-count Count and log number of rows across relevant database tables on (pre)import completion
-d, --dry-run Do not commit changes to the database
-e, --require-empty-database Abort import if the database is not empty
-p, --pre-import Import immutable repository-scoped data to speed up a following import
-r, --all-repositories Import all repository-scoped data
-h, --help Help for import
-1, --step-one pre-import Perform step one of a multi-step import: alias for pre-import
-2, --step-two all-repositories Perform step two of a multi-step import: alias for all-repositories
-3, --step-three common-blobs Perform step three of a multi-step import: alias for common-blobs
EOS
def self.parse_options!(args, parser, options)
return unless args.include? CMD_NAME
loop do
break if args.shift == CMD_NAME
end
parser.on('-h', '--help', 'Usage help') do
parser.set_summary_width(SUMMARY_WIDTH)
Kernel.puts USAGE
Kernel.exit 0
end
parser.on('-B', '--common-blobs', indent('import all blob metadata from common storage', DESC_INDENT)) do
options[:common_blobs] = '--common-blobs'
end
parser.on('-c', '--row-count', indent('count and log number of rows across relevant database tables on (pre)import completion', DESC_INDENT)) do
options[:row_count] = '--row-count'
end
parser.on('-d', '--dry-run', indent('do not commit changes to the database', DESC_INDENT)) do
options[:dry_run] = '--dry-run'
end
parser.on('-e', '--require-empty-database', indent('abort import if the database is not empty', DESC_INDENT)) do
options[:empty] = '--require-empty-database'
end
parser.on('-p', '--pre-import', indent('import immutable repository-scoped data to speed up a following import', DESC_INDENT)) do
options[:pre_import] = '--pre-import'
end
parser.on('-r', '--all-repositories', indent('import all repository-scoped data', DESC_INDENT)) do
options[:all_repositories] = '--all-repositories'
options[:needs_read_only] = true
end
parser.on('-1', '--step-one', indent('perform step one of a multi-step import: alias for pre-import', DESC_INDENT)) do
options[:step_one] = '--step-one'
end
parser.on('-2', '--step-two', indent('perform step two of a multi-step import: alias for all-repositories', DESC_INDENT)) do
options[:step_two] = '--step-two'
options[:needs_read_only] = true
end
parser.on('-3', '--step-three', indent('perform step three of a multi-step import: alias for common-blobs', DESC_INDENT)) do
options[:step_three] = '--step-three'
end
parser.order!(args)
options
end
end
......@@ -105,7 +105,7 @@
end
def self.populate_subcommands(options)
database_docs_url = 'https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/database-migrations.md?ref_type=heads#administration'
database_docs_url = 'https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/database-migrations.md#administration'
{
'up' => OptionParser.new do |opts|
......
......@@ -2,8 +2,9 @@
require 'optparse'
require_relative './migrate'
require_relative './import'
module RegistryDatabase
EXEC_PATH = '/opt/gitlab/embedded/bin/registry'.freeze
CONFIG_PATH = '/var/opt/gitlab/registry/config.yml'.freeze
......@@ -5,9 +6,9 @@
module RegistryDatabase
EXEC_PATH = '/opt/gitlab/embedded/bin/registry'.freeze
CONFIG_PATH = '/var/opt/gitlab/registry/config.yml'.freeze
USAGE ||= <<~EOS.freeze
USAGE = <<~EOS.freeze
Usage:
gitlab-ctl registry-database command subcommand [options]
......@@ -16,6 +17,7 @@
COMMANDS:
migrate Manage schema migrations
import Import filesystem metadata into the database
EOS
def self.parse_options!(ctl, args)
......@@ -52,7 +54,7 @@
end
def self.populate_commands(options)
database_docs_url = 'https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/database-migrations.md?ref_type=heads#administration'
database_docs_url = 'https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/database-migrations.md#administration'
{
'migrate' => OptionParser.new do |opts|
......@@ -64,6 +66,13 @@
exit 128
end
end,
'import' => OptionParser.new do |parser|
Import.parse_options!(ARGV, parser, options)
rescue OptionParser::ParseError => e
warn "#{e}\n\n#{Import::USAGE}"
exit 128
end,
}
end
......@@ -85,7 +94,9 @@
command = set_command(options)
puts "Executing command:\n#{command.join(' ')}\n"
begin
status = Kernel.system(*command)
Kernel.exit!(1) unless status
ensure
......@@ -88,9 +99,9 @@
begin
status = Kernel.system(*command)
Kernel.exit!(1) unless status
ensure
start!
start! unless running?
end
end
def self.set_command(options)
......@@ -93,6 +104,7 @@
end
end
def self.set_command(options)
command = [EXEC_PATH, "database", options[:command], options[:subcommand]]
command = [EXEC_PATH, "database", options[:command]]
options.delete(:command)
......@@ -98,3 +110,3 @@
options.delete(:command)
command += [options[:subcommand]] unless options[:subcommand].nil?
options.delete(:subcommand)
......@@ -100,4 +112,5 @@
options.delete(:subcommand)
needs_stop = options[:needs_stop]
options.delete(:needs_stop)
......@@ -101,7 +114,10 @@
needs_stop = options[:needs_stop]
options.delete(:needs_stop)
continue?(needs_stop)
needs_read_only = options[:needs_read_only]
options.delete(:needs_read_only)
continue?(needs_stop, needs_read_only)
command += ["-n", options[:limit]] unless options[:limit].nil?
options.delete(:limit)
......@@ -124,6 +140,11 @@
!GitlabCtl::Util.run_command("gitlab-ctl status #{service_name}").error?
end
def self.read_only?
config = YAML.load_file(CONFIG_PATH)
config.dig('storage', 'maintenance', 'readonly', 'enabled') == true
end
def self.start!
puts "Starting service #{service_name}"
......@@ -148,9 +169,8 @@
"registry"
end
def self.continue?(needs_stop)
return unless needs_stop && running?
puts 'WARNING: Migrations cannot be applied while the container registry is running. '\
'Stop the registry before proceeding? (y/n)'.color(:yellow)
def self.continue?(needs_stop, needs_read_only)
if needs_stop && running?
puts 'WARNING: Command cannot run while the container registry is running. ' \
'Stop the registry before proceeding? (y/n)'.color(:yellow)
......@@ -156,7 +176,8 @@
if $stdin.gets.chomp.casecmp('y').zero?
stop!
else
puts "Exiting..."
exit 1
if $stdin.gets.chomp.casecmp('y').zero?
stop!
else
puts "Exiting..."
exit 1
end
end
......@@ -162,3 +183,8 @@
end
return unless running? && needs_read_only && !read_only?
puts 'Command requires the container registry to be in read-only mode. Exiting...'
exit 1
end
end
require 'optparse'
require_relative('../../../../../files/gitlab-ctl-commands/lib/registry/import')
RSpec.describe Import do
describe '.parse_options!' do
before do
allow(Kernel).to receive(:exit) { |code| raise "Kernel.exit(#{code})" }
end
options_data = [
[:common_blobs, 'common-blobs', false],
[:row_count, 'row-count', false],
[:dry_run, 'dry-run', false],
[:empty, 'require-empty-database', false],
[:pre_import, 'pre-import', false],
[:all_repositories, 'all-repositories', true],
[:step_one, 'step-one', false],
[:step_two, 'step-two', true],
[:step_three, 'step-three', false]
]
options_data.each do |option, option_name, read_only|
it "correctly parses the #{option_name} option#{' with read-only mode' if read_only}" do
expected_options = { option => "--#{option_name}" }
expected_options[:needs_read_only] = true if read_only
result = Import.parse_options!(%W[import --#{option_name}], OptionParser.new, {})
expect(result).to eq(expected_options)
end
end
end
end
......@@ -42,5 +42,12 @@
expect(received).to have_key(:command)
end
end
context 'when command is import' do
let(:command) { 'import' }
it_behaves_like 'unknown option is specified'
it_behaves_like 'parses command options'
end
end
end
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment