Commit d36ce290 authored by Jacob Vosmaer (GitLab)'s avatar Jacob Vosmaer (GitLab) Committed by Douwe Maan
Browse files

Find and mark more Git disk access locations

parent e00a544c41a8
......@@ -412,7 +412,10 @@ def sanitize_repo_path(project, message)
exports_path = File.join(Settings.shared['path'], 'tmp/project_exports')
filtered_message = message.strip.gsub(exports_path, "[REPO EXPORT PATH]")
disk_path = Gitlab.config.repositories.storages[project.repository_storage].legacy_disk_path
disk_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
Gitlab.config.repositories.storages[project.repository_storage].legacy_disk_path
end
filtered_message.gsub(disk_path.chomp('/'), "[REPOS PATH]")
end
......
module Backup
Error = Class.new(StandardError)
end
......@@ -44,7 +44,7 @@ def dump
end
report_success(success)
abort 'Backup failed' unless success
raise Backup::Error, 'Backup failed' unless success
end
def restore
......@@ -72,7 +72,7 @@ def restore
end
report_success(success)
abort 'Restore failed' unless success
abort Backup::Error, 'Restore failed' unless success
end
protected
......
......@@ -26,7 +26,7 @@ def dump
unless status.zero?
puts output
abort 'Backup failed'
raise Backup::Error, 'Backup failed'
end
run_pipeline!([%W(tar --exclude=lost+found -C #{@backup_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600])
......@@ -39,7 +39,11 @@ def dump
def restore
backup_existing_files_dir
run_pipeline!([%w(gzip -cd), %W(tar --unlink-first --recursive-unlink -C #{app_files_dir} -xf -)], in: backup_tarball)
run_pipeline!([%w(gzip -cd), %W(#{tar} --unlink-first --recursive-unlink -C #{app_files_dir} -xf -)], in: backup_tarball)
end
def tar
system(*%w[gtar --version], out: '/dev/null') ? 'gtar' : 'tar'
end
def backup_existing_files_dir
......@@ -61,7 +65,7 @@ def backup_existing_files_dir
def run_pipeline!(cmd_list, options = {})
status_list = Open3.pipeline(*cmd_list, options)
abort 'Backup failed' unless status_list.compact.all?(&:success?)
raise Backup::Error, 'Backup failed' unless status_list.compact.all?(&:success?)
end
end
end
......@@ -27,7 +27,7 @@ def pack
progress.puts "done".color(:green)
else
puts "creating archive #{tar_file} failed".color(:red)
abort 'Backup failed'
raise Backup::Error, 'Backup failed'
end
upload
......@@ -52,7 +52,7 @@ def upload
progress.puts "done".color(:green)
else
puts "uploading backup to #{remote_directory} failed".color(:red)
abort 'Backup failed'
raise Backup::Error, 'Backup failed'
end
end
......@@ -66,7 +66,7 @@ def cleanup
progress.puts "done".color(:green)
else
puts "deleting tmp directory '#{dir}' failed".color(:red)
abort 'Backup failed'
raise Backup::Error, 'Backup failed'
end
end
end
......
......@@ -17,7 +17,10 @@ def dump
Project.find_each(batch_size: 1000) do |project|
progress.print " * #{display_repo_path(project)} ... "
path_to_project_repo = path_to_repo(project)
path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
path_to_repo(project)
end
path_to_project_bundle = path_to_bundle(project)
# Create namespace dir or hashed path if missing
......@@ -51,7 +54,9 @@ def dump
end
wiki = ProjectWiki.new(project)
path_to_wiki_repo = path_to_repo(wiki)
path_to_wiki_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
path_to_repo(wiki)
end
path_to_wiki_bundle = path_to_bundle(wiki)
if File.exist?(path_to_wiki_repo)
......@@ -111,7 +116,9 @@ def restore_custom_hooks(project)
# TODO: Need to find a way to do this for gitaly
# Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/1195
in_path(path_to_tars(project)) do |dir|
path_to_project_repo = path_to_repo(project)
path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
path_to_repo(project)
end
cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir})
output, status = Gitlab::Popen.popen(cmd)
......
......@@ -109,7 +109,7 @@ def initialize(storage, relative_path, gl_repository)
end
def ==(other)
path == other.path
[storage, relative_path] == [other.storage, other.relative_path]
end
def path
......
......@@ -26,10 +26,6 @@ def bundle_to_disk
@shared.error(e)
false
end
def path_to_repo
@project.repository.path_to_repo
end
end
end
end
......@@ -22,12 +22,8 @@ def project_filename
"project.wiki.bundle"
end
def path_to_repo
@wiki.repository.path_to_repo
end
def wiki_repository_exists?
File.exist?(@wiki.repository.path_to_repo) && !@wiki.repository.empty?
@wiki.repository.exists? && !@wiki.repository.empty?
end
end
end
......
......@@ -128,10 +128,12 @@ def warn_user_is_not_gitlab
end
def all_repos
Gitlab.config.repositories.storages.each_value do |repository_storage|
IO.popen(%W(find #{repository_storage.legacy_disk_path} -mindepth 2 -type d -name *.git)) do |find|
find.each_line do |path|
yield path.chomp
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
Gitlab.config.repositories.storages.each_value do |repository_storage|
IO.popen(%W(find #{repository_storage.legacy_disk_path} -mindepth 2 -type d -name *.git)) do |find|
find.each_line do |path|
yield path.chomp
end
end
end
end
......
......@@ -12,7 +12,7 @@ namespace :gitlab do
namespaces = Namespace.pluck(:path)
namespaces << HASHED_REPOSITORY_NAME # add so that it will be ignored
Gitlab.config.repositories.storages.each do |name, repository_storage|
git_base_path = repository_storage.legacy_disk_path
git_base_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access { repository_storage.legacy_disk_path }
all_dirs = Dir.glob(git_base_path + '/*')
puts git_base_path.color(:yellow)
......@@ -54,7 +54,8 @@ namespace :gitlab do
move_suffix = "+orphaned+#{Time.now.to_i}"
Gitlab.config.repositories.storages.each do |name, repository_storage|
repo_root = repository_storage.legacy_disk_path
repo_root = Gitlab::GitalyClient::StorageSettings.allow_disk_access { repository_storage.legacy_disk_path }
# Look for global repos (legacy, depth 1) and normal repos (depth 2)
IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find|
find.each_line do |path|
......
......@@ -87,11 +87,13 @@
def wiki_exists?(project)
wiki = ProjectWiki.new(project)
File.exist?(wiki.repository.path_to_repo) && !wiki.repository.empty?
wiki.repository.exists? && !wiki.repository.empty?
end
def project_hook_exists?(project)
Gitlab::Git::Hook.new('post-receive', project.repository.raw_repository).exists?
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
Gitlab::Git::Hook.new('post-receive', project.repository.raw_repository).exists?
end
end
def click_import_project_tab
......
......@@ -46,7 +46,9 @@
end
it 'calls tar command with unlink' do
expect(subject).to receive(:run_pipeline!).with([%w(gzip -cd), %w(tar --unlink-first --recursive-unlink -C /var/gitlab-registry -xf -)], any_args)
expect(subject).to receive(:tar).and_return('blabla-tar')
expect(subject).to receive(:run_pipeline!).with([%w(gzip -cd), %w(blabla-tar --unlink-first --recursive-unlink -C /var/gitlab-registry -xf -)], any_args)
subject.restore
end
end
......
......@@ -34,7 +34,9 @@
let(:timestamp) { Time.utc(2017, 3, 22) }
let(:temp_dirs) do
Gitlab.config.repositories.storages.map do |name, storage|
File.join(storage.legacy_disk_path, '..', 'repositories.old.' + timestamp.to_i.to_s)
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
File.join(storage.legacy_disk_path, '..', 'repositories.old.' + timestamp.to_i.to_s)
end
end
end
......
......@@ -20,6 +20,13 @@
Rainbow.enabled = @rainbow
end
around do |example|
# TODO migrate BareRepositoryImport https://gitlab.com/gitlab-org/gitaly/issues/953
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
example.run
end
end
shared_examples 'importing a repository' do
describe '.execute' do
it 'creates a project for a repository in storage' do
......
......@@ -62,8 +62,10 @@
before do
gitlab_shell.create_repository(repository_storage, hashed_path)
repository = Rugged::Repository.new(repo_path)
repository.config['gitlab.fullpath'] = 'to/repo'
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
repository = Rugged::Repository.new(repo_path)
repository.config['gitlab.fullpath'] = 'to/repo'
end
end
after do
......
......@@ -159,6 +159,7 @@
let(:feature2) { 'feature2' }
around do |example|
# discover_default_branch will be moved to gitaly-ruby
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
example.run
end
......@@ -373,6 +374,7 @@ def submodule_url(path)
context '#submodules' do
around do |example|
# TODO #submodules will be removed, has been migrated to gitaly
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
example.run
end
......@@ -1055,6 +1057,7 @@ def commit_files(commit)
describe "#rugged_commits_between" do
around do |example|
# TODO #rugged_commits_between will be removed, has been migrated to gitaly
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
example.run
end
......@@ -1703,6 +1706,7 @@ def commit_files(commit)
let(:refs) { ['deadbeef', SeedRepo::RubyBlob::ID, '909e6157199'] }
around do |example|
# TODO #batch_existence isn't used anywhere, can we remove it?
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
example.run
end
......
......@@ -9,9 +9,11 @@ def args_for_popen(args_list)
end
def stub_popen_rev_list(*additional_args, with_lazy_block: true, output:)
repo_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access { repository.path }
params = [
args_for_popen(additional_args),
repository.path,
repo_path,
{},
hash_including(lazy_block: with_lazy_block ? anything : nil)
]
......
......@@ -552,7 +552,7 @@ def disable_protocol(protocol)
it 'returns not found' do
project.add_guest(user)
repo = project.repository
FileUtils.rm_rf(repo.path)
Gitlab::GitalyClient::StorageSettings.allow_disk_access { FileUtils.rm_rf(repo.path) }
# Sanity check for rm_rf
expect(repo.exists?).to eq(false)
......@@ -750,20 +750,22 @@ def stub_git_hooks
def merge_into_protected_branch
@protected_branch_merge_commit ||= begin
stub_git_hooks
project.repository.add_branch(user, unprotected_branch, 'feature')
target_branch = project.repository.lookup('feature')
source_branch = project.repository.create_file(
user,
'filename',
'This is the file content',
message: 'This is a good commit message',
branch_name: unprotected_branch)
rugged = project.repository.rugged
author = { email: "email@example.com", time: Time.now, name: "Example Git User" }
merge_index = rugged.merge_commits(target_branch, source_branch)
Rugged::Commit.create(rugged, author: author, committer: author, message: "commit message", parents: [target_branch, source_branch], tree: merge_index.write_tree(rugged))
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
stub_git_hooks
project.repository.add_branch(user, unprotected_branch, 'feature')
target_branch = project.repository.lookup('feature')
source_branch = project.repository.create_file(
user,
'filename',
'This is the file content',
message: 'This is a good commit message',
branch_name: unprotected_branch)
rugged = project.repository.rugged
author = { email: "email@example.com", time: Time.now, name: "Example Git User" }
merge_index = rugged.merge_commits(target_branch, source_branch)
Rugged::Commit.create(rugged, author: author, committer: author, message: "commit message", parents: [target_branch, source_branch], tree: merge_index.write_tree(rugged))
end
end
end
......
......@@ -41,8 +41,10 @@
after do
FileUtils.rm_rf(export_path)
FileUtils.rm_rf(project_with_repo.repository.path_to_repo)
FileUtils.rm_rf(project.repository.path_to_repo)
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
FileUtils.rm_rf(project_with_repo.repository.path_to_repo)
FileUtils.rm_rf(project.repository.path_to_repo)
end
end
it 'can access the MR' do
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment