Commit 65443926 authored by Zeger-Jan van de Weg's avatar Zeger-Jan van de Weg
Browse files

Gitlab::Shell works on shard name, not path

Direct disk access is done through Gitaly now, so the legacy path was
deprecated. This path was used in Gitlab::Shell however. This required
the refactoring in this commit.

Added is the removal of direct path access on the project model, as that
lookup wasn't needed anymore is most cases.

Closes https://gitlab.com/gitlab-org/gitaly/issues/1111
parent 0393b83c9727
......@@ -400,7 +400,8 @@ def sanitize_repo_path(project, message)
exports_path = File.join(Settings.shared['path'], 'tmp/project_exports')
filtered_message = message.strip.gsub(exports_path, "[REPO EXPORT PATH]")
filtered_message.gsub(project.repository_storage_path.chomp('/'), "[REPOS PATH]")
disk_path = Gitlab.config.repositories.storages[project.repository_storage].legacy_disk_path
filtered_message.gsub(disk_path.chomp('/'), "[REPOS PATH]")
end
def project_child_container_class(view_path)
......
......@@ -45,25 +45,25 @@ def move_dir
# Hooks
# Save the storage paths before the projects are destroyed to use them on after destroy
# Save the storages before the projects are destroyed to use them on after destroy
def prepare_for_destroy
old_repository_storage_paths
old_repository_storages
end
private
def move_repositories
# Move the namespace directory in all storage paths used by member projects
repository_storage_paths.each do |repository_storage_path|
# Move the namespace directory in all storages used by member projects
repository_storages.each do |repository_storage|
# Ensure old directory exists before moving it
gitlab_shell.add_namespace(repository_storage_path, full_path_was)
gitlab_shell.add_namespace(repository_storage, full_path_was)
# Ensure new directory exists before moving it (if there's a parent)
gitlab_shell.add_namespace(repository_storage_path, parent.full_path) if parent
gitlab_shell.add_namespace(repository_storage, parent.full_path) if parent
unless gitlab_shell.mv_namespace(repository_storage_path, full_path_was, full_path)
unless gitlab_shell.mv_namespace(repository_storage, full_path_was, full_path)
Rails.logger.error "Exception moving path #{repository_storage_path} from #{full_path_was} to #{full_path}"
Rails.logger.error "Exception moving path #{repository_storage} from #{full_path_was} to #{full_path}"
# if we cannot move namespace directory we should rollback
# db changes in order to prevent out of sync between db and fs
......@@ -72,33 +72,33 @@ def move_repositories
end
end
def old_repository_storage_paths
@old_repository_storage_paths ||= repository_storage_paths
def old_repository_storages
@old_repository_storage_paths ||= repository_storages
end
def repository_storage_paths
def repository_storages
# We need to get the storage paths for all the projects, even the ones that are
# pending delete. Unscoping also get rids of the default order, which causes
# problems with SELECT DISTINCT.
Project.unscoped do
all_projects.select('distinct(repository_storage)').to_a.map(&:repository_storage_path)
all_projects.select('distinct(repository_storage)').to_a.map(&:repository_storage)
end
end
def rm_dir
# Remove the namespace directory in all storages paths used by member projects
old_repository_storage_paths.each do |repository_storage_path|
old_repository_storages.each do |repository_storage|
# Move namespace directory into trash.
# We will remove it later async
new_path = "#{full_path}+#{id}+deleted"
if gitlab_shell.mv_namespace(repository_storage_path, full_path, new_path)
if gitlab_shell.mv_namespace(repository_storage, full_path, new_path)
Gitlab::AppLogger.info %Q(Namespace directory "#{full_path}" moved to "#{new_path}")
# Remove namespace directroy async with delay so
# GitLab has time to remove all projects first
run_after_commit do
GitlabShellWorker.perform_in(5.minutes, :rm_namespace, repository_storage_path, new_path)
GitlabShellWorker.perform_in(5.minutes, :rm_namespace, repository_storage, new_path)
end
end
end
......
......@@ -512,10 +512,6 @@ def empty_repo?
repository.empty?
end
def repository_storage_path
Gitlab.config.repositories.storages[repository_storage]&.legacy_disk_path
end
def team
@team ||= ProjectTeam.new(self)
end
......@@ -1106,7 +1102,7 @@ def expire_caches_before_rename(old_path)
# Check if repository already exists on disk
def check_repository_path_availability
return true if skip_disk_validation
return false unless repository_storage_path
return false unless repository_storage
expires_full_path_cache # we need to clear cache to validate renames correctly
......@@ -1907,14 +1903,14 @@ def wiki_reference_count
def check_repository_absence!
return if skip_disk_validation
if repository_storage_path.blank? || repository_with_same_path_already_exists?
if repository_storage.blank? || repository_with_same_path_already_exists?
errors.add(:base, 'There is already a repository with that name on disk')
throw :abort
end
end
def repository_with_same_path_already_exists?
gitlab_shell.exists?(repository_storage_path, "#{disk_path}.git")
gitlab_shell.exists?(repository_storage, "#{disk_path}.git")
end
# set last_activity_at to the same as created_at
......
......@@ -21,7 +21,7 @@ def initialize(project, user = nil)
end
delegate :empty?, to: :pages
delegate :repository_storage_path, :hashed_storage?, to: :project
delegate :repository_storage, :hashed_storage?, to: :project
def path
@project.path + '.wiki'
......
......@@ -84,9 +84,14 @@ def cleanup
# Return absolute path to repository
def path_to_repo
@path_to_repo ||= File.expand_path(
File.join(repository_storage_path, disk_path + '.git')
)
@path_to_repo ||=
begin
storage = Gitlab.config.repositories.storages[@project.repository_storage]
File.expand_path(
File.join(storage.legacy_disk_path, disk_path + '.git')
)
end
end
def inspect
......@@ -915,10 +920,6 @@ def fetch_ref(source_repository, source_ref:, target_ref:)
raw_repository.fetch_ref(source_repository.raw_repository, source_ref: source_ref, target_ref: target_ref)
end
def repository_storage_path
@project.repository_storage_path
end
def rebase(user, merge_request)
raw.rebase(user, merge_request.id, branch: merge_request.source_branch,
branch_sha: merge_request.source_branch_sha,
......
module Storage
class HashedProject
attr_accessor :project
delegate :gitlab_shell, :repository_storage_path, to: :project
delegate :gitlab_shell, :repository_storage, to: :project
ROOT_PATH_PREFIX = '@hashed'.freeze
......@@ -24,7 +24,7 @@ def disk_path
end
def ensure_storage_path_exists
gitlab_shell.add_namespace(repository_storage_path, base_dir)
gitlab_shell.add_namespace(repository_storage, base_dir)
end
def rename_repo
......
module Storage
class LegacyProject
attr_accessor :project
delegate :namespace, :gitlab_shell, :repository_storage_path, to: :project
delegate :namespace, :gitlab_shell, :repository_storage, to: :project
def initialize(project)
@project = project
......@@ -24,18 +24,18 @@ def disk_path
def ensure_storage_path_exists
return unless namespace
gitlab_shell.add_namespace(repository_storage_path, base_dir)
gitlab_shell.add_namespace(repository_storage, base_dir)
end
def rename_repo
new_full_path = project.build_full_path
if gitlab_shell.mv_repository(repository_storage_path, project.full_path_was, new_full_path)
if gitlab_shell.mv_repository(repository_storage, project.full_path_was, new_full_path)
# If repository moved successfully we need to send update instructions to users.
# However we cannot allow rollback since we moved repository
# So we basically we mute exceptions in next actions
begin
gitlab_shell.mv_repository(repository_storage_path, "#{project.full_path_was}.wiki", "#{new_full_path}.wiki")
gitlab_shell.mv_repository(repository_storage, "#{project.full_path_was}.wiki", "#{new_full_path}.wiki")
return true
rescue => e
Rails.logger.error "Exception renaming #{project.full_path_was} -> #{new_full_path}: #{e}"
......
......@@ -91,7 +91,7 @@ def remove_repository(path)
project.run_after_commit do
# self is now project
GitlabShellWorker.perform_in(5.minutes, :remove_repository, self.repository_storage_path, new_path)
GitlabShellWorker.perform_in(5.minutes, :remove_repository, self.repository_storage, new_path)
end
else
false
......@@ -100,9 +100,9 @@ def remove_repository(path)
def mv_repository(from_path, to_path)
# There is a possibility project does not have repository or wiki
return true unless gitlab_shell.exists?(project.repository_storage_path, from_path + '.git')
return true unless gitlab_shell.exists?(project.repository_storage, from_path + '.git')
gitlab_shell.mv_repository(project.repository_storage_path, from_path, to_path)
gitlab_shell.mv_repository(project.repository_storage, from_path, to_path)
end
def attempt_rollback(project, message)
......
......@@ -47,8 +47,8 @@ def execute
private
def move_repository(from_name, to_name)
from_exists = gitlab_shell.exists?(project.repository_storage_path, "#{from_name}.git")
to_exists = gitlab_shell.exists?(project.repository_storage_path, "#{to_name}.git")
from_exists = gitlab_shell.exists?(project.repository_storage, "#{from_name}.git")
to_exists = gitlab_shell.exists?(project.repository_storage, "#{to_name}.git")
# If we don't find the repository on either original or target we should log that as it could be an issue if the
# project was not originally empty.
......@@ -60,7 +60,7 @@ def move_repository(from_name, to_name)
return true
end
gitlab_shell.mv_repository(project.repository_storage_path, from_name, to_name)
gitlab_shell.mv_repository(project.repository_storage, from_name, to_name)
end
def rollback_folder_move
......
......@@ -127,7 +127,7 @@ def rollback_folder_move
end
def move_repo_folder(from_name, to_name)
gitlab_shell.mv_repository(project.repository_storage_path, from_name, to_name)
gitlab_shell.mv_repository(project.repository_storage, from_name, to_name)
end
def execute_system_hooks
......
......@@ -13,7 +13,9 @@ def perform(*args)
# See https://gitlab.com/gitlab-org/gitaly/issues/1110
if args.empty?
source_project = target_project.forked_from_project
return target_project.mark_import_as_failed('Source project cannot be found.') unless source_project
unless source_project
return target_project.mark_import_as_failed('Source project cannot be found.')
end
fork_repository(target_project, source_project.repository_storage, source_project.disk_path)
else
......
......@@ -59,17 +59,17 @@ def rename_path(path)
end
def move_namespace(group_id, path_was, path)
repository_storage_paths = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{group_id}").map do |row|
Gitlab.config.repositories.storages[row['repository_storage']].legacy_disk_path
repository_storages = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{group_id}").map do |row|
row['repository_storage']
end.compact
# Move the namespace directory in all storages paths used by member projects
repository_storage_paths.each do |repository_storage_path|
repository_storages.each do |repository_storage|
# Ensure old directory exists before moving it
gitlab_shell.add_namespace(repository_storage_path, path_was)
gitlab_shell.add_namespace(repository_storage, path_was)
unless gitlab_shell.mv_namespace(repository_storage_path, path_was, path)
Rails.logger.error "Exception moving path #{repository_storage_path} from #{path_was} to #{path}"
unless gitlab_shell.mv_namespace(repository_storage, path_was, path)
Rails.logger.error "Exception moving on shard #{repository_storage} from #{path_was} to #{path}"
# if we cannot move namespace directory we should rollback
# db changes in order to prevent out of sync between db and fs
......
......@@ -53,8 +53,8 @@ def route_exists?(path)
select_all("SELECT id, path FROM routes WHERE path = '#{quote_string(path)}'").present?
end
def path_exists?(path, repository_storage_path)
repository_storage_path && gitlab_shell.exists?(repository_storage_path, path)
def path_exists?(shard, repository_storage_path)
repository_storage_path && gitlab_shell.exists?(shard, repository_storage_path)
end
# Accepts invalid path like test.git and returns test_git or
......@@ -70,8 +70,8 @@ def new_path(path)
def check_routes(base, counter, path)
route_exists = route_exists?(path)
Gitlab.config.repositories.storages.each_value do |storage|
if route_exists || path_exists?(path, storage.legacy_disk_path)
Gitlab.config.repositories.storages.each do |shard, storage|
if route_exists || path_exists?(shard, storage.legacy_disk_path)
counter += 1
path = "#{base}#{counter}"
......@@ -83,17 +83,17 @@ def check_routes(base, counter, path)
end
def move_namespace(namespace_id, path_was, path)
repository_storage_paths = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{namespace_id}").map do |row|
Gitlab.config.repositories.storages[row['repository_storage']].legacy_disk_path
repository_storages = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{namespace_id}").map do |row|
row['repository_storage']
end.compact
# Move the namespace directory in all storages paths used by member projects
repository_storage_paths.each do |repository_storage_path|
# Move the namespace directory in all storages used by member projects
repository_storages.each do |repository_storage|
# Ensure old directory exists before moving it
gitlab_shell.add_namespace(repository_storage_path, path_was)
gitlab_shell.add_namespace(repository_storage, path_was)
unless gitlab_shell.mv_namespace(repository_storage_path, path_was, path)
Rails.logger.error "Exception moving path #{repository_storage_path} from #{path_was} to #{path}"
unless gitlab_shell.mv_namespace(repository_storage, path_was, path)
Rails.logger.error "Exception moving on shard #{repository_storage} from #{path_was} to #{path}"
# if we cannot move namespace directory we should rollback
# db changes in order to prevent out of sync between db and fs
......
......@@ -75,10 +75,11 @@ def create_project
end
def mv_repo(project)
FileUtils.mv(repo_path, File.join(project.repository_storage_path, project.disk_path + '.git'))
storage_path = storage_path_for_shard(project.repository_storage)
FileUtils.mv(repo_path, project.repository.path_to_repo)
if bare_repo.wiki_exists?
FileUtils.mv(wiki_path, File.join(project.repository_storage_path, project.disk_path + '.wiki.git'))
FileUtils.mv(wiki_path, File.join(storage_path, project.disk_path + '.wiki.git'))
end
true
......@@ -88,6 +89,10 @@ def mv_repo(project)
false
end
def storage_path_for_shard(shard)
Gitlab.config.repositories.storages[shard].legacy_disk_path
end
def find_or_create_groups
return nil unless group_path.present?
......
......@@ -62,21 +62,20 @@ def rename_user(old_username, new_username)
end
def move_repositories(namespace, old_full_path, new_full_path)
repo_paths_for_namespace(namespace).each do |repository_storage_path|
repo_shards_for_namespace(namespace).each do |repository_storage|
# Ensure old directory exists before moving it
gitlab_shell.add_namespace(repository_storage_path, old_full_path)
gitlab_shell.add_namespace(repository_storage, old_full_path)
unless gitlab_shell.mv_namespace(repository_storage_path, old_full_path, new_full_path)
message = "Exception moving path #{repository_storage_path} \
from #{old_full_path} to #{new_full_path}"
unless gitlab_shell.mv_namespace(repository_storage, old_full_path, new_full_path)
message = "Exception moving on shard #{repository_storage} from #{old_full_path} to #{new_full_path}"
Rails.logger.error message
end
end
end
def repo_paths_for_namespace(namespace)
def repo_shards_for_namespace(namespace)
projects_for_namespace(namespace).distinct.select(:repository_storage)
.map(&:repository_storage_path)
.map(&:repository_storage)
end
def projects_for_namespace(namespace)
......
......@@ -51,7 +51,7 @@ def revert_renames
end
def move_repository(project, old_path, new_path)
unless gitlab_shell.mv_repository(project.repository_storage_path,
unless gitlab_shell.mv_repository(project.repository_storage,
old_path,
new_path)
Rails.logger.error "Error moving #{old_path} to #{new_path}"
......
......@@ -65,11 +65,11 @@ def generate_and_link_secret_token
# Init new repository
#
# storage - project's storage name
# storage - the shard key
# name - project disk path
#
# Ex.
# create_repository("/path/to/storage", "gitlab/gitlab-ci")
# create_repository("default", "gitlab/gitlab-ci")
#
def create_repository(storage, name)
relative_path = name.dup
......@@ -291,13 +291,13 @@ def list_key_ids(&block)
# Add empty directory for storing repositories
#
# Ex.
# add_namespace("/path/to/storage", "gitlab")
# add_namespace("default", "gitlab")
#
def add_namespace(storage, name)
Gitlab::GitalyClient.migrate(:add_namespace,
status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
if enabled
gitaly_namespace_client(storage).add(name)
Gitlab::GitalyClient::NamespaceService.new(storage).add(name)
else
path = full_path(storage, name)
FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name)
......@@ -313,13 +313,13 @@ def add_namespace(storage, name)
# Every repository inside this directory will be removed too
#
# Ex.
# rm_namespace("/path/to/storage", "gitlab")
# rm_namespace("default", "gitlab")
#
def rm_namespace(storage, name)
Gitlab::GitalyClient.migrate(:remove_namespace,
status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
if enabled
gitaly_namespace_client(storage).remove(name)
Gitlab::GitalyClient::NamespaceService.new(storage).remove(name)
else
FileUtils.rm_r(full_path(storage, name), force: true)
end
......@@ -338,7 +338,8 @@ def mv_namespace(storage, old_name, new_name)
Gitlab::GitalyClient.migrate(:rename_namespace,
status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
if enabled
gitaly_namespace_client(storage).rename(old_name, new_name)
Gitlab::GitalyClient::NamespaceService.new(storage)
.rename(old_name, new_name)
else
break false if exists?(storage, new_name) || !exists?(storage, old_name)
......@@ -374,7 +375,8 @@ def exists?(storage, dir_name)
Gitlab::GitalyClient.migrate(:namespace_exists,
status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
if enabled
gitaly_namespace_client(storage).exists?(dir_name)
Gitlab::GitalyClient::NamespaceService.new(storage)
.exists?(dir_name)
else
File.exist?(full_path(storage, dir_name))
end
......@@ -398,7 +400,7 @@ def gitlab_shell_user_home
def full_path(storage, dir_name)
raise ArgumentError.new("Directory name can't be blank") if dir_name.blank?
File.join(storage, dir_name)
File.join(Gitlab.config.repositories.storages[storage].legacy_disk_path, dir_name)
end
def gitlab_shell_projects_path
......@@ -475,14 +477,6 @@ def gitlab_shell_fast_execute_helper(cmd, vars = {})
Bundler.with_original_env { Popen.popen(cmd, nil, vars) }
end
def gitaly_namespace_client(storage_path)
storage, _value = Gitlab.config.repositories.storages.find do |storage, value|
value.legacy_disk_path == storage_path
end
Gitlab::GitalyClient::NamespaceService.new(storage)
end
def git_timeout
Gitlab.config.gitlab_shell.git_timeout
end
......
......@@ -427,10 +427,7 @@ namespace :gitlab do
user = User.find_by(username: username)
if user
repo_dirs = user.authorized_projects.map do |p|
File.join(
p.repository_storage_path,
"#{p.disk_path}.git"
)
p.repository.path_to_repo
end
repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) }
......
......@@ -10,9 +10,8 @@ namespace :gitlab do
end
scope.find_each do |project|
base = File.join(project.repository_storage_path, project.disk_path)
puts base + '.git'
puts base + '.wiki.git'
puts project.repository.path_to_repo
puts project.wiki.repository.path_to_repo
end
end
end
......@@ -125,7 +125,7 @@
user.reload
expect(response.status).to eq(302)
expect(gitlab_shell.exists?(project.repository_storage_path, "#{new_username}/#{project.path}.git")).to be_truthy
expect(gitlab_shell.exists?(project.repository_storage, "#{new_username}/#{project.path}.git")).to be_truthy
end
end
......@@ -143,7 +143,7 @@
user.reload
expect(response.status).to eq(302)
expect(gitlab_shell.exists?(project.repository_storage_path, "#{project.disk_path}.git")).to be_truthy
expect(gitlab_shell.exists?(project.repository_storage, "#{project.disk_path}.git")).to be_truthy
expect(before_disk_path).to eq(project.disk_path)
end
end
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment