Commit a84ccc74 authored by Gabriel Mazetto's avatar Gabriel Mazetto
Browse files

Moving away from the "extend" based factory to a more traditional one.

Using `extend` dynamically can lead to bad performance as it
invalidates the method's cache.
parent 642da64440be
module Storage
module HashedProject
extend ActiveSupport::Concern
# Base directory
#
# @return [String] directory where repository is stored
def base_dir
%Q(#{disk_hash[0..1]}/#{disk_hash[2..3]}) if disk_hash
end
# Disk path is used to build repository and project's wiki path on disk
#
# @return [String] combination of base_dir and the repository own name without `.git` or `.wiki.git` extensions
def disk_path
%Q(#{base_dir}/#{disk_hash})
end
def ensure_storage_path_exist
gitlab_shell.add_namespace(repository_storage_path, base_dir)
end
def rename_repo
# TODO: We cannot wipe most of this method until we provide migration path for Container Registries
path_was = previous_changes['path'].first
old_path_with_namespace = File.join(namespace.full_path, path_was)
new_path_with_namespace = File.join(namespace.full_path, path)
Rails.logger.error "Attempting to rename #{old_path_with_namespace} -> #{new_path_with_namespace}"
if has_container_registry_tags?
Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry tags are present!"
# we currently doesn't support renaming repository if it contains images in container registry
raise StandardError.new('Project cannot be renamed, because images are present in its container registry')
end
begin
# TODO: we can avoid cache expiration if cache is based on UUID or just project_id
expire_caches_before_rename(old_path_with_namespace)
expires_full_path_cache
send_move_instructions(old_path_with_namespace)
@old_path_with_namespace = old_path_with_namespace
SystemHooksService.new.execute_hooks_for(self, :rename)
@repository = nil
rescue => e
Rails.logger.error "Exception renaming #{old_path_with_namespace} -> #{new_path_with_namespace}: #{e}"
# Returning false does not rollback after_* transaction but gives
# us information about failing some of tasks
false
end
Gitlab::AppLogger.info "Project was renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}"
# TODO: When we move Uploads and Pages to use UUID we can disable this transfers as well
Gitlab::UploadsTransfer.new.rename_project(path_was, path, namespace.full_path)
Gitlab::PagesTransfer.new.rename_project(path_was, path, namespace.full_path)
end
private
# Generates the hash for the project path and name on disk
# If you need to refer to the repository on disk, use the `#disk_path`
def disk_hash
@disk_hash ||= Digest::SHA2.hexdigest(self.id.to_s) if self.id
end
end
end
module Storage
module LegacyProject
extend ActiveSupport::Concern
# Base directory
#
# @return [String] directory where repository is stored
def base_dir
namespace.full_path
end
# Disk path is used to build repository and project's wiki path on disk
#
# @return [String] combination of base_dir and the repository own name without `.git` or `.wiki.git` extensions
def disk_path
full_path
end
def ensure_storage_path_exist
gitlab_shell.add_namespace(repository_storage_path, base_dir)
end
def rename_repo
path_was = previous_changes['path'].first
old_path_with_namespace = File.join(namespace.full_path, path_was)
new_path_with_namespace = File.join(namespace.full_path, path)
Rails.logger.error "Attempting to rename #{old_path_with_namespace} -> #{new_path_with_namespace}"
if has_container_registry_tags?
Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry tags are present!"
# we currently doesn't support renaming repository if it contains images in container registry
raise StandardError.new('Project cannot be renamed, because images are present in its container registry')
end
expire_caches_before_rename(old_path_with_namespace)
if gitlab_shell.mv_repository(repository_storage_path, old_path_with_namespace, new_path_with_namespace)
# If repository moved successfully we need to send update instructions to users.
# However we cannot allow rollback since we moved repository
# So we basically we mute exceptions in next actions
begin
gitlab_shell.mv_repository(repository_storage_path, "#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki")
send_move_instructions(old_path_with_namespace)
expires_full_path_cache
@old_path_with_namespace = old_path_with_namespace
SystemHooksService.new.execute_hooks_for(self, :rename)
@repository = nil
rescue => e
Rails.logger.error "Exception renaming #{old_path_with_namespace} -> #{new_path_with_namespace}: #{e}"
# Returning false does not rollback after_* transaction but gives
# us information about failing some of tasks
false
end
else
Rails.logger.error "Repository could not be renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}"
# if we cannot move namespace directory we should rollback
# db changes in order to prevent out of sync between db and fs
raise StandardError.new('repository cannot be renamed')
end
Gitlab::AppLogger.info "Project was renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}"
Gitlab::UploadsTransfer.new.rename_project(path_was, path, namespace.full_path)
Gitlab::PagesTransfer.new.rename_project(path_was, path, namespace.full_path)
end
end
end
......@@ -32,6 +32,8 @@
:merge_requests_enabled?, :issues_enabled?, to: :project_feature,
allow_nil: true
delegate :base_dir, :disk_path, :ensure_storage_path_exist, :rename_repo, to: :storage
default_value_for :archived, false
default_value_for :visibility_level, gitlab_config_features.visibility_level
default_value_for :container_registry_enabled, gitlab_config_features.container_registry
......@@ -59,7 +61,7 @@
after_validation :check_pending_delete
# Storage specific hooks
after_initialize :load_storage
after_initialize :use_hashed_storage
after_create :ensure_storage_path_exist
after_save :ensure_storage_path_exist, if: :namespace_id_changed?
......@@ -1424,6 +1426,14 @@
private
def load_storage
return unless has_attribute?(:storage_version)
def storage
@storage ||=
if !has_attribute?(:storage_version) # during migration
Storage::LegacyProject.new(self)
elsif self.storage_version && self.storage_version >= 1
Storage::HashedProject.new(self)
else
Storage::LegacyProject.new(self)
end
end
......@@ -1429,5 +1439,4 @@
if self.storage_version && self.storage_version >= 1
self.extend Storage::HashedProject
elsif !self.persisted? && current_application_settings.hashed_storage_enabled
def use_hashed_storage
if !self.persisted? && current_application_settings.hashed_storage_enabled
self.storage_version = LATEST_STORAGE_VERSION
......@@ -1433,7 +1442,4 @@
self.storage_version = LATEST_STORAGE_VERSION
self.extend Storage::HashedProject
else
self.extend Storage::LegacyProject
end
end
......
module Storage
class HashedProject
attr_accessor :project
delegate :namespace, :gitlab_shell, :repository_storage_path, to: :project
def initialize(project)
@project = project
end
# Base directory
#
# @return [String] directory where repository is stored
def base_dir
%Q(#{disk_hash[0..1]}/#{disk_hash[2..3]}) if disk_hash
end
# Disk path is used to build repository and project's wiki path on disk
#
# @return [String] combination of base_dir and the repository own name without `.git` or `.wiki.git` extensions
def disk_path
%Q(#{base_dir}/#{disk_hash})
end
def ensure_storage_path_exist
gitlab_shell.add_namespace(repository_storage_path, base_dir)
end
def rename_repo
# TODO: We cannot wipe most of this method until we provide migration path for Container Registries
path_was = project.previous_changes['path'].first
old_path_with_namespace = File.join(namespace.full_path, path_was)
new_path_with_namespace = File.join(namespace.full_path, project.path)
Rails.logger.error "Attempting to rename #{old_path_with_namespace} -> #{new_path_with_namespace}"
if project.has_container_registry_tags?
Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry tags are present!"
# we currently doesn't support renaming repository if it contains images in container registry
raise StandardError.new('Project cannot be renamed, because images are present in its container registry')
end
begin
# TODO: we can avoid cache expiration if cache is based on UUID or just project_id
project.expire_caches_before_rename(old_path_with_namespace)
project.expires_full_path_cache
project.send_move_instructions(old_path_with_namespace)
@old_path_with_namespace = old_path_with_namespace
SystemHooksService.new.execute_hooks_for(project, :rename)
@repository = nil
rescue => e
Rails.logger.error "Exception renaming #{old_path_with_namespace} -> #{new_path_with_namespace}: #{e}"
# Returning false does not rollback after_* transaction but gives
# us information about failing some of tasks
false
end
Gitlab::AppLogger.info "Project was renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}"
# TODO: When we move Uploads and Pages to use UUID we can disable this transfers as well
Gitlab::UploadsTransfer.new.rename_project(path_was, project.path, namespace.full_path)
Gitlab::PagesTransfer.new.rename_project(path_was, project.path, namespace.full_path)
end
private
# Generates the hash for the project path and name on disk
# If you need to refer to the repository on disk, use the `#disk_path`
def disk_hash
@disk_hash ||= Digest::SHA2.hexdigest(project.id.to_s) if project.id
end
end
end
module Storage
class LegacyProject
attr_accessor :project
delegate :namespace, :gitlab_shell, :repository_storage_path, to: :project
def initialize(project)
@project = project
end
# Base directory
#
# @return [String] directory where repository is stored
def base_dir
namespace.full_path
end
# Disk path is used to build repository and project's wiki path on disk
#
# @return [String] combination of base_dir and the repository own name without `.git` or `.wiki.git` extensions
def disk_path
project.full_path
end
def ensure_storage_path_exist
return unless namespace
gitlab_shell.add_namespace(repository_storage_path, base_dir)
end
def rename_repo
path_was = project.previous_changes['path'].first
old_path_with_namespace = File.join(base_dir, path_was)
new_path_with_namespace = File.join(base_dir, project.path)
Rails.logger.error "Attempting to rename #{old_path_with_namespace} -> #{new_path_with_namespace}"
if project.has_container_registry_tags?
Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry tags are present!"
# we currently doesn't support renaming repository if it contains images in container registry
raise StandardError.new('Project cannot be renamed, because images are present in its container registry')
end
project.expire_caches_before_rename(old_path_with_namespace)
if gitlab_shell.mv_repository(repository_storage_path, old_path_with_namespace, new_path_with_namespace)
# If repository moved successfully we need to send update instructions to users.
# However we cannot allow rollback since we moved repository
# So we basically we mute exceptions in next actions
begin
gitlab_shell.mv_repository(repository_storage_path, "#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki")
project.send_move_instructions(old_path_with_namespace)
project.expires_full_path_cache
@old_path_with_namespace = old_path_with_namespace
SystemHooksService.new.execute_hooks_for(project, :rename)
@repository = nil
rescue => e
Rails.logger.error "Exception renaming #{old_path_with_namespace} -> #{new_path_with_namespace}: #{e}"
# Returning false does not rollback after_* transaction but gives
# us information about failing some of tasks
false
end
else
Rails.logger.error "Repository could not be renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}"
# if we cannot move namespace directory we should rollback
# db changes in order to prevent out of sync between db and fs
raise StandardError.new('repository cannot be renamed')
end
Gitlab::AppLogger.info "Project was renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}"
Gitlab::UploadsTransfer.new.rename_project(path_was, project.path, base_dir)
Gitlab::PagesTransfer.new.rename_project(path_was, project.path, base_dir)
end
end
end
......@@ -8,7 +8,6 @@
disable_ddl_transaction!
def up
# rubocop:disable Migration/AddColumnWithDefaultToLargeTable
add_column :projects, :storage_version, :integer, limit: 2
add_concurrent_index :projects, :storage_version
end
......
......@@ -2395,8 +2395,8 @@
end
context 'hashed storage' do
let(:project) { create(:project, :repository, :hashed) }
let(:project) { create(:project, :repository) }
let(:gitlab_shell) { Gitlab::Shell.new }
let(:hash) { '6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b' }
before do
......@@ -2399,7 +2399,8 @@
let(:gitlab_shell) { Gitlab::Shell.new }
let(:hash) { '6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b' }
before do
stub_application_setting(hashed_storage_enabled: true)
allow(Digest::SHA2).to receive(:hexdigest) { hash }
end
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment