Read about our upcoming Code of Conduct on this issue

Commit 98615318 authored by Gabriel Mazetto's avatar Gabriel Mazetto
Browse files

Move storage specific code from Namespace and Project to concerns

parent dde9ae2472f2
module Storage
module LegacyNamespace
extend ActiveSupport::Concern
def move_dir
if any_project_has_container_registry_tags?
raise Gitlab::UpdatePathError.new('Namespace cannot be moved, because at least one project has tags in container registry')
end
# Move the namespace directory in all storages paths used by member projects
repository_storage_paths.each do |repository_storage_path|
# Ensure old directory exists before moving it
gitlab_shell.add_namespace(repository_storage_path, full_path_was)
unless gitlab_shell.mv_namespace(repository_storage_path, full_path_was, full_path)
Rails.logger.error "Exception moving path #{repository_storage_path} from #{full_path_was} to #{full_path}"
# if we cannot move namespace directory we should rollback
# db changes in order to prevent out of sync between db and fs
raise Gitlab::UpdatePathError.new('namespace directory cannot be moved')
end
end
Gitlab::UploadsTransfer.new.rename_namespace(full_path_was, full_path)
Gitlab::PagesTransfer.new.rename_namespace(full_path_was, full_path)
remove_exports!
# If repositories moved successfully we need to
# send update instructions to users.
# However we cannot allow rollback since we moved namespace dir
# So we basically we mute exceptions in next actions
begin
send_update_instructions
true
rescue
# Returning false does not rollback after_* transaction but gives
# us information about failing some of tasks
false
end
end
# Hooks
# Save the storage paths before the projects are destroyed to use them on after destroy
def prepare_for_destroy
old_repository_storage_paths
end
private
def send_update_instructions
projects.each do |project|
project.send_move_instructions("#{full_path_was}/#{project.path}")
end
end
def old_repository_storage_paths
@old_repository_storage_paths ||= repository_storage_paths
end
def repository_storage_paths
# We need to get the storage paths for all the projects, even the ones that are
# pending delete. Unscoping also get rids of the default order, which causes
# problems with SELECT DISTINCT.
Project.unscoped do
all_projects.select('distinct(repository_storage)').to_a.map(&:repository_storage_path)
end
end
def rm_dir
# Remove the namespace directory in all storages paths used by member projects
old_repository_storage_paths.each do |repository_storage_path|
# Move namespace directory into trash.
# We will remove it later async
new_path = "#{full_path}+#{id}+deleted"
if gitlab_shell.mv_namespace(repository_storage_path, full_path, new_path)
message = "Namespace directory \"#{full_path}\" moved to \"#{new_path}\""
Gitlab::AppLogger.info message
# Remove namespace directroy async with delay so
# GitLab has time to remove all projects first
run_after_commit do
GitlabShellWorker.perform_in(5.minutes, :rm_namespace, repository_storage_path, new_path)
end
end
end
remove_exports!
end
def remove_exports!
Gitlab::Popen.popen(%W(find #{export_path} -not -path #{export_path} -delete))
end
def export_path
File.join(Gitlab::ImportExport.storage_path, full_path_was)
end
def full_path_was
if parent
parent.full_path + '/' + path_was
else
path_was
end
end
end
end
module Storage
module LegacyProject
extend ActiveSupport::Concern
def disk_path
full_path
end
def ensure_dir_exist
gitlab_shell.add_namespace(repository_storage_path, namespace.full_path)
end
def rename_repo
path_was = previous_changes['path'].first
old_path_with_namespace = File.join(namespace.full_path, path_was)
new_path_with_namespace = File.join(namespace.full_path, path)
Rails.logger.error "Attempting to rename #{old_path_with_namespace} -> #{new_path_with_namespace}"
if has_container_registry_tags?
Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry tags are present!"
# we currently doesn't support renaming repository if it contains images in container registry
raise StandardError.new('Project cannot be renamed, because images are present in its container registry')
end
expire_caches_before_rename(old_path_with_namespace)
if gitlab_shell.mv_repository(repository_storage_path, old_path_with_namespace, new_path_with_namespace)
# If repository moved successfully we need to send update instructions to users.
# However we cannot allow rollback since we moved repository
# So we basically we mute exceptions in next actions
begin
gitlab_shell.mv_repository(repository_storage_path, "#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki")
send_move_instructions(old_path_with_namespace)
expires_full_path_cache
@old_path_with_namespace = old_path_with_namespace
SystemHooksService.new.execute_hooks_for(self, :rename)
@repository = nil
rescue => e
Rails.logger.error "Exception renaming #{old_path_with_namespace} -> #{new_path_with_namespace}: #{e}"
# Returning false does not rollback after_* transaction but gives
# us information about failing some of tasks
false
end
else
Rails.logger.error "Repository could not be renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}"
# if we cannot move namespace directory we should rollback
# db changes in order to prevent out of sync between db and fs
raise StandardError.new('repository cannot be renamed')
end
Gitlab::AppLogger.info "Project was renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}"
Gitlab::UploadsTransfer.new.rename_project(path_was, path, namespace.full_path)
Gitlab::PagesTransfer.new.rename_project(path_was, path, namespace.full_path)
end
def create_repository(force: false)
# Forked import is handled asynchronously
return if forked? && !force
if gitlab_shell.add_repository(repository_storage_path, path_with_namespace)
repository.after_create
true
else
errors.add(:base, 'Failed to create repository via gitlab-shell')
false
end
end
end
end
...@@ -8,6 +8,7 @@ class Namespace < ActiveRecord::Base ...@@ -8,6 +8,7 @@ class Namespace < ActiveRecord::Base
include Gitlab::VisibilityLevel include Gitlab::VisibilityLevel
include Routable include Routable
include AfterCommitQueue include AfterCommitQueue
include Storage::LegacyNamespace
# Prevent users from creating unreasonably deep level of nesting. # Prevent users from creating unreasonably deep level of nesting.
# The number 20 was taken based on maximum nesting level of # The number 20 was taken based on maximum nesting level of
...@@ -41,10 +42,11 @@ class Namespace < ActiveRecord::Base ...@@ -41,10 +42,11 @@ class Namespace < ActiveRecord::Base
delegate :name, to: :owner, allow_nil: true, prefix: true delegate :name, to: :owner, allow_nil: true, prefix: true
after_update :move_dir, if: :path_changed?
after_commit :refresh_access_of_projects_invited_groups, on: :update, if: -> { previous_changes.key?('share_with_group_lock') } after_commit :refresh_access_of_projects_invited_groups, on: :update, if: -> { previous_changes.key?('share_with_group_lock') }
# Save the storage paths before the projects are destroyed to use them on after destroy # Legacy Storage specific hooks
after_update :move_dir, if: :path_changed?
before_destroy(prepend: true) { prepare_for_destroy } before_destroy(prepend: true) { prepare_for_destroy }
after_destroy :rm_dir after_destroy :rm_dir
...@@ -118,53 +120,10 @@ def human_name ...@@ -118,53 +120,10 @@ def human_name
owner_name owner_name
end end
def move_dir
if any_project_has_container_registry_tags?
raise Gitlab::UpdatePathError.new('Namespace cannot be moved, because at least one project has tags in container registry')
end
# Move the namespace directory in all storages paths used by member projects
repository_storage_paths.each do |repository_storage_path|
# Ensure old directory exists before moving it
gitlab_shell.add_namespace(repository_storage_path, full_path_was)
unless gitlab_shell.mv_namespace(repository_storage_path, full_path_was, full_path)
Rails.logger.error "Exception moving path #{repository_storage_path} from #{full_path_was} to #{full_path}"
# if we cannot move namespace directory we should rollback
# db changes in order to prevent out of sync between db and fs
raise Gitlab::UpdatePathError.new('namespace directory cannot be moved')
end
end
Gitlab::UploadsTransfer.new.rename_namespace(full_path_was, full_path)
Gitlab::PagesTransfer.new.rename_namespace(full_path_was, full_path)
remove_exports!
# If repositories moved successfully we need to
# send update instructions to users.
# However we cannot allow rollback since we moved namespace dir
# So we basically we mute exceptions in next actions
begin
send_update_instructions
rescue
# Returning false does not rollback after_* transaction but gives
# us information about failing some of tasks
false
end
end
def any_project_has_container_registry_tags? def any_project_has_container_registry_tags?
all_projects.any?(&:has_container_registry_tags?) all_projects.any?(&:has_container_registry_tags?)
end end
def send_update_instructions
projects.each do |project|
project.send_move_instructions("#{full_path_was}/#{project.path}")
end
end
def kind def kind
type == 'Group' ? 'group' : 'user' type == 'Group' ? 'group' : 'user'
end end
...@@ -206,14 +165,6 @@ def parent_changed? ...@@ -206,14 +165,6 @@ def parent_changed?
parent_id_changed? parent_id_changed?
end end
def prepare_for_destroy
old_repository_storage_paths
end
def old_repository_storage_paths
@old_repository_storage_paths ||= repository_storage_paths
end
# Includes projects from this namespace and projects from all subgroups # Includes projects from this namespace and projects from all subgroups
# that belongs to this namespace # that belongs to this namespace
def all_projects def all_projects
...@@ -232,37 +183,6 @@ def soft_delete_without_removing_associations ...@@ -232,37 +183,6 @@ def soft_delete_without_removing_associations
private private
def repository_storage_paths
# We need to get the storage paths for all the projects, even the ones that are
# pending delete. Unscoping also get rids of the default order, which causes
# problems with SELECT DISTINCT.
Project.unscoped do
all_projects.select('distinct(repository_storage)').to_a.map(&:repository_storage_path)
end
end
def rm_dir
# Remove the namespace directory in all storages paths used by member projects
old_repository_storage_paths.each do |repository_storage_path|
# Move namespace directory into trash.
# We will remove it later async
new_path = "#{full_path}+#{id}+deleted"
if gitlab_shell.mv_namespace(repository_storage_path, full_path, new_path)
message = "Namespace directory \"#{full_path}\" moved to \"#{new_path}\""
Gitlab::AppLogger.info message
# Remove namespace directroy async with delay so
# GitLab has time to remove all projects first
run_after_commit do
GitlabShellWorker.perform_in(5.minutes, :rm_namespace, repository_storage_path, new_path)
end
end
end
remove_exports!
end
def refresh_access_of_projects_invited_groups def refresh_access_of_projects_invited_groups
Group Group
.joins(project_group_links: :project) .joins(project_group_links: :project)
...@@ -270,22 +190,6 @@ def refresh_access_of_projects_invited_groups ...@@ -270,22 +190,6 @@ def refresh_access_of_projects_invited_groups
.find_each(&:refresh_members_authorized_projects) .find_each(&:refresh_members_authorized_projects)
end end
def remove_exports!
Gitlab::Popen.popen(%W(find #{export_path} -not -path #{export_path} -delete))
end
def export_path
File.join(Gitlab::ImportExport.storage_path, full_path_was)
end
def full_path_was
if parent
parent.full_path + '/' + path_was
else
path_was
end
end
def nesting_level_allowed def nesting_level_allowed
if ancestors.count > Group::NUMBER_OF_ANCESTORS_ALLOWED if ancestors.count > Group::NUMBER_OF_ANCESTORS_ALLOWED
errors.add(:parent_id, "has too deep level of nesting") errors.add(:parent_id, "has too deep level of nesting")
......
...@@ -17,6 +17,7 @@ class Project < ActiveRecord::Base ...@@ -17,6 +17,7 @@ class Project < ActiveRecord::Base
include ProjectFeaturesCompatibility include ProjectFeaturesCompatibility
include SelectForProjectAuthorization include SelectForProjectAuthorization
include Routable include Routable
include Storage::LegacyProject
extend Gitlab::ConfigHelper extend Gitlab::ConfigHelper
...@@ -45,7 +46,6 @@ class Project < ActiveRecord::Base ...@@ -45,7 +46,6 @@ class Project < ActiveRecord::Base
after_create :ensure_dir_exist after_create :ensure_dir_exist
after_create :create_project_feature, unless: :project_feature after_create :create_project_feature, unless: :project_feature
after_save :ensure_dir_exist, if: :namespace_id_changed?
after_save :update_project_statistics, if: :namespace_id_changed? after_save :update_project_statistics, if: :namespace_id_changed?
# set last_activity_at to the same as created_at # set last_activity_at to the same as created_at
...@@ -67,6 +67,10 @@ def set_last_repository_updated_at ...@@ -67,6 +67,10 @@ def set_last_repository_updated_at
after_validation :check_pending_delete after_validation :check_pending_delete
# Legacy Storage specific hooks
after_save :ensure_dir_exist, if: :namespace_id_changed?
acts_as_taggable acts_as_taggable
attr_accessor :new_default_branch attr_accessor :new_default_branch
...@@ -974,56 +978,6 @@ def personal? ...@@ -974,56 +978,6 @@ def personal?
!group !group
end end
def rename_repo
path_was = previous_changes['path'].first
old_path_with_namespace = File.join(namespace.full_path, path_was)
new_path_with_namespace = File.join(namespace.full_path, path)
Rails.logger.error "Attempting to rename #{old_path_with_namespace} -> #{new_path_with_namespace}"
if has_container_registry_tags?
Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry tags are present!"
# we currently doesn't support renaming repository if it contains images in container registry
raise StandardError.new('Project cannot be renamed, because images are present in its container registry')
end
expire_caches_before_rename(old_path_with_namespace)
if gitlab_shell.mv_repository(repository_storage_path, old_path_with_namespace, new_path_with_namespace)
# If repository moved successfully we need to send update instructions to users.
# However we cannot allow rollback since we moved repository
# So we basically we mute exceptions in next actions
begin
gitlab_shell.mv_repository(repository_storage_path, "#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki")
send_move_instructions(old_path_with_namespace)
expires_full_path_cache
@old_path_with_namespace = old_path_with_namespace
SystemHooksService.new.execute_hooks_for(self, :rename)
@repository = nil
rescue => e
Rails.logger.error "Exception renaming #{old_path_with_namespace} -> #{new_path_with_namespace}: #{e}"
# Returning false does not rollback after_* transaction but gives
# us information about failing some of tasks
false
end
else
Rails.logger.error "Repository could not be renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}"
# if we cannot move namespace directory we should rollback
# db changes in order to prevent out of sync between db and fs
raise StandardError.new('repository cannot be renamed')
end
Gitlab::AppLogger.info "Project was renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}"
Gitlab::UploadsTransfer.new.rename_project(path_was, path, namespace.full_path)
Gitlab::PagesTransfer.new.rename_project(path_was, path, namespace.full_path)
end
# Expires various caches before a project is renamed. # Expires various caches before a project is renamed.
def expire_caches_before_rename(old_path) def expire_caches_before_rename(old_path)
repo = Repository.new(old_path, self) repo = Repository.new(old_path, self)
...@@ -1109,19 +1063,6 @@ def origin_merge_requests ...@@ -1109,19 +1063,6 @@ def origin_merge_requests
merge_requests.where(source_project_id: self.id) merge_requests.where(source_project_id: self.id)
end end
def create_repository(force: false)
# Forked import is handled asynchronously
return if forked? && !force
if gitlab_shell.add_repository(repository_storage_path, path_with_namespace)
repository.after_create
true
else
errors.add(:base, 'Failed to create repository via gitlab-shell')
false
end
end
def ensure_repository def ensure_repository
create_repository(force: true) unless repository_exists? create_repository(force: true) unless repository_exists?
end end
...@@ -1327,10 +1268,6 @@ def remove_exports ...@@ -1327,10 +1268,6 @@ def remove_exports
status.zero? status.zero?
end end
def ensure_dir_exist
gitlab_shell.add_namespace(repository_storage_path, namespace.full_path)
end
def predefined_variables def predefined_variables
[ [
{ key: 'CI_PROJECT_ID', value: id.to_s, public: true }, { key: 'CI_PROJECT_ID', value: id.to_s, public: true },
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment