Read about our upcoming Code of Conduct on this issue

Commit 830ab2f4 authored by Francisco Javier López's avatar Francisco Javier López Committed by Douwe Maan
Browse files

Extend API for importing a project export with overwrite support

parent 640d3b313940
......@@ -286,6 +286,10 @@ def hashed_storage?(_feature)
false
end
def refresh_project_authorizations
refresh_members_authorized_projects(blocking: false)
end
private
def update_two_factor_requirement
......
......@@ -252,6 +252,10 @@ def features
[]
end
def refresh_project_authorizations
owner.refresh_authorized_projects
end
private
def path_or_parent_changed?
......
......@@ -1472,7 +1472,9 @@ def write_repository_config(gl_full_path: full_path)
end
def rename_repo_notify!
send_move_instructions(full_path_was)
# When we import a project overwriting the original project, there
# is a move operation. In that case we don't want to send the instructions.
send_move_instructions(full_path_was) unless started?
expires_full_path_cache
self.old_path_with_namespace = full_path_was
......
module Projects
class BaseMoveRelationsService < BaseService
attr_reader :source_project
def execute(source_project, remove_remaining_elements: true)
return if source_project.blank?
@source_project = source_project
true
end
private
def prepare_relation(relation, id_param = :id)
if Gitlab::Database.postgresql?
relation
else
relation.model.where("#{id_param}": relation.pluck(id_param))
end
end
end
end
......@@ -46,6 +46,20 @@ def execute
raise
end
def attempt_repositories_rollback
return unless @project
flush_caches(@project)
unless mv_repository(removal_path(repo_path), repo_path)
raise_error('Failed to restore project repository. Please contact the administrator.')
end
unless mv_repository(removal_path(wiki_path), wiki_path)
raise_error('Failed to restore wiki repository. Please contact the administrator.')
end
end
private
def repo_path
......@@ -70,12 +84,9 @@ def remove_repository(path)
# Skip repository removal. We use this flag when remove user or group
return true if params[:skip_repo] == true
# There is a possibility project does not have repository or wiki
return true unless gitlab_shell.exists?(project.repository_storage_path, path + '.git')
new_path = removal_path(path)
if gitlab_shell.mv_repository(project.repository_storage_path, path, new_path)
if mv_repository(path, new_path)
log_info("Repository \"#{path}\" moved to \"#{new_path}\"")
project.run_after_commit do
......@@ -87,6 +98,13 @@ def remove_repository(path)
end
end
def mv_repository(from_path, to_path)
# There is a possibility project does not have repository or wiki
return true unless gitlab_shell.exists?(project.repository_storage_path, from_path + '.git')
gitlab_shell.mv_repository(project.repository_storage_path, from_path, to_path)
end
def attempt_rollback(project, message)
return unless project
......
......@@ -15,9 +15,18 @@ def execute
file = params.delete(:file)
FileUtils.copy_entry(file.path, import_upload_path)
@overwrite = params.delete(:overwrite)
data = {}
data[:override_params] = @override_params if @override_params
if overwrite_project?
data[:original_path] = params[:path]
params[:path] += "-#{tmp_filename}"
end
params[:import_type] = 'gitlab_project'
params[:import_source] = import_upload_path
params[:import_data] = { data: { override_params: @override_params } } if @override_params
params[:import_data] = { data: data } if data.present?
::Projects::CreateService.new(current_user, params).execute
end
......@@ -31,5 +40,17 @@ def import_upload_path
def tmp_filename
SecureRandom.hex
end
def overwrite_project?
@overwrite && project_with_same_full_path?
end
def project_with_same_full_path?
Project.find_by_full_path("#{current_namespace.full_path}/#{params[:path]}").present?
end
def current_namespace
@current_namespace ||= Namespace.find_by(id: params[:namespace_id])
end
end
end
module Projects
class MoveAccessService < BaseMoveRelationsService
def execute(source_project, remove_remaining_elements: true)
return unless super
@project.with_transaction_returning_status do
if @project.namespace != source_project.namespace
@project.run_after_commit do
source_project.namespace.refresh_project_authorizations
self.namespace.refresh_project_authorizations
end
end
::Projects::MoveProjectMembersService.new(@project, @current_user)
.execute(source_project, remove_remaining_elements: remove_remaining_elements)
::Projects::MoveProjectGroupLinksService.new(@project, @current_user)
.execute(source_project, remove_remaining_elements: remove_remaining_elements)
::Projects::MoveProjectAuthorizationsService.new(@project, @current_user)
.execute(source_project, remove_remaining_elements: remove_remaining_elements)
success
end
end
end
end
module Projects
class MoveDeployKeysProjectsService < BaseMoveRelationsService
def execute(source_project, remove_remaining_elements: true)
return unless super
Project.transaction(requires_new: true) do
move_deploy_keys_projects
remove_remaining_deploy_keys_projects if remove_remaining_elements
success
end
end
private
def move_deploy_keys_projects
prepare_relation(non_existent_deploy_keys_projects)
.update_all(project_id: @project.id)
end
def non_existent_deploy_keys_projects
source_project.deploy_keys_projects
.joins(:deploy_key)
.where.not(keys: { fingerprint: @project.deploy_keys.select(:fingerprint) })
end
def remove_remaining_deploy_keys_projects
source_project.deploy_keys_projects.destroy_all
end
end
end
module Projects
class MoveForksService < BaseMoveRelationsService
def execute(source_project, remove_remaining_elements: true)
return unless super && source_project.fork_network
Project.transaction(requires_new: true) do
move_forked_project_links
move_fork_network_members
update_root_project
refresh_forks_count
success
end
end
private
def move_forked_project_links
# Update ancestor
ForkedProjectLink.where(forked_to_project: source_project)
.update_all(forked_to_project_id: @project.id)
# Update the descendants
ForkedProjectLink.where(forked_from_project: source_project)
.update_all(forked_from_project_id: @project.id)
end
def move_fork_network_members
ForkNetworkMember.where(project: source_project).update_all(project_id: @project.id)
ForkNetworkMember.where(forked_from_project: source_project).update_all(forked_from_project_id: @project.id)
end
def update_root_project
# Update root network project
ForkNetwork.where(root_project: source_project).update_all(root_project_id: @project.id)
end
def refresh_forks_count
Projects::ForksCountService.new(@project).refresh_cache
end
end
end
module Projects
class MoveLfsObjectsProjectsService < BaseMoveRelationsService
def execute(source_project, remove_remaining_elements: true)
return unless super
Project.transaction(requires_new: true) do
move_lfs_objects_projects
remove_remaining_lfs_objects_project if remove_remaining_elements
success
end
end
private
def move_lfs_objects_projects
prepare_relation(non_existent_lfs_objects_projects)
.update_all(project_id: @project.lfs_storage_project.id)
end
def remove_remaining_lfs_objects_project
source_project.lfs_objects_projects.destroy_all
end
def non_existent_lfs_objects_projects
source_project.lfs_objects_projects.where.not(lfs_object: @project.lfs_objects)
end
end
end
module Projects
class MoveNotificationSettingsService < BaseMoveRelationsService
def execute(source_project, remove_remaining_elements: true)
return unless super
Project.transaction(requires_new: true) do
move_notification_settings
remove_remaining_notification_settings if remove_remaining_elements
success
end
end
private
def move_notification_settings
prepare_relation(non_existent_notifications)
.update_all(source_id: @project.id)
end
# Remove remaining notification settings from source_project
def remove_remaining_notification_settings
source_project.notification_settings.destroy_all
end
# Get users of current notification_settings
def users_in_target_project
@project.notification_settings.select(:user_id)
end
# Look for notification_settings in source_project that are not in the target project
def non_existent_notifications
source_project.notification_settings
.select(:id)
.where.not(user_id: users_in_target_project)
end
end
end
# NOTE: This service cannot be used directly because it is part of a
# a bigger process. Instead, use the service MoveAccessService which moves
# project memberships, project group links, authorizations and refreshes
# the authorizations if neccessary
module Projects
class MoveProjectAuthorizationsService < BaseMoveRelationsService
def execute(source_project, remove_remaining_elements: true)
return unless super
Project.transaction(requires_new: true) do
move_project_authorizations
remove_remaining_authorizations if remove_remaining_elements
success
end
end
private
def move_project_authorizations
prepare_relation(non_existent_authorization, :user_id)
.update_all(project_id: @project.id)
end
def remove_remaining_authorizations
# I think because the Project Authorization table does not have a primary key
# it brings a lot a problems/bugs. First, Rails raises PG::SyntaxException if we use
# destroy_all instead of delete_all.
source_project.project_authorizations.delete_all(:delete_all)
end
# Look for authorizations in source_project that are not in the target project
def non_existent_authorization
source_project.project_authorizations
.select(:user_id)
.where.not(user: @project.authorized_users)
end
end
end
# NOTE: This service cannot be used directly because it is part of a
# a bigger process. Instead, use the service MoveAccessService which moves
# project memberships, project group links, authorizations and refreshes
# the authorizations if neccessary
module Projects
class MoveProjectGroupLinksService < BaseMoveRelationsService
def execute(source_project, remove_remaining_elements: true)
return unless super
Project.transaction(requires_new: true) do
move_group_links
remove_remaining_project_group_links if remove_remaining_elements
success
end
end
private
def move_group_links
prepare_relation(non_existent_group_links)
.update_all(project_id: @project.id)
end
# Remove remaining project group links from source_project
def remove_remaining_project_group_links
source_project.reload.project_group_links.destroy_all
end
def group_links_in_target_project
@project.project_group_links.select(:group_id)
end
# Look for groups in source_project that are not in the target project
def non_existent_group_links
source_project.project_group_links
.where.not(group_id: group_links_in_target_project)
end
end
end
# NOTE: This service cannot be used directly because it is part of a
# a bigger process. Instead, use the service MoveAccessService which moves
# project memberships, project group links, authorizations and refreshes
# the authorizations if neccessary
module Projects
class MoveProjectMembersService < BaseMoveRelationsService
def execute(source_project, remove_remaining_elements: true)
return unless super
Project.transaction(requires_new: true) do
move_project_members
remove_remaining_members if remove_remaining_elements
success
end
end
private
def move_project_members
prepare_relation(non_existent_members).update_all(source_id: @project.id)
end
def remove_remaining_members
# Remove remaining members and authorizations from source_project
source_project.project_members.destroy_all
end
def project_members_in_target_project
@project.project_members.select(:user_id)
end
# Look for members in source_project that are not in the target project
def non_existent_members
source_project.members
.select(:id)
.where.not(user_id: @project.project_members.select(:user_id))
end
end
end
module Projects
class MoveUsersStarProjectsService < BaseMoveRelationsService
def execute(source_project, remove_remaining_elements: true)
return unless super
user_stars = source_project.users_star_projects
return unless user_stars.any?
Project.transaction(requires_new: true) do
user_stars.update_all(project_id: @project.id)
Project.reset_counters @project.id, :users_star_projects
Project.reset_counters source_project.id, :users_star_projects
success
end
end
end
end
module Projects
class OverwriteProjectService < BaseService
def execute(source_project)
return unless source_project && source_project.namespace == @project.namespace
Project.transaction do
move_before_destroy_relationships(source_project)
destroy_old_project(source_project)
rename_project(source_project.name, source_project.path)
@project
end
# Projects::DestroyService can raise Exceptions, but we don't want
# to pass that kind of exception to the caller. Instead, we change it
# for a StandardError exception
rescue Exception => e # rubocop:disable Lint/RescueException
attempt_restore_repositories(source_project)
if e.class == Exception
raise StandardError, e.message
else
raise
end
end
private
def move_before_destroy_relationships(source_project)
options = { remove_remaining_elements: false }
::Projects::MoveUsersStarProjectsService.new(@project, @current_user).execute(source_project, options)
::Projects::MoveAccessService.new(@project, @current_user).execute(source_project, options)
::Projects::MoveDeployKeysProjectsService.new(@project, @current_user).execute(source_project, options)
::Projects::MoveNotificationSettingsService.new(@project, @current_user).execute(source_project, options)
::Projects::MoveForksService.new(@project, @current_user).execute(source_project, options)
::Projects::MoveLfsObjectsProjectsService.new(@project, @current_user).execute(source_project, options)
add_source_project_to_fork_network(source_project)
end
def destroy_old_project(source_project)
# Delete previous project (synchronously) and unlink relations
::Projects::DestroyService.new(source_project, @current_user).execute
end
def rename_project(name, path)
# Update de project's name and path to the original name/path
::Projects::UpdateService.new(@project,
@current_user,
{ name: name, path: path })
.execute
end
def attempt_restore_repositories(project)
::Projects::DestroyService.new(project, @current_user).attempt_repositories_rollback
end
def add_source_project_to_fork_network(source_project)
return unless @project.fork_network
# Because he have moved all references in the fork network from the source_project
# we won't be able to query the database (only through its cached data),
# for its former relationships. That's why we're adding it to the network
# as a fork of the target project
ForkNetworkMember.create!(fork_network: @project.fork_network,
project: source_project,
forked_from_project: @project)
end
end
end
---
title: Extend API for importing a project export with overwrite support
merge_request: 17883
author:
type: added
......@@ -111,6 +111,7 @@ POST /projects/import
| `namespace` | integer/string | no | The ID or path of the namespace that the project will be imported to. Defaults to the current user's namespace |
| `file` | string | yes | The file to be uploaded |
| `path` | string | yes | Name and path for new project |
| `overwrite` | boolean | no | If there is a project with the same path the import will overwrite it. Default to false |
| `override_params` | Hash | no | Supports all fields defined in the [Project API](projects.md)] |
The override params passed will take precendence over all values defined inside the export file.
......
......@@ -26,6 +26,7 @@ def validate_file!
requires :path, type: String, desc: 'The new project path and name'
requires :file, type: File, desc: 'The project export file to be imported'
optional :namespace, type: String, desc: "The ID or name of the namespace that the project will be imported into. Defaults to the current user's namespace."
optional :overwrite, type: Boolean, default: false, desc: 'If there is a project in the same namespace and with the same name overwrite it'
optional :override_params,
type: Hash,
desc: 'New project params to override values in the export' do
......@@ -50,7 +51,8 @@ def validate_file!
project_params = {
path: import_params[:path],
namespace_id: namespace.id,
file: import_params[:file]['tempfile']
file: import_params[:file]['tempfile'],
overwrite: import_params[:overwrite]
}
override_params = import_params.delete(:override_params)
......
module Gitlab
module ImportExport
class Importer
include Gitlab::Allowable
include Gitlab::Utils::StrongMemoize
def self.imports_repository?
true
end
......@@ -13,12 +16,14 @@ def initialize(project)
end
def execute
if import_file && check_version! && restorers.all?(&:restore)
if import_file && check_version! && restorers.all?(&:restore) && overwrite_project
project_tree.restored_project
else
raise Projects::ImportService::Error.new(@shared.errors.join(', '))
end
rescue => e
raise Projects::ImportService::Error.new(e.message)
ensure
remove_import_file
end
......@@ -26,7 +31,7 @@ def execute
def restorers
[repo_restorer, wiki_restorer, project_tree, avatar_restorer,
uploads_restorer, lfs_restorer]
uploads_restorer, lfs_restorer, statistics_restorer]
end
def import_file
...