Read about our upcoming Code of Conduct on this issue

Commit 7f76422c authored by James Lopez's avatar James Lopez
Browse files

Refactor code to remove object storage flag from Import/Export

Updated docs, refactor import/export code
Fix AvatarUploader path issue
Fix project export upload webhook error
parent a8bc05ccac7d
......@@ -191,10 +191,8 @@ def export
end
def download_export
if export_project_object_storage?
if @project.export_project_object_exists?
send_upload(@project.import_export_upload.export_file)
elsif export_project_path
send_file export_project_path, disposition: 'attachment'
else
redirect_to(
edit_project_path(@project, anchor: 'js-export-project'),
......@@ -425,12 +423,4 @@ def redirect_git_extension
def whitelist_query_limiting
Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42440')
end
def export_project_path
@export_project_path ||= @project.export_project_path
end
def export_project_object_storage?
@project.export_project_object_exists?
end
end
......@@ -25,8 +25,6 @@ def move_dir
Gitlab::PagesTransfer.new.rename_namespace(full_path_was, full_path)
end
remove_exports!
# If repositories moved successfully we need to
# send update instructions to users.
# However we cannot allow rollback since we moved namespace dir
......@@ -101,8 +99,6 @@ def rm_dir
end
end
end
remove_exports!
end
def remove_legacy_exports!
......
......@@ -253,18 +253,6 @@ def full_path_was
end
end
# Exports belonging to projects with legacy storage are placed in a common
# subdirectory of the namespace, so a simple `rm -rf` is sufficient to remove
# them.
#
# Exports of projects using hashed storage are placed in a location defined
# only by the project ID, so each must be removed individually.
def remove_exports!
remove_legacy_exports!
all_projects.with_storage_feature(:repository).find_each(&:remove_exports)
end
def refresh_project_authorizations
owner.refresh_authorized_projects
end
......
......@@ -1733,16 +1733,12 @@ def export_path
import_export_shared.archive_path
end
def export_project_path
Dir.glob("#{export_path}/*export.tar.gz").max_by { |f| File.ctime(f) }
end
def export_status
if export_in_progress?
:started
elsif after_export_in_progress?
:after_export_action
elsif export_project_path || export_project_object_exists?
elsif export_project_object_exists?
:finished
else
:none
......@@ -1757,21 +1753,15 @@ def after_export_in_progress?
import_export_shared.after_export_in_progress?
end
def remove_exports(path = export_path)
if path.present?
FileUtils.rm_rf(path)
elsif export_project_object_exists?
import_export_upload.remove_export_file!
import_export_upload.save
end
end
def remove_exports
return unless export_project_object_exists?
def remove_exported_project_file
remove_exports(export_project_path)
import_export_upload.remove_export_file!
import_export_upload.save
end
def export_project_object_exists?
Gitlab::ImportExport.object_storage? && import_export_upload&.export_file&.file
import_export_upload&.export_file&.file
end
def full_path_slug
......
......@@ -18,6 +18,10 @@ def move_to_cache
false
end
def absolute_path
self.class.absolute_path(model.avatar)
end
private
def dynamic_segment
......
---
title: Update Import/Export to only use new storage uploaders logic
merge_request: 21409
author:
type: added
......@@ -9,6 +9,7 @@
> application settings (`/admin/application_settings`) under 'Import sources'.
> - The exports are stored in a temporary [shared directory][tmp] and are deleted
> every 24 hours by a specific worker.
> - ImportExport can use object storage automatically starting from GitLab 11.3
The GitLab Import/Export version can be checked by using:
......@@ -30,12 +31,6 @@ sudo gitlab-rake gitlab:import_export:data
bundle exec rake gitlab:import_export:data RAILS_ENV=production
```
In order to enable Object Storage on the Export, you can use the [feature flag][feature-flags]:
```
import_export_object_storage
```
[ce-3050]: https://gitlab.com/gitlab-org/gitlab-ce/issues/3050
[feature-flags]: https://docs.gitlab.com/ee/api/features.html
[tmp]: ../../development/shared_files.md
......@@ -21,11 +21,7 @@ class ProjectExport < Grape::API
detail 'This feature was introduced in GitLab 10.6.'
end
get ':id/export/download' do
path = user_project.export_project_path
if path
present_disk_file!(path, File.basename(path), 'application/gzip')
elsif user_project.export_project_object_exists?
if user_project.export_project_object_exists?
present_carrierwave_file!(user_project.import_export_upload.export_file)
else
render_api_error!('404 Not found or has expired', 404)
......
......@@ -40,10 +40,6 @@ def export_filename(project:)
"#{basename[0..FILENAME_LIMIT]}_export.tar.gz"
end
def object_storage?
Feature.enabled?(:import_export_object_storage)
end
def version
VERSION
end
......
......@@ -23,7 +23,7 @@ def initialize(url:, http_method: PUT_METHOD)
def strategy_execute
handle_response_error(send_file)
project.remove_exported_project_file
project.remove_exports
end
def handle_response_error(response)
......@@ -40,15 +40,11 @@ def handle_response_error(response)
def send_file
Gitlab::HTTP.public_send(http_method.downcase, url, send_file_options) # rubocop:disable GitlabSecurity/PublicSend
ensure
export_file.close if export_file && !object_storage?
export_file.close if export_file
end
def export_file
if object_storage?
project.import_export_upload.export_file.file.open
else
File.open(project.export_project_path)
end
project.import_export_upload.export_file.open
end
def send_file_options
......@@ -63,11 +59,7 @@ def headers
end
def export_size
if object_storage?
project.import_export_upload.export_file.file.size
else
File.size(project.export_project_path)
end
project.import_export_upload.export_file.file.size
end
end
end
......
......@@ -19,7 +19,7 @@ def restore
private
def avatar_export_file
@avatar_export_file ||= Dir["#{avatar_export_path}/*"].first
@avatar_export_file ||= Dir["#{avatar_export_path}/**/*"].first
end
def avatar_export_path
......
module Gitlab
module ImportExport
class AvatarSaver
include Gitlab::ImportExport::CommandLineUtil
def initialize(project:, shared:)
@project = project
@shared = shared
......@@ -14,19 +12,12 @@ def save
Gitlab::ImportExport::UploadsManager.new(
project: @project,
shared: @shared,
relative_export_path: 'avatar',
from: avatar_path
relative_export_path: 'avatar'
).save
rescue => e
@shared.error(e)
false
end
private
def avatar_path
@project.avatar.path
end
end
end
end
......@@ -92,8 +92,6 @@ def wiki_repo_path
end
def remove_import_file
return unless Gitlab::ImportExport.object_storage?
upload = @project.import_export_upload
return unless upload&.import_file&.file
......
......@@ -199,7 +199,7 @@ def reader
end
def excluded_keys_for_relation(relation)
@reader.attributes_finder.find_excluded_keys(relation)
reader.attributes_finder.find_excluded_keys(relation)
end
end
end
......
......@@ -18,7 +18,7 @@ def save
Rails.logger.info("Saved project export #{archive_file}")
save_on_object_storage if use_object_storage?
save_on_object_storage
else
@shared.error(Gitlab::ImportExport::Error.new(error_message))
false
......@@ -27,10 +27,8 @@ def save
@shared.error(e)
false
ensure
if use_object_storage?
remove_archive
remove_export_path
end
remove_archive
remove_export_path
end
private
......@@ -59,12 +57,8 @@ def save_on_object_storage
upload.save!
end
def use_object_storage?
Gitlab::ImportExport.object_storage?
end
def error_message
"Unable to save #{archive_file} into #{@shared.export_path}. Object storage enabled: #{use_object_storage?}"
"Unable to save #{archive_file} into #{@shared.export_path}."
end
end
end
......
......@@ -5,18 +5,13 @@ class UploadsManager
UPLOADS_BATCH_SIZE = 100
def initialize(project:, shared:, relative_export_path: 'uploads', from: nil)
def initialize(project:, shared:, relative_export_path: 'uploads')
@project = project
@shared = shared
@relative_export_path = relative_export_path
@from = from || default_uploads_path
end
def save
if File.file?(@from) && @relative_export_path == 'avatar'
copy_files(@from, File.join(uploads_export_path, @project.avatar.filename))
end
copy_project_uploads
true
......@@ -55,17 +50,11 @@ def copy_project_uploads
copy_files(uploader.absolute_path, File.join(uploads_export_path, uploader.upload.path))
else
next unless Gitlab::ImportExport.object_storage?
download_and_copy(uploader)
end
end
end
def default_uploads_path
FileUploader.absolute_base_dir(@project)
end
def uploads_export_path
@uploads_export_path ||= File.join(@shared.export_path, @relative_export_path)
end
......
......@@ -2,30 +2,14 @@ module Gitlab
module ImportExport
class UploadsRestorer < UploadsSaver
def restore
if Gitlab::ImportExport.object_storage?
Gitlab::ImportExport::UploadsManager.new(
project: @project,
shared: @shared
).restore
elsif File.directory?(uploads_export_path)
copy_files(uploads_export_path, uploads_path)
true
else
true # Proceed without uploads
end
Gitlab::ImportExport::UploadsManager.new(
project: @project,
shared: @shared
).restore
rescue => e
@shared.error(e)
false
end
def uploads_path
FileUploader.absolute_base_dir(@project)
end
def uploads_export_path
@uploads_export_path ||= File.join(@shared.export_path, 'uploads')
end
end
end
end
module Gitlab
module ImportExport
class UploadsSaver
include Gitlab::ImportExport::CommandLineUtil
def initialize(project:, shared:)
@project = project
@shared = shared
......
module Gitlab
module TemplateHelper
include Gitlab::Utils::StrongMemoize
def prepare_template_environment(file)
return unless file
if Gitlab::ImportExport.object_storage?
params[:import_export_upload] = ImportExportUpload.new(import_file: file)
else
FileUtils.mkdir_p(File.dirname(import_upload_path))
FileUtils.copy_entry(file.path, import_upload_path)
params[:import_source] = import_upload_path
end
end
def import_upload_path
strong_memoize(:import_upload_path) do
Gitlab::ImportExport.import_upload_path(filename: tmp_filename)
end
params[:import_export_upload] = ImportExportUpload.new(import_file: file)
end
def tmp_filename
......
......@@ -6,6 +6,8 @@ namespace :gitlab do
desc "GitLab | Update project templates"
task :update_project_templates do
include Gitlab::ImportExport::CommandLineUtil
if Rails.env.production?
puts "This rake task is not meant fo production instances".red
exit(1)
......@@ -52,7 +54,7 @@ namespace :gitlab do
end
Projects::ImportExport::ExportService.new(project, admin).execute
FileUtils.cp(project.export_project_path, template.archive_path)
download_or_copy_upload(project.import_export_upload.export_file, template.archive_path)
Projects::DestroyService.new(admin, project).execute
puts "Exported #{template.name}".green
end
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment