Commit c42197ad authored by James Lopez's avatar James Lopez
Browse files

Update Import/Export to use object storage (based on aa feature flag)

parent 7df607327bda
...@@ -2,6 +2,7 @@ class ProjectsController < Projects::ApplicationController ...@@ -2,6 +2,7 @@ class ProjectsController < Projects::ApplicationController
include IssuableCollections include IssuableCollections
include ExtractsPath include ExtractsPath
include PreviewMarkdown include PreviewMarkdown
include SendFileUpload
before_action :whitelist_query_limiting, only: [:create] before_action :whitelist_query_limiting, only: [:create]
before_action :authenticate_user!, except: [:index, :show, :activity, :refs] before_action :authenticate_user!, except: [:index, :show, :activity, :refs]
...@@ -188,9 +189,9 @@ def export ...@@ -188,9 +189,9 @@ def export
end end
def download_export def download_export
export_project_path = @project.export_project_path if export_project_object_storage?
send_upload(@project.import_export_upload.export_file)
if export_project_path elsif export_project_path
send_file export_project_path, disposition: 'attachment' send_file export_project_path, disposition: 'attachment'
else else
redirect_to( redirect_to(
...@@ -265,8 +266,6 @@ def refs ...@@ -265,8 +266,6 @@ def refs
render json: options.to_json render json: options.to_json
end end
private
# Render project landing depending of which features are available # Render project landing depending of which features are available
# So if page is not availble in the list it renders the next page # So if page is not availble in the list it renders the next page
# #
...@@ -424,4 +423,12 @@ def redirect_git_extension ...@@ -424,4 +423,12 @@ def redirect_git_extension
def whitelist_query_limiting def whitelist_query_limiting
Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42440') Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42440')
end end
def export_project_path
@export_project_path ||= @project.export_project_path
end
def export_project_object_storage?
@project.export_project_object_exists?
end
end end
class ImportExportUpload < ActiveRecord::Base
include WithUploads
include ObjectStorage::BackgroundMove
belongs_to :project
mount_uploader :import_file, ImportExportUploader
mount_uploader :export_file, ImportExportUploader
def retrieve_upload(_identifier, paths)
Upload.find_by(model: self, path: paths)
end
end
...@@ -171,6 +171,7 @@ class Project < ActiveRecord::Base ...@@ -171,6 +171,7 @@ class Project < ActiveRecord::Base
has_one :fork_network, through: :fork_network_member has_one :fork_network, through: :fork_network_member
has_one :import_state, autosave: true, class_name: 'ProjectImportState', inverse_of: :project has_one :import_state, autosave: true, class_name: 'ProjectImportState', inverse_of: :project
has_one :import_export_upload, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
# Merge Requests for target project should be removed with it # Merge Requests for target project should be removed with it
has_many :merge_requests, foreign_key: 'target_project_id' has_many :merge_requests, foreign_key: 'target_project_id'
...@@ -1712,7 +1713,7 @@ def export_status ...@@ -1712,7 +1713,7 @@ def export_status
:started :started
elsif after_export_in_progress? elsif after_export_in_progress?
:after_export_action :after_export_action
elsif export_project_path elsif export_project_path || export_project_object_exists?
:finished :finished
else else
:none :none
...@@ -1727,16 +1728,21 @@ def after_export_in_progress? ...@@ -1727,16 +1728,21 @@ def after_export_in_progress?
import_export_shared.after_export_in_progress? import_export_shared.after_export_in_progress?
end end
def remove_exports def remove_exports(path = export_path)
return nil unless export_path.present? if path.present?
FileUtils.rm_rf(path)
FileUtils.rm_rf(export_path) elsif export_project_object_exists?
import_export_upload.remove_export_file!
import_export_upload.save
end
end end
def remove_exported_project_file def remove_exported_project_file
return unless export_project_path.present? remove_exports(export_project_path)
end
FileUtils.rm_f(export_project_path) def export_project_object_exists?
Gitlab::ImportExport.object_storage? && import_export_upload&.export_file&.file
end end
def full_path_slug def full_path_slug
......
...@@ -10,7 +10,9 @@ def initialize(mmin = LAST_MODIFIED_TIME_IN_MINUTES) ...@@ -10,7 +10,9 @@ def initialize(mmin = LAST_MODIFIED_TIME_IN_MINUTES)
def execute def execute
Gitlab::Metrics.measure(:import_export_clean_up) do Gitlab::Metrics.measure(:import_export_clean_up) do
next unless File.directory?(path) clean_up_export_object_files
break unless File.directory?(path)
clean_up_export_files clean_up_export_files
end end
...@@ -21,4 +23,11 @@ def execute ...@@ -21,4 +23,11 @@ def execute
def clean_up_export_files def clean_up_export_files
Gitlab::Popen.popen(%W(find #{path} -not -path #{path} -mmin +#{mmin} -delete)) Gitlab::Popen.popen(%W(find #{path} -not -path #{path} -mmin +#{mmin} -delete))
end end
def clean_up_export_object_files
ImportExportUpload.where('updated_at < ?', mmin.minutes.ago).each do |upload|
upload.remove_export_file!
upload.save!
end
end
end end
class ImportExportUploader < AttachmentUploader
EXTENSION_WHITELIST = %w[tar.gz].freeze
def extension_whitelist
EXTENSION_WHITELIST
end
def move_to_store
true
end
def move_to_cache
false
end
end
...@@ -31,7 +31,7 @@ ...@@ -31,7 +31,7 @@
%li Any encrypted tokens %li Any encrypted tokens
%p %p
Once the exported file is ready, you will receive a notification email with a download link, or you can download it from this page. Once the exported file is ready, you will receive a notification email with a download link, or you can download it from this page.
- if project.export_project_path - if project.export_status == :finished
= link_to 'Download export', download_export_project_path(project), = link_to 'Download export', download_export_project_path(project),
rel: 'nofollow', download: '', method: :get, class: "btn btn-default" rel: 'nofollow', download: '', method: :get, class: "btn btn-default"
= link_to 'Generate new export', generate_new_export_project_path(project), = link_to 'Generate new export', generate_new_export_project_path(project),
......
---
title: Add Object Storage to project export
merge_request: 20105
author:
type: added
class CreateImportExportUploads < ActiveRecord::Migration
DOWNTIME = false
def change
create_table :import_export_uploads do |t|
t.datetime_with_timezone :updated_at, null: false
t.references :project, index: true, foreign_key: { on_delete: :cascade }, unique: true
t.text :import_file
t.text :export_file
end
add_index :import_export_uploads, :updated_at
end
end
...@@ -949,6 +949,16 @@ ...@@ -949,6 +949,16 @@
add_index "identities", ["user_id"], name: "index_identities_on_user_id", using: :btree add_index "identities", ["user_id"], name: "index_identities_on_user_id", using: :btree
create_table "import_export_uploads", force: :cascade do |t|
t.datetime_with_timezone "updated_at", null: false
t.integer "project_id"
t.text "import_file"
t.text "export_file"
end
add_index "import_export_uploads", ["project_id"], name: "index_import_export_uploads_on_project_id", using: :btree
add_index "import_export_uploads", ["updated_at"], name: "index_import_export_uploads_on_updated_at", using: :btree
create_table "internal_ids", id: :bigserial, force: :cascade do |t| create_table "internal_ids", id: :bigserial, force: :cascade do |t|
t.integer "project_id" t.integer "project_id"
t.integer "usage", null: false t.integer "usage", null: false
......
...@@ -30,5 +30,12 @@ sudo gitlab-rake gitlab:import_export:data ...@@ -30,5 +30,12 @@ sudo gitlab-rake gitlab:import_export:data
bundle exec rake gitlab:import_export:data RAILS_ENV=production bundle exec rake gitlab:import_export:data RAILS_ENV=production
``` ```
In order to enable Object Storage on the Export, you can use the [feature flag][feature-flags]:
```
import_export_object_storage
```
[ce-3050]: https://gitlab.com/gitlab-org/gitlab-ce/issues/3050 [ce-3050]: https://gitlab.com/gitlab-org/gitlab-ce/issues/3050
[feature-flags]: https://docs.gitlab.com/ee/api/features.html
[tmp]: ../../development/shared_files.md [tmp]: ../../development/shared_files.md
...@@ -23,9 +23,13 @@ class ProjectExport < Grape::API ...@@ -23,9 +23,13 @@ class ProjectExport < Grape::API
get ':id/export/download' do get ':id/export/download' do
path = user_project.export_project_path path = user_project.export_project_path
render_api_error!('404 Not found or has expired', 404) unless path if path
present_disk_file!(path, File.basename(path), 'application/gzip')
present_disk_file!(path, File.basename(path), 'application/gzip') elsif user_project.export_project_object_exists?
present_carrierwave_file!(user_project.import_export_upload.export_file)
else
render_api_error!('404 Not found or has expired', 404)
end
end end
desc 'Start export' do desc 'Start export' do
......
...@@ -40,6 +40,10 @@ def export_filename(project:) ...@@ -40,6 +40,10 @@ def export_filename(project:)
"#{basename[0..FILENAME_LIMIT]}_export.tar.gz" "#{basename[0..FILENAME_LIMIT]}_export.tar.gz"
end end
def object_storage?
Feature.enabled?(:import_export_object_storage)
end
def version def version
VERSION VERSION
end end
......
...@@ -24,9 +24,10 @@ def initialize(attributes = {}) ...@@ -24,9 +24,10 @@ def initialize(attributes = {})
end end
def execute(current_user, project) def execute(current_user, project)
return unless project&.export_project_path
@project = project @project = project
return unless @project.export_status == :finished
@current_user = current_user @current_user = current_user
if invalid? if invalid?
...@@ -51,9 +52,12 @@ def to_json(options = {}) ...@@ -51,9 +52,12 @@ def to_json(options = {})
end end
def self.lock_file_path(project) def self.lock_file_path(project)
return unless project&.export_path return unless project.export_path || object_storage?
File.join(project.export_path, AFTER_EXPORT_LOCK_FILE_NAME) lock_path = project.import_export_shared.archive_path
FileUtils.mkdir_p(lock_path)
File.join(lock_path, AFTER_EXPORT_LOCK_FILE_NAME)
end end
protected protected
...@@ -77,6 +81,10 @@ def delete_after_export_lock ...@@ -77,6 +81,10 @@ def delete_after_export_lock
def log_validation_errors def log_validation_errors
errors.full_messages.each { |msg| project.import_export_shared.add_error_message(msg) } errors.full_messages.each { |msg| project.import_export_shared.add_error_message(msg) }
end end
def object_storage?
project.export_project_object_exists?
end
end end
end end
end end
......
...@@ -38,14 +38,20 @@ def handle_response_error(response) ...@@ -38,14 +38,20 @@ def handle_response_error(response)
private private
def send_file def send_file
export_file = File.open(project.export_project_path) Gitlab::HTTP.public_send(http_method.downcase, url, send_file_options) # rubocop:disable GitlabSecurity/PublicSend
Gitlab::HTTP.public_send(http_method.downcase, url, send_file_options(export_file)) # rubocop:disable GitlabSecurity/PublicSend
ensure ensure
export_file.close if export_file export_file.close if export_file && !object_storage?
end
def export_file
if object_storage?
project.import_export_upload.export_file.file.open
else
File.open(project.export_project_path)
end
end end
def send_file_options(export_file) def send_file_options
{ {
body_stream: export_file, body_stream: export_file,
headers: headers headers: headers
...@@ -53,7 +59,15 @@ def send_file_options(export_file) ...@@ -53,7 +59,15 @@ def send_file_options(export_file)
end end
def headers def headers
{ 'Content-Length' => File.size(project.export_project_path).to_s } { 'Content-Length' => export_size.to_s }
end
def export_size
if object_storage?
project.import_export_upload.export_file.file.size
else
File.size(project.export_project_path)
end
end end
end end
end end
......
...@@ -15,15 +15,22 @@ def initialize(project:, shared:) ...@@ -15,15 +15,22 @@ def initialize(project:, shared:)
def save def save
if compress_and_save if compress_and_save
remove_export_path remove_export_path
Rails.logger.info("Saved project export #{archive_file}") Rails.logger.info("Saved project export #{archive_file}")
archive_file
save_on_object_storage if use_object_storage?
else else
@shared.error(Gitlab::ImportExport::Error.new("Unable to save #{archive_file} into #{@shared.export_path}")) @shared.error(Gitlab::ImportExport::Error.new(error_message))
false false
end end
rescue => e rescue => e
@shared.error(e) @shared.error(e)
false false
ensure
if use_object_storage?
remove_archive
remove_export_path
end
end end
private private
...@@ -36,9 +43,29 @@ def remove_export_path ...@@ -36,9 +43,29 @@ def remove_export_path
FileUtils.rm_rf(@shared.export_path) FileUtils.rm_rf(@shared.export_path)
end end
def remove_archive
FileUtils.rm_rf(@shared.archive_path)
end
def archive_file def archive_file
@archive_file ||= File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project)) @archive_file ||= File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project))
end end
def save_on_object_storage
upload = ImportExportUpload.find_or_initialize_by(project: @project)
File.open(archive_file) { |file| upload.export_file = file }
upload.save!
end
def use_object_storage?
Gitlab::ImportExport.object_storage?
end
def error_message
"Unable to save #{archive_file} into #{@shared.export_path}. Object storage enabled: #{use_object_storage?}"
end
end end
end end
end end
...@@ -790,23 +790,55 @@ def update_project(**parameters) ...@@ -790,23 +790,55 @@ def update_project(**parameters)
project.add_master(user) project.add_master(user)
end end
context 'when project export is enabled' do context 'object storage disabled' do
it 'returns 302' do before do
get :download_export, namespace_id: project.namespace, id: project stub_feature_flags(import_export_object_storage: false)
end
expect(response).to have_gitlab_http_status(302) context 'when project export is enabled' do
it 'returns 302' do
get :download_export, namespace_id: project.namespace, id: project
expect(response).to have_gitlab_http_status(302)
end
end
context 'when project export is disabled' do
before do
stub_application_setting(project_export_enabled?: false)
end
it 'returns 404' do
get :download_export, namespace_id: project.namespace, id: project
expect(response).to have_gitlab_http_status(404)
end
end end
end end
context 'when project export is disabled' do context 'object storage enabled' do
before do before do
stub_application_setting(project_export_enabled?: false) stub_feature_flags(import_export_object_storage: true)
end end
it 'returns 404' do context 'when project export is enabled' do
get :download_export, namespace_id: project.namespace, id: project it 'returns 302' do
get :download_export, namespace_id: project.namespace, id: project
expect(response).to have_gitlab_http_status(404) expect(response).to have_gitlab_http_status(302)
end
end
context 'when project export is disabled' do
before do
stub_application_setting(project_export_enabled?: false)
end
it 'returns 404' do
get :download_export, namespace_id: project.namespace, id: project
expect(response).to have_gitlab_http_status(404)
end
end end
end end
end end
......
FactoryBot.define do
factory :import_export_upload do
project { create(:project) }
end
end
...@@ -103,6 +103,22 @@ ...@@ -103,6 +103,22 @@
end end
trait :with_export do trait :with_export do
before(:create) do |_project, _evaluator|
allow(Feature).to receive(:enabled?).with(:import_export_object_storage) { false }
allow(Feature).to receive(:enabled?).with('import_export_object_storage') { false }
end
after(:create) do |project, _evaluator|
ProjectExportWorker.new.perform(project.creator.id, project.id)
end
end
trait :with_object_export do
before(:create) do |_project, _evaluator|
allow(Feature).to receive(:enabled?).with(:import_export_object_storage) { true }
allow(Feature).to receive(:enabled?).with('import_export_object_storage') { true }
end
after(:create) do |project, evaluator| after(:create) do |project, evaluator|
ProjectExportWorker.new.perform(project.creator.id, project.id) ProjectExportWorker.new.perform(project.creator.id, project.id)
end end
......
...@@ -25,6 +25,7 @@ ...@@ -25,6 +25,7 @@
before do before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
stub_feature_flags(import_export_object_storage: false)
end end
after do after do
......