This instance will be upgraded to Heptapod 0.27.1 on 2021-12-02 at 15:00 UTC+1 (a few minutes of down time)

Commit c42197ad authored by James Lopez's avatar James Lopez
Browse files

Update Import/Export to use object storage (based on aa feature flag)

parent 7df607327bda
require 'spec_helper'
describe Gitlab::ImportExport::AfterExportStrategies::BaseAfterExportStrategy do
let!(:service) { described_class.new }
let!(:project) { create(:project, :with_object_export) }
let(:shared) { project.import_export_shared }
let!(:user) { create(:user) }
describe '#execute' do
before do
allow(service).to receive(:strategy_execute)
stub_feature_flags(import_export_object_storage: true)
end
it 'returns if project exported file is not found' do
allow(project).to receive(:export_project_object_exists?).and_return(false)
expect(service).not_to receive(:strategy_execute)
service.execute(user, project)
end
it 'creates a lock file in the export dir' do
allow(service).to receive(:delete_after_export_lock)
service.execute(user, project)
expect(lock_path_exist?).to be_truthy
end
context 'when the method succeeds' do
it 'removes the lock file' do
service.execute(user, project)
expect(lock_path_exist?).to be_falsey
end
end
context 'when the method fails' do
before do
allow(service).to receive(:strategy_execute).and_call_original
end
context 'when validation fails' do
before do
allow(service).to receive(:invalid?).and_return(true)
end
it 'does not create the lock file' do
expect(service).not_to receive(:create_or_update_after_export_lock)
service.execute(user, project)
end
it 'does not execute main logic' do
expect(service).not_to receive(:strategy_execute)
service.execute(user, project)
end
it 'logs validation errors in shared context' do
expect(service).to receive(:log_validation_errors)
service.execute(user, project)
end
end
context 'when an exception is raised' do
it 'removes the lock' do
expect { service.execute(user, project) }.to raise_error(NotImplementedError)
expect(lock_path_exist?).to be_falsey
end
end
end
end
describe '#log_validation_errors' do
it 'add the message to the shared context' do
errors = %w(test_message test_message2)
allow(service).to receive(:invalid?).and_return(true)
allow(service.errors).to receive(:full_messages).and_return(errors)
expect(shared).to receive(:add_error_message).twice.and_call_original
service.execute(user, project)
expect(shared.errors).to eq errors
end
end
describe '#to_json' do
it 'adds the current strategy class to the serialized attributes' do
params = { param1: 1 }
result = params.merge(klass: described_class.to_s).to_json
expect(described_class.new(params).to_json).to eq result
end
end
def lock_path_exist?
File.exist?(described_class.lock_file_path(project))
end
end
......@@ -9,6 +9,7 @@
describe '#execute' do
before do
allow(service).to receive(:strategy_execute)
stub_feature_flags(import_export_object_storage: false)
end
it 'returns if project exported file is not found' do
......
......@@ -24,13 +24,34 @@
end
describe '#execute' do
it 'removes the exported project file after the upload' do
allow(strategy).to receive(:send_file)
allow(strategy).to receive(:handle_response_error)
context 'without object storage' do
before do
stub_feature_flags(import_export_object_storage: false)
end
it 'removes the exported project file after the upload' do
allow(strategy).to receive(:send_file)
allow(strategy).to receive(:handle_response_error)
expect(project).to receive(:remove_exported_project_file)
strategy.execute(user, project)
end
end
context 'with object storage' do
before do
stub_feature_flags(import_export_object_storage: true)
end
expect(project).to receive(:remove_exported_project_file)
it 'removes the exported project file after the upload' do
allow(strategy).to receive(:send_file)
allow(strategy).to receive(:handle_response_error)
strategy.execute(user, project)
expect(project).to receive(:remove_exported_project_file)
strategy.execute(user, project)
end
end
end
end
......@@ -293,6 +293,7 @@ project:
- deploy_tokens
- settings
- ci_cd_settings
- import_export_upload
award_emoji:
- awardable
- user
......
require 'spec_helper'
require 'fileutils'
describe Gitlab::ImportExport::Saver do
let!(:project) { create(:project, :public, name: 'project') }
let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" }
let(:shared) { project.import_export_shared }
subject { described_class.new(project: project, shared: shared) }
before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
FileUtils.mkdir_p(shared.export_path)
FileUtils.touch("#{shared.export_path}/tmp.bundle")
end
after do
FileUtils.rm_rf(export_path)
end
context 'local archive' do
it 'saves the repo to disk' do
stub_feature_flags(import_export_object_storage: false)
subject.save
expect(shared.errors).to be_empty
expect(Dir.empty?(shared.archive_path)).to be false
end
end
context 'object storage' do
it 'saves the repo using object storage' do
stub_feature_flags(import_export_object_storage: true)
stub_uploads_object_storage(ImportExportUploader)
subject.save
expect(ImportExportUpload.find_by(project: project).export_file.url)
.to match(%r[\/uploads\/-\/system\/import_export_upload\/export_file.*])
end
end
end
require 'spec_helper'
describe ImportExportUpload do
subject { described_class.new(project: create(:project)) }
shared_examples 'stores the Import/Export file' do |method|
it 'stores the import file' do
subject.public_send("#{method}=", fixture_file_upload('spec/fixtures/project_export.tar.gz'))
subject.save!
url = "/uploads/-/system/import_export_upload/#{method}/#{subject.id}/project_export.tar.gz"
expect(subject.public_send(method).url).to eq(url)
end
end
context 'import' do
it_behaves_like 'stores the Import/Export file', :import_file
end
context 'export' do
it_behaves_like 'stores the Import/Export file', :export_file
end
end
......@@ -2782,6 +2782,10 @@ def enable_lfs
let(:legacy_project) { create(:project, :legacy_storage, :with_export) }
let(:project) { create(:project, :with_export) }
before do
stub_feature_flags(import_export_object_storage: false)
end
it 'removes the exports directory for the project' do
expect(File.exist?(project.export_path)).to be_truthy
......@@ -2830,12 +2834,14 @@ def enable_lfs
let(:project) { create(:project, :with_export) }
it 'removes the exported project file' do
stub_feature_flags(import_export_object_storage: false)
exported_file = project.export_project_path
expect(File.exist?(exported_file)).to be_truthy
allow(FileUtils).to receive(:rm_f).and_call_original
expect(FileUtils).to receive(:rm_f).with(exported_file).and_call_original
allow(FileUtils).to receive(:rm_rf).and_call_original
expect(FileUtils).to receive(:rm_rf).with(exported_file).and_call_original
project.remove_exported_project_file
......
......@@ -192,6 +192,13 @@
context 'when upload complete' do
before do
FileUtils.rm_rf(project_after_export.export_path)
if project_after_export.export_project_object_exists?
upload = project_after_export.import_export_upload
upload.remove_export_file!
upload.save
end
end
it_behaves_like '404 response' do
......@@ -261,6 +268,22 @@
it_behaves_like 'get project export download not found'
end
end
context 'when an uploader is used' do
before do
stub_uploads_object_storage(ImportExportUploader)
[project, project_finished, project_after_export].each do |p|
p.add_master(user)
upload = ImportExportUpload.new(project: p)
upload.export_file = fixture_file_upload('spec/fixtures/project_export.tar.gz', "`/tar.gz")
upload.save!
end
end
it_behaves_like 'get project download by strategy'
end
end
describe 'POST /projects/:project_id/export' do
......
......@@ -38,6 +38,24 @@
end
end
context 'with uploader exports' do
it 'removes old files' do
upload = create(:import_export_upload,
updated_at: 2.days.ago,
export_file: fixture_file_upload('spec/fixtures/project_export.tar.gz'))
expect { service.execute }.to change { upload.reload.export_file.file.nil? }.to(true)
end
it 'does not remove new files' do
upload = create(:import_export_upload,
updated_at: 1.hour.ago,
export_file: fixture_file_upload('spec/fixtures/project_export.tar.gz'))
expect { service.execute }.not_to change { upload.reload.export_file.file.nil? }
end
end
def in_directory_with_files(mtime:)
Dir.mktmpdir do |tmpdir|
stub_repository_downloads_path(tmpdir)
......
require 'spec_helper'
describe ImportExportUploader do
let(:model) { build_stubbed(:import_export_upload) }
let(:upload) { create(:upload, model: model) }
subject { described_class.new(model, :import_file) }
context "object_store is REMOTE" do
before do
stub_uploads_object_storage
end
include_context 'with storage', described_class::Store::REMOTE
it_behaves_like 'builds correct paths',
store_dir: %r[import_export_upload/import_file/],
upload_path: %r[import_export_upload/import_file/]
end
end
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment