Commit 85746b43 authored by Kamil Trzciński's avatar Kamil Trzciński
Browse files

Merge branch 'zj-auto-upload-job-artifacts' into 'master'

Transfer job archives after creation

See merge request gitlab-org/gitlab-ee!3646
parent bf99a14d529e
module Ci
class JobArtifact < ActiveRecord::Base
include AfterCommitQueue
extend Gitlab::Ci::Model
belongs_to :project
......@@ -9,6 +10,12 @@ class JobArtifact < ActiveRecord::Base
mount_uploader :file, JobArtifactUploader
after_save if: :file_changed?, on: [:create, :update] do
run_after_commit do
file.schedule_migration_to_object_storage
end
end
enum file_type: {
archive: 1,
metadata: 2
......
class LfsObject < ActiveRecord::Base
prepend EE::LfsObject
include AfterCommitQueue
has_many :lfs_objects_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
has_many :projects, through: :lfs_objects_projects
......@@ -8,6 +11,12 @@ class LfsObject < ActiveRecord::Base
mount_uploader :file, LfsObjectUploader
after_save if: :file_changed?, on: [:create, :update] do
run_after_commit do
file.schedule_migration_to_object_storage
end
end
def project_allowed_access?(project)
projects.exists?(project.lfs_storage_project.id)
end
......
class LfsObjectUploader < ObjectStoreUploader
storage_options Gitlab.config.lfs
after :store, :schedule_migration_to_object_storage
def self.local_store_path
Gitlab.config.lfs.storage_path
......
......@@ -116,10 +116,13 @@ def migrate!(new_store)
end
end
def schedule_migration_to_object_storage(new_file)
if self.class.object_store_enabled? && licensed? && file_storage?
ObjectStorageUploadWorker.perform_async(self.class.name, model.class.name, mounted_as, model.id)
end
def schedule_migration_to_object_storage(*args)
return unless self.class.object_store_enabled?
return unless self.class.background_upload_enabled?
return unless self.licensed?
return unless self.file_storage?
ObjectStorageUploadWorker.perform_async(self.class.name, model.class.name, mounted_as, model.id)
end
def fog_directory
......
......@@ -2,6 +2,8 @@ class ObjectStorageUploadWorker
include Sidekiq::Worker
include DedicatedSidekiqQueue
sidekiq_options retry: 5
def perform(uploader_class_name, subject_class_name, file_field, subject_id)
uploader_class = uploader_class_name.constantize
subject_class = subject_class_name.constantize
......@@ -9,7 +11,9 @@ def perform(uploader_class_name, subject_class_name, file_field, subject_id)
return unless uploader_class.object_store_enabled?
return unless uploader_class.background_upload_enabled?
subject = subject_class.find(subject_id)
subject = subject_class.find_by(id: subject_id)
return unless subject
file = subject.public_send(file_field) # rubocop:disable GitlabSecurity/PublicSend
return unless file.licensed?
......
---
title: Transfer job archives to object storage after creation
merge_request:
author:
type: added
......@@ -679,6 +679,7 @@ test:
object_store:
enabled: false
remote_directory: artifacts # The bucket name
background_upload: false
connection:
provider: AWS # Only AWS supported at the moment
aws_access_key_id: AWS_ACCESS_KEY_ID
......
require 'spec_helper'
describe LfsObject do
describe '#local_store?' do
it 'returns true when file_store is nil' do
subject.file_store = nil
expect(subject.local_store?).to eq true
end
it 'returns true when file_store is equal to LfsObjectUploader::LOCAL_STORE' do
subject.file_store = LfsObjectUploader::LOCAL_STORE
expect(subject.local_store?).to eq true
end
it 'returns false whe file_store is equal to LfsObjectUploader::REMOTE_STORE' do
subject.file_store = LfsObjectUploader::REMOTE_STORE
expect(subject.local_store?).to eq false
end
end
describe '#destroy' do
subject { create(:lfs_object, :with_file) }
context 'when running in a Geo primary node' do
set(:primary) { create(:geo_node, :primary) }
set(:secondary) { create(:geo_node) }
it 'logs an event to the Geo event log' do
expect { subject.destroy }.to change(Geo::LfsObjectDeletedEvent, :count).by(1)
end
end
end
describe '#schedule_migration_to_object_storage' do
before do
stub_lfs_setting(enabled: true)
end
subject { create(:lfs_object, :with_file) }
context 'when object storage is disabled' do
before do
stub_lfs_object_storage(enabled: false)
end
it 'does not schedule the migration' do
expect(ObjectStorageUploadWorker).not_to receive(:perform_async)
subject
end
end
context 'when object storage is enabled' do
context 'when background upload is enabled' do
context 'when is licensed' do
before do
stub_lfs_object_storage(background_upload: true)
end
it 'schedules the model for migration' do
expect(ObjectStorageUploadWorker).to receive(:perform_async).with('LfsObjectUploader', described_class.name, :file, kind_of(Numeric))
subject
end
end
context 'when is unlicensed' do
before do
stub_lfs_object_storage(background_upload: true, licensed: false)
end
it 'does not schedule the migration' do
expect(ObjectStorageUploadWorker).not_to receive(:perform_async)
subject
end
end
end
context 'when background upload is disabled' do
before do
stub_lfs_object_storage(background_upload: false)
end
it 'schedules the model for migration' do
expect(ObjectStorageUploadWorker).not_to receive(:perform_async)
subject
end
end
end
end
end
......@@ -17,7 +17,7 @@ def perform
context 'when object storage is enabled' do
before do
stub_lfs_object_storage
stub_lfs_object_storage(background_upload: true)
end
it 'uploads object to storage' do
......@@ -60,7 +60,7 @@ def perform
context 'and remote storage is defined' do
before do
stub_artifacts_object_storage
stub_artifacts_object_storage(background_upload: true)
end
it "migrates file to remote storage" do
......@@ -94,7 +94,7 @@ def perform
context 'and remote storage is defined' do
before do
stub_artifacts_object_storage
stub_artifacts_object_storage(background_upload: true)
end
it "migrates file to remote storage" do
......
......@@ -12,6 +12,64 @@
it { is_expected.to respond_to(:created_at) }
it { is_expected.to respond_to(:updated_at) }
describe 'callbacks' do
subject { create(:ci_job_artifact, :archive) }
describe '#schedule_migration_to_object_storage' do
context 'when object storage is disabled' do
before do
stub_artifacts_object_storage(enabled: false)
end
it 'does not schedule the migration' do
expect(ObjectStorageUploadWorker).not_to receive(:perform_async)
subject
end
end
context 'when object storage is enabled' do
context 'when background upload is enabled' do
context 'when is licensed' do
before do
stub_artifacts_object_storage(background_upload: true)
end
it 'schedules the model for migration' do
expect(ObjectStorageUploadWorker).to receive(:perform_async).with('JobArtifactUploader', described_class.name, :file, kind_of(Numeric))
subject
end
end
context 'when is unlicensed' do
before do
stub_artifacts_object_storage(background_upload: true, licensed: false)
end
it 'does not schedule the migration' do
expect(ObjectStorageUploadWorker).not_to receive(:perform_async)
subject
end
end
end
context 'when background upload is disabled' do
before do
stub_artifacts_object_storage(background_upload: false)
end
it 'schedules the model for migration' do
expect(ObjectStorageUploadWorker).not_to receive(:perform_async)
subject
end
end
end
end
end
describe '#set_size' do
it 'sets the size' do
expect(artifact.size).to eq(106365)
......
......@@ -1010,7 +1010,7 @@
context 'with object storage enabled' do
before do
stub_lfs_object_storage
stub_lfs_object_storage(background_upload: true)
end
it 'schedules migration of file to object storage' do
......
module StubConfiguration
def stub_object_storage_uploader(config:, uploader:, remote_directory:, enabled: true, licensed: true)
def stub_object_storage_uploader(config:, uploader:, remote_directory:, enabled: true, licensed: true, background_upload: false)
Fog.mock!
allow(config).to receive(:enabled) { enabled }
allow(config).to receive(:background_upload) { background_upload }
stub_licensed_features(object_storage: licensed) unless licensed == :skip
......
......@@ -49,7 +49,7 @@
context 'with object storage enabled' do
before do
stub_lfs_object_storage
stub_lfs_object_storage(background_upload: true)
end
it 'is scheduled to run after creation' do
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment