This instance will be upgraded to Heptapod 0.31.0 (final) on 2022-05-24 at 14:00 UTC+2 (a few minutes of down time)

Commit 68e32474 authored by Michael Kozono's avatar Michael Kozono
Browse files

Kick off follow up background migration jobs

To process the unhashed_upload_files table.
parent d0f61b95139a
module Gitlab
module BackgroundMigration
class PopulateUntrackedUploads
class UnhashedUploadFile < ActiveRecord::Base
self.table_name = 'unhashed_upload_files'
scope :untracked, -> { where(tracked: false) }
def ensure_tracked!
# TODO
# unless unhashed_upload_file.in_uploads?
# unhashed_upload_file.add_to_uploads
# end
#
# unhashed_upload_file.mark_as_tracked
end
def model_id
# TODO
end
def model_type
# TODO
end
def uploader
# TODO
end
end
class Upload < ActiveRecord::Base
self.table_name = 'uploads'
end
def perform(start_id, end_id)
return unless migrate?
files = UnhashedUploadFile.untracked.where(id: start_id..end_id)
files.each do |unhashed_upload_file|
unhashed_upload_file.ensure_tracked!
end
end
private
def migrate?
UnhashedUploadFile.table_exists? && Upload.table_exists?
end
end
end
end
module Gitlab
module BackgroundMigration
class PrepareUnhashedUploads
# For bulk_queue_background_migration_jobs_by_range
include Database::MigrationHelpers
FILE_PATH_BATCH_SIZE = 500
UPLOAD_DIR = "#{CarrierWave.root}/uploads"
FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'
class UnhashedUploadFile < ActiveRecord::Base
include EachBatch
self.table_name = 'unhashed_upload_files'
end
......@@ -74,7 +80,7 @@ def insert_file_paths(file_paths)
end
def schedule_populate_untracked_uploads_jobs
# TODO
bulk_queue_background_migration_jobs_by_range(UnhashedUploadFile, FOLLOW_UP_MIGRATION)
end
end
end
......
require 'spec_helper'
describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, schema: 20171103140253 do
describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, :sidekiq, schema: 20171103140253 do
let!(:unhashed_upload_files) { table(:unhashed_upload_files) }
let(:user1) { create(:user) }
......@@ -9,6 +9,18 @@
let(:project2) { create(:project) }
let(:appearance) { create(:appearance) }
matcher :be_scheduled_migration do |*expected|
match do |migration|
BackgroundMigrationWorker.jobs.any? do |job|
job['args'] == [migration, expected]
end
end
failure_message do |migration|
"Migration `#{migration}` with args `#{expected.inspect}` not scheduled!"
end
end
context 'when files were uploaded before and after hashed storage was enabled' do
before do
fixture = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg')
......@@ -28,16 +40,29 @@
end
it 'adds unhashed files to the unhashed_upload_files table' do
expect do
described_class.new.perform
end.to change { unhashed_upload_files.count }.from(0).to(5)
Sidekiq::Testing.fake! do
expect do
described_class.new.perform
end.to change { unhashed_upload_files.count }.from(0).to(5)
end
end
it 'does not add hashed files to the unhashed_upload_files table' do
described_class.new.perform
Sidekiq::Testing.fake! do
described_class.new.perform
hashed_file_path = project2.uploads.where(uploader: 'FileUploader').first.path
expect(unhashed_upload_files.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey
hashed_file_path = project2.uploads.where(uploader: 'FileUploader').first.path
expect(unhashed_upload_files.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey
end
end
it 'correctly schedules the follow-up background migration jobs' do
Sidekiq::Testing.fake! do
described_class.new.perform
expect(described_class::FOLLOW_UP_MIGRATION).to be_scheduled_migration(1, 5)
expect(BackgroundMigrationWorker.jobs.size).to eq(1)
end
end
# E.g. from a previous failed run of this background migration
......@@ -47,9 +72,11 @@
end
it 'clears existing data before adding new data' do
expect do
described_class.new.perform
end.to change { unhashed_upload_files.count }.from(1).to(5)
Sidekiq::Testing.fake! do
expect do
described_class.new.perform
end.to change { unhashed_upload_files.count }.from(1).to(5)
end
end
end
......@@ -61,9 +88,11 @@
end
it 'does not add files from /uploads/tmp' do
expect do
described_class.new.perform
end.to change { unhashed_upload_files.count }.from(0).to(5)
Sidekiq::Testing.fake! do
expect do
described_class.new.perform
end.to change { unhashed_upload_files.count }.from(0).to(5)
end
end
end
end
......@@ -72,9 +101,11 @@
# may not have an upload directory because they have no uploads.
context 'when no files were ever uploaded' do
it 'does not add to the unhashed_upload_files table (and does not raise error)' do
expect do
described_class.new.perform
end.not_to change { unhashed_upload_files.count }.from(0)
Sidekiq::Testing.fake! do
expect do
described_class.new.perform
end.not_to change { unhashed_upload_files.count }.from(0)
end
end
end
end
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment