This instance will be upgraded to Heptapod 0.31.0 (final) on 2022-05-24 at 14:00 UTC+2 (a few minutes of down time)

Commit 72093336 authored by Michael Kozono's avatar Michael Kozono
Browse files

Rename table to untracked_files_for_uploads

parent 352895a7fe66
......@@ -7,31 +7,31 @@ class TrackUntrackedUploads < ActiveRecord::Migration
disable_ddl_transaction!
DOWNTIME = false
MIGRATION = 'PrepareUnhashedUploads'
MIGRATION = 'PrepareUntrackedUploads'
def up
unless table_exists?(:unhashed_upload_files)
create_table :unhashed_upload_files do |t|
unless table_exists?(:untracked_files_for_uploads)
create_table :untracked_files_for_uploads do |t|
t.string :path, null: false
t.boolean :tracked, default: false, null: false
t.timestamps_with_timezone null: false
end
end
unless index_exists?(:unhashed_upload_files, :path)
add_index :unhashed_upload_files, :path, unique: true
unless index_exists?(:untracked_files_for_uploads, :path)
add_index :untracked_files_for_uploads, :path, unique: true
end
unless index_exists?(:unhashed_upload_files, :tracked)
add_index :unhashed_upload_files, :tracked
unless index_exists?(:untracked_files_for_uploads, :tracked)
add_index :untracked_files_for_uploads, :tracked
end
BackgroundMigrationWorker.perform_async(MIGRATION)
end
def down
if table_exists?(:unhashed_upload_files)
drop_table :unhashed_upload_files
if table_exists?(:untracked_files_for_uploads)
drop_table :untracked_files_for_uploads
end
end
end
......@@ -1719,15 +1719,15 @@
add_index "u2f_registrations", ["key_handle"], name: "index_u2f_registrations_on_key_handle", using: :btree
add_index "u2f_registrations", ["user_id"], name: "index_u2f_registrations_on_user_id", using: :btree
create_table "unhashed_upload_files", force: :cascade do |t|
create_table "untracked_files_for_uploads", force: :cascade do |t|
t.string "path", null: false
t.boolean "tracked", default: false, null: false
t.datetime_with_timezone "created_at", null: false
t.datetime_with_timezone "updated_at", null: false
end
add_index "unhashed_upload_files", ["path"], name: "index_unhashed_upload_files_on_path", unique: true, using: :btree
add_index "unhashed_upload_files", ["tracked"], name: "index_unhashed_upload_files_on_tracked", using: :btree
add_index "untracked_files_for_uploads", ["path"], name: "index_untracked_files_for_uploads_on_path", unique: true, using: :btree
add_index "untracked_files_for_uploads", ["tracked"], name: "index_untracked_files_for_uploads_on_tracked", using: :btree
create_table "uploads", force: :cascade do |t|
t.integer "size", limit: 8, null: false
......
module Gitlab
module BackgroundMigration
class PopulateUntrackedUploads
class UnhashedUploadFile < ActiveRecord::Base
self.table_name = 'unhashed_upload_files'
class UntrackedFile < ActiveRecord::Base
self.table_name = 'untracked_files_for_uploads'
# Ends with /:random_hex/:filename
FILE_UPLOADER_PATH_PATTERN = %r{/\h+/[^/]+\z}
......@@ -84,7 +84,7 @@ def mark_as_tracked
end
def upload_path
# UnhashedUploadFile#path is absolute, but Upload#path depends on uploader
# UntrackedFile#path is absolute, but Upload#path depends on uploader
if uploader == 'FileUploader'
# Path relative to project directory in uploads
matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH_PATTERN)
......@@ -118,7 +118,7 @@ def file_size
# Not including a leading slash
def path_relative_to_upload_dir
base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR)}/}
base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR)}/}
@path_relative_to_upload_dir ||= path.sub(base, '')
end
......@@ -218,10 +218,10 @@ def uploader_class
def perform(start_id, end_id)
return unless migrate?
files = UnhashedUploadFile.untracked.where(id: start_id..end_id)
files.each do |unhashed_upload_file|
files = UntrackedFile.untracked.where(id: start_id..end_id)
files.each do |untracked_file|
begin
unhashed_upload_file.ensure_tracked!
untracked_file.ensure_tracked!
rescue StandardError => e
Rails.logger.warn "Failed to add untracked file to uploads: #{e.message}"
......@@ -235,7 +235,7 @@ def perform(start_id, end_id)
private
def migrate?
UnhashedUploadFile.table_exists? && Upload.table_exists?
UntrackedFile.table_exists? && Upload.table_exists?
end
end
end
......
module Gitlab
module BackgroundMigration
class PrepareUnhashedUploads
class PrepareUntrackedUploads
# For bulk_queue_background_migration_jobs_by_range
include Database::MigrationHelpers
......@@ -8,31 +8,31 @@ class PrepareUnhashedUploads
UPLOAD_DIR = "#{CarrierWave.root}/uploads".freeze
FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze
class UnhashedUploadFile < ActiveRecord::Base
class UntrackedFile < ActiveRecord::Base
include EachBatch
self.table_name = 'unhashed_upload_files'
self.table_name = 'untracked_files_for_uploads'
end
def perform
return unless migrate?
clear_unhashed_upload_file_paths
store_unhashed_upload_file_paths
clear_untracked_file_paths
store_untracked_file_paths
schedule_populate_untracked_uploads_jobs
end
private
def migrate?
UnhashedUploadFile.table_exists?
UntrackedFile.table_exists?
end
def clear_unhashed_upload_file_paths
UnhashedUploadFile.delete_all
def clear_untracked_file_paths
UntrackedFile.delete_all
end
def store_unhashed_upload_file_paths
def store_untracked_file_paths
return unless Dir.exist?(UPLOAD_DIR)
each_file_batch(UPLOAD_DIR, FILE_PATH_BATCH_SIZE) do |file_paths|
......@@ -89,7 +89,7 @@ def insert_file_paths(file_paths)
end
def insert_file_path(file_path)
table_columns_and_values = 'unhashed_upload_files (path, created_at, updated_at) VALUES (?, ?, ?)'
table_columns_and_values = 'untracked_files_for_uploads (path, created_at, updated_at) VALUES (?, ?, ?)'
sql = if Gitlab::Database.postgresql?
"INSERT INTO #{table_columns_and_values} ON CONFLICT DO NOTHING;"
......@@ -103,7 +103,7 @@ def insert_file_path(file_path)
end
def schedule_populate_untracked_uploads_jobs
bulk_queue_background_migration_jobs_by_range(UnhashedUploadFile, FOLLOW_UP_MIGRATION)
bulk_queue_background_migration_jobs_by_range(UntrackedFile, FOLLOW_UP_MIGRATION)
end
end
end
......
require 'spec_helper'
describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, :sidekiq, schema: 20171103140253 do
let!(:unhashed_upload_files) { table(:unhashed_upload_files) }
describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :sidekiq, schema: 20171103140253 do
let!(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) }
let(:user1) { create(:user) }
let(:user2) { create(:user) }
......@@ -39,20 +39,20 @@
UploadService.new(project2, uploaded_file, FileUploader).execute
end
it 'adds unhashed files to the unhashed_upload_files table' do
it 'adds unhashed files to the untracked_files_for_uploads table' do
Sidekiq::Testing.fake! do
expect do
described_class.new.perform
end.to change { unhashed_upload_files.count }.from(0).to(5)
end.to change { untracked_files_for_uploads.count }.from(0).to(5)
end
end
it 'does not add hashed files to the unhashed_upload_files table' do
it 'does not add hashed files to the untracked_files_for_uploads table' do
Sidekiq::Testing.fake! do
described_class.new.perform
hashed_file_path = project2.uploads.where(uploader: 'FileUploader').first.path
expect(unhashed_upload_files.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey
expect(untracked_files_for_uploads.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey
end
end
......@@ -66,16 +66,16 @@
end
# E.g. from a previous failed run of this background migration
context 'when there is existing data in unhashed_upload_files' do
context 'when there is existing data in untracked_files_for_uploads' do
before do
unhashed_upload_files.create(path: '/foo/bar.jpg')
untracked_files_for_uploads.create(path: '/foo/bar.jpg')
end
it 'clears existing data before adding new data' do
Sidekiq::Testing.fake! do
expect do
described_class.new.perform
end.to change { unhashed_upload_files.count }.from(1).to(5)
end.to change { untracked_files_for_uploads.count }.from(1).to(5)
end
end
end
......@@ -91,7 +91,7 @@
Sidekiq::Testing.fake! do
expect do
described_class.new.perform
end.to change { unhashed_upload_files.count }.from(0).to(5)
end.to change { untracked_files_for_uploads.count }.from(0).to(5)
end
end
end
......@@ -100,11 +100,11 @@
# Very new or lightly-used installations that are running this migration
# may not have an upload directory because they have no uploads.
context 'when no files were ever uploaded' do
it 'does not add to the unhashed_upload_files table (and does not raise error)' do
it 'does not add to the untracked_files_for_uploads table (and does not raise error)' do
Sidekiq::Testing.fake! do
expect do
described_class.new.perform
end.not_to change { unhashed_upload_files.count }.from(0)
end.not_to change { untracked_files_for_uploads.count }.from(0)
end
end
end
......
......@@ -4,8 +4,8 @@
describe TrackUntrackedUploads, :migration, :sidekiq do
include TrackUntrackedUploadsHelpers
class UnhashedUploadFile < ActiveRecord::Base
self.table_name = 'unhashed_upload_files'
class UntrackedFile < ActiveRecord::Base
self.table_name = 'untracked_files_for_uploads'
end
matcher :be_scheduled_migration do
......@@ -29,10 +29,10 @@ class UnhashedUploadFile < ActiveRecord::Base
end
end
it 'ensures the unhashed_upload_files table exists' do
it 'ensures the untracked_files_for_uploads table exists' do
expect do
migrate!
end.to change { table_exists?(:unhashed_upload_files) }.from(false).to(true)
end.to change { table_exists?(:untracked_files_for_uploads) }.from(false).to(true)
end
it 'has a path field long enough for really long paths' do
......@@ -48,7 +48,7 @@ class UnhashedUploadFile < ActiveRecord::Base
component # filename
].flatten.join('/')
record = UnhashedUploadFile.create!(path: long_path)
record = UntrackedFile.create!(path: long_path)
expect(record.reload.path.size).to eq(5711)
end
......@@ -132,12 +132,12 @@ class UnhashedUploadFile < ActiveRecord::Base
end
end
it 'all UnhashedUploadFile records are marked as tracked' do
it 'all UntrackedFile records are marked as tracked' do
Sidekiq::Testing.inline! do
migrate!
expect(UnhashedUploadFile.count).to eq(8)
expect(UnhashedUploadFile.count).to eq(UnhashedUploadFile.where(tracked: true).count)
expect(UntrackedFile.count).to eq(8)
expect(UntrackedFile.count).to eq(UntrackedFile.where(tracked: true).count)
end
end
end
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment