Commit 9679fbc4 authored by Michael Kozono's avatar Michael Kozono
Browse files

Add untracked files to uploads

parent cacfd8f56bc5
......@@ -4,27 +4,149 @@ class PopulateUntrackedUploads
class UnhashedUploadFile < ActiveRecord::Base
self.table_name = 'unhashed_upload_files'
# Ends with /:random_hex/:filename
FILE_UPLOADER_PATH_PATTERN = /\/\h+\/[^\/]+\z/
# These regex patterns are tested against a relative path, relative to
# the upload directory.
# For convenience, if there exists a capture group in the pattern, then
# it indicates the model_id.
PATH_PATTERNS = [
{
pattern: /\A-\/system\/appearance\/logo\/(\d+)/,
uploader: 'AttachmentUploader',
model_type: 'Appearance',
},
{
pattern: /\A-\/system\/appearance\/header_logo\/(\d+)/,
uploader: 'AttachmentUploader',
model_type: 'Appearance',
},
{
pattern: /\A-\/system\/note\/attachment\/(\d+)/,
uploader: 'AttachmentUploader',
model_type: 'Note',
},
{
pattern: /\A-\/system\/user\/avatar\/(\d+)/,
uploader: 'AvatarUploader',
model_type: 'User',
},
{
pattern: /\A-\/system\/group\/avatar\/(\d+)/,
uploader: 'AvatarUploader',
model_type: 'Group',
},
{
pattern: /\A-\/system\/project\/avatar\/(\d+)/,
uploader: 'AvatarUploader',
model_type: 'Project',
},
{
pattern: FILE_UPLOADER_PATH_PATTERN,
uploader: 'FileUploader',
model_type: 'Project'
},
]
scope :untracked, -> { where(tracked: false) }
def ensure_tracked!
# TODO
# unless unhashed_upload_file.in_uploads?
# unhashed_upload_file.add_to_uploads
# end
#
# unhashed_upload_file.mark_as_tracked
return if persisted? && tracked?
unless in_uploads?
add_to_uploads
end
mark_as_tracked
end
def model_id
# TODO
def in_uploads?
# Even though we are checking relative paths, path is enough to
# uniquely identify uploads. There is no ambiguity between
# FileUploader paths and other Uploader paths because we use the /-/
# separator kind of like an escape character. Project full_path will
# never conflict with an upload path starting with "uploads/-/".
Upload.exists?(path: upload_path)
end
def model_type
# TODO
def add_to_uploads
Upload.create!(
path: upload_path,
uploader: uploader,
model_type: model_type,
model_id: model_id,
size: file_size
)
end
def mark_as_tracked
self.tracked = true
self.save!
end
def upload_path
# UnhashedUploadFile#path is absolute, but Upload#path depends on uploader
if uploader == 'FileUploader'
# Path relative to project directory in uploads
matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH_PATTERN)
matchd[0].sub(/\A\//, '') # remove leading slash
else
path_relative_to_carrierwave_root
end
end
def uploader
# TODO
PATH_PATTERNS.each do |path_pattern_map|
if path_relative_to_upload_dir.match(path_pattern_map[:pattern])
return path_pattern_map[:uploader]
end
end
end
def model_type
PATH_PATTERNS.each do |path_pattern_map|
if path_relative_to_upload_dir.match(path_pattern_map[:pattern])
return path_pattern_map[:model_type]
end
end
end
def model_id
PATH_PATTERNS.each do |path_pattern_map|
matchd = path_relative_to_upload_dir.match(path_pattern_map[:pattern])
# If something is captured (matchd[1] is not nil), it is a model_id
return matchd[1] if matchd && matchd[1]
end
# Only the FileUploader pattern will not match an ID
file_uploader_model_id
end
def file_size
File.size(path)
end
# Not including a leading slash
def path_relative_to_upload_dir
@path_relative_to_upload_dir ||= path.sub(/\A#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}\//, '')
end
# Not including a leading slash
def path_relative_to_carrierwave_root
"uploads/#{path_relative_to_upload_dir}"
end
private
def file_uploader_model_id
pattern_to_capture_full_path = /\A(.+)#{FILE_UPLOADER_PATH_PATTERN}/
matchd = path_relative_to_upload_dir.match(pattern_to_capture_full_path)
raise "Could not capture project full_path from a FileUploader path: \"#{path_relative_to_upload_dir}\"" unless matchd
full_path = matchd[1]
project = Project.find_by_full_path(full_path)
project.id.to_s
end
end
......
......@@ -2,6 +2,10 @@
require Rails.root.join('db', 'post_migrate', '20171103140253_track_untracked_uploads')
describe TrackUntrackedUploads, :migration, :sidekiq do
class UnhashedUploadFile < ActiveRecord::Base
self.table_name = 'unhashed_upload_files'
end
matcher :be_scheduled_migration do
match do |migration|
BackgroundMigrationWorker.jobs.any? do |job|
......@@ -30,10 +34,6 @@
end
it 'has a path field long enough for really long paths' do
class UnhashedUploadFile < ActiveRecord::Base
self.table_name = 'unhashed_upload_files'
end
migrate!
max_length_namespace_path = max_length_project_path = max_length_filename = 'a' * 255
......@@ -57,7 +57,8 @@ class UnhashedUploadFile < ActiveRecord::Base
uploaded_file = fixture_file_upload(fixture)
user1.update(avatar: uploaded_file)
project1.update(avatar: uploaded_file)
UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload
upload_result = UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload
@project1_markdown_upload_path = upload_result[:url].sub(/\A\/uploads\//, '')
appearance.update(logo: uploaded_file)
# Untracked, by doing normal file upload then deleting records from DB
......@@ -65,48 +66,62 @@ class UnhashedUploadFile < ActiveRecord::Base
user2.update(avatar: uploaded_file)
user2.uploads.delete_all
project2.update(avatar: uploaded_file)
UploadService.new(project2, uploaded_file, FileUploader).execute # Markdown upload
upload_result = UploadService.new(project2, uploaded_file, FileUploader).execute # Markdown upload
@project2_markdown_upload_path = upload_result[:url].sub(/\A\/uploads\//, '')
project2.uploads.delete_all
appearance.update(header_logo: uploaded_file)
appearance.uploads.last.destroy
end
it 'schedules background migrations' do
it 'tracks untracked migrations' do
Sidekiq::Testing.inline! do
migrate!
# Tracked uploads still exist
expect(user1.uploads.first.attributes).to include({
"path" => "uploads/-/system/user/avatar/1/rails_sample.jpg",
expect(user1.reload.uploads.first.attributes).to include({
"path" => "uploads/-/system/user/avatar/#{user1.id}/rails_sample.jpg",
"uploader" => "AvatarUploader"
})
expect(project1.uploads.first.attributes).to include({
"path" => "uploads/-/system/project/avatar/1/rails_sample.jpg",
expect(project1.reload.uploads.first.attributes).to include({
"path" => "uploads/-/system/project/avatar/#{project1.id}/rails_sample.jpg",
"uploader" => "AvatarUploader"
})
expect(appearance.uploads.first.attributes).to include({
"path" => "uploads/-/system/appearance/logo/1/rails_sample.jpg",
expect(appearance.reload.uploads.first.attributes).to include({
"path" => "uploads/-/system/appearance/logo/#{appearance.id}/rails_sample.jpg",
"uploader" => "AttachmentUploader"
})
expect(project1.uploads.last.path).to match(/\w+\/rails_sample\.jpg/)
expect(project1.uploads.last.uploader).to eq('FileUploader')
expect(project1.uploads.last.attributes).to include({
"path" => @project1_markdown_upload_path,
"uploader" => "FileUploader"
})
# Untracked uploads are now tracked
expect(user2.uploads.first.attributes).to include({
"path" => "uploads/-/system/user/avatar/2/rails_sample.jpg",
expect(user2.reload.uploads.first.attributes).to include({
"path" => "uploads/-/system/user/avatar/#{user2.id}/rails_sample.jpg",
"uploader" => "AvatarUploader"
})
expect(project2.uploads.first.attributes).to include({
"path" => "uploads/-/system/project/avatar/2/rails_sample.jpg",
expect(project2.reload.uploads.first.attributes).to include({
"path" => "uploads/-/system/project/avatar/#{project2.id}/rails_sample.jpg",
"uploader" => "AvatarUploader"
})
expect(appearance.uploads.count).to eq(2)
expect(appearance.reload.uploads.count).to eq(2)
expect(appearance.uploads.last.attributes).to include({
"path" => "uploads/-/system/appearance/header_logo/1/rails_sample.jpg",
"path" => "uploads/-/system/appearance/header_logo/#{appearance.id}/rails_sample.jpg",
"uploader" => "AttachmentUploader"
})
expect(project2.uploads.last.path).to match(/\w+\/rails_sample\.jpg/)
expect(project2.uploads.last.uploader).to eq('FileUploader')
expect(project2.uploads.last.attributes).to include({
"path" => @project2_markdown_upload_path,
"uploader" => "FileUploader"
})
end
end
it 'all UnhashedUploadFile records are marked as tracked' do
Sidekiq::Testing.inline! do
migrate!
expect(UnhashedUploadFile.count).to eq(8)
expect(UnhashedUploadFile.count).to eq(UnhashedUploadFile.where(tracked: true).count)
end
end
end
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment