Read about our upcoming Code of Conduct on this issue

This instance will be upgraded to Heptapod 0.28.1 on 2022-01-26 at 16:00 UTC+1 (a few minutes of down time)

Commit 6a0bbb5a authored by James Lopez's avatar James Lopez
Browse files

using shared path for project import uploads and refactored gitlab remove export worker

parent bdabaed511e4
......@@ -47,6 +47,9 @@ v 8.11.0 (unreleased)
- Fix RequestProfiler::Middleware error when code is reloaded in development
- Catch what warden might throw when profiling requests to re-throw it
v 8.10.4 (unreleased)
- Fix Import/Export project import not working in HA mode
v 8.10.3
- Fix Import/Export issue importing milestones and labels not associated properly. !5426
- Fix timing problems running imports on production. !5523
......
......@@ -12,13 +12,14 @@ def create
return redirect_back_or_default(options: { alert: "You need to upload a GitLab project export archive." })
end
imported_file = project_params[:file].path + "-import"
import_upload_path = Gitlab::ImportExport.import_upload_path(filename: project_params[:file].original_filename)
FileUtils.copy_entry(project_params[:file].path, imported_file)
FileUtils.mkdir_p(File.dirname(import_upload_path))
FileUtils.copy_entry(project_params[:file].path, import_upload_path)
@project = Gitlab::ImportExport::ProjectCreator.new(project_params[:namespace_id],
current_user,
File.expand_path(imported_file),
import_upload_path,
project_params[:path]).execute
if @project.saved?
......
......@@ -378,11 +378,6 @@ def trending(since = 1.month.ago)
joins(join_body).reorder('join_note_counts.amount DESC')
end
# Deletes gitlab project export files older than 24 hours
def remove_gitlab_exports!
Gitlab::Popen.popen(%W(find #{Gitlab::ImportExport.storage_path} -not -path #{Gitlab::ImportExport.storage_path} -mmin +1440 -delete))
end
end
def repository_storage_path
......
class ImportExportCleanUpService
LAST_MODIFIED_TIME_IN_MINUTES = 1440
attr_reader :mmin, :path
def initialize(mmin = LAST_MODIFIED_TIME_IN_MINUTES)
@mmin = mmin
@path = Gitlab::ImportExport.storage_path
end
def execute
Gitlab::Metrics.measure(:import_export_clean_up) do
return unless File.directory?(path)
clean_up_export_files
end
end
private
def clean_up_export_files
Gitlab::Popen.popen(%W(find #{path} -not -path #{path} -mmin +#{mmin} -delete))
end
end
class RepositoryArchiveCleanUpService
LAST_MODIFIED_TIME_IN_MINUTES = 120
attr_reader :mmin, :path
def initialize(mmin = LAST_MODIFIED_TIME_IN_MINUTES)
@mmin = mmin
@path = Gitlab.config.gitlab.repository_downloads_path
......@@ -17,8 +19,6 @@ def execute
private
attr_reader :mmin, :path
def clean_up_old_archives
run(%W(find #{path} -not -path #{path} -type f \( -name \*.tar -o -name \*.bz2 -o -name \*.tar.gz -o -name \*.zip \) -maxdepth 2 -mmin +#{mmin} -delete))
end
......
class GitlabRemoveProjectExportWorker
class ImportExportProjectCleanupWorker
include Sidekiq::Worker
sidekiq_options queue: :default
def perform
Project.remove_gitlab_exports!
ImportExportCleanUpService.new.execute
end
end
......@@ -287,9 +287,9 @@ def host(url)
Settings.cron_jobs['repository_archive_cache_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['repository_archive_cache_worker']['cron'] ||= '0 * * * *'
Settings.cron_jobs['repository_archive_cache_worker']['job_class'] = 'RepositoryArchiveCacheWorker'
Settings.cron_jobs['gitlab_remove_project_export_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['gitlab_remove_project_export_worker']['cron'] ||= '0 * * * *'
Settings.cron_jobs['gitlab_remove_project_export_worker']['job_class'] = 'GitlabRemoveProjectExportWorker'
Settings.cron_jobs['import_export_project_cleanup_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['import_export_project_cleanup_worker']['cron'] ||= '0 * * * *'
Settings.cron_jobs['import_export_project_cleanup_worker']['job_class'] = 'ImportExportProjectCleanupWorker'
Settings.cron_jobs['requests_profiles_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['requests_profiles_worker']['cron'] ||= '0 0 * * *'
Settings.cron_jobs['requests_profiles_worker']['job_class'] = 'RequestsProfilesWorker'
......
......@@ -13,6 +13,10 @@ def storage_path
File.join(Settings.shared['path'], 'tmp/project_exports')
end
def import_upload_path(filename:)
File.join(storage_path, 'uploads', filename)
end
def project_filename
"project.json"
end
......
require 'spec_helper'
describe ImportExportCleanUpService, services: true do
describe '#execute' do
let(:service) { described_class.new }
let(:tmp_import_export_folder) { 'tmp/project_exports' }
context 'when the import/export directory does not exist' do
it 'does not remove any archives' do
path = '/invalid/path/'
stub_repository_downloads_path(path)
expect(File).to receive(:directory?).with(path + tmp_import_export_folder).and_return(false).at_least(:once)
expect(service).not_to receive(:clean_up_export_files)
service.execute
end
end
context 'when the import/export directory exists' do
it 'removes old files' do
in_directory_with_files(mtime: 2.days.ago) do |dir, files|
service.execute
files.each { |file| expect(File.exist?(file)).to eq false }
expect(File.directory?(dir)).to eq false
end
end
it 'does not remove new files' do
in_directory_with_files(mtime: 2.hours.ago) do |dir, files|
service.execute
files.each { |file| expect(File.exist?(file)).to eq true }
expect(File.directory?(dir)).to eq true
end
end
end
def in_directory_with_files(mtime:)
Dir.mktmpdir do |tmpdir|
stub_repository_downloads_path(tmpdir)
dir = File.join(tmpdir, tmp_import_export_folder, 'subfolder')
FileUtils.mkdir_p(dir)
files = FileUtils.touch(file_list(dir) + [dir], mtime: mtime)
yield(dir, files)
end
end
def stub_repository_downloads_path(path)
new_shared_settings = Settings.shared.merge('path' => path)
allow(Settings).to receive(:shared).and_return(new_shared_settings)
end
def file_list(dir)
Array.new(5) do |num|
File.join(dir, "random-#{num}.tar.gz")
end
end
end
end
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment