Commit 67145764 authored by Nick Thomas's avatar Nick Thomas
Browse files

Use BFG object maps to clean projects

parent 01d258e93b46
......@@ -50,10 +50,11 @@ function hideOrShowHelpBlock(form) {
}
$(() => {
const $form = $('form.js-requires-input');
if ($form) {
$('form.js-requires-input').each((i, el) => {
const $form = $(el);
$form.requiresInput();
hideOrShowHelpBlock($form);
$('.select2.js-select-namespace').change(() => hideOrShowHelpBlock($form));
}
});
});
export default (buttonSelector, fileSelector) => {
const btn = document.querySelector(buttonSelector);
const fileInput = document.querySelector(fileSelector);
const form = btn.closest('form');
btn.addEventListener('click', () => {
fileInput.click();
});
fileInput.addEventListener('change', () => {
form.querySelector('.js-filename').textContent = fileInput.value.replace(/^.*[\\\/]/, ''); // eslint-disable-line no-useless-escape
});
};
......@@ -3,8 +3,8 @@ import initSettingsPanels from '~/settings_panels';
import setupProjectEdit from '~/project_edit';
import initConfirmDangerModal from '~/confirm_danger_modal';
import mountBadgeSettings from '~/pages/shared/mount_badge_settings';
import fileUpload from '~/lib/utils/file_upload';
import initProjectLoadingSpinner from '../shared/save_project_loader';
import projectAvatar from '../shared/project_avatar';
import initProjectPermissionsSettings from '../shared/permissions';
document.addEventListener('DOMContentLoaded', () => {
......@@ -12,7 +12,7 @@ document.addEventListener('DOMContentLoaded', () => {
setupProjectEdit();
// Initialize expandable settings panels
initSettingsPanels();
projectAvatar();
fileUpload('.js-choose-project-avatar-button', '.js-project-avatar-input');
initProjectPermissionsSettings();
initConfirmDangerModal();
mountBadgeSettings(PROJECT_BADGE);
......
......@@ -7,6 +7,7 @@ import initDeployKeys from '~/deploy_keys';
import ProtectedBranchCreate from '~/protected_branches/protected_branch_create';
import ProtectedBranchEditList from '~/protected_branches/protected_branch_edit_list';
import DueDateSelectors from '~/due_date_select';
import fileUpload from '~/lib/utils/file_upload';
export default () => {
new ProtectedTagCreate();
......@@ -16,4 +17,5 @@ export default () => {
new ProtectedBranchCreate();
new ProtectedBranchEditList();
new DueDateSelectors();
fileUpload('.js-choose-file', '.js-object-map-input');
};
import $ from 'jquery';
export default function projectAvatar() {
$('.js-choose-project-avatar-button').bind('click', function onClickAvatar() {
const form = $(this).closest('form');
return form.find('.js-project-avatar-input').click();
});
$('.js-project-avatar-input').bind('change', function onClickAvatarInput() {
const form = $(this).closest('form');
const filename = $(this)
.val()
.replace(/^.*[\\\/]/, ''); // eslint-disable-line no-useless-escape
return form.find('.js-avatar-filename').text(filename);
});
}
......@@ -5,6 +5,7 @@ module Settings
class RepositoryController < Projects::ApplicationController
before_action :authorize_admin_project!
before_action :remote_mirror, only: [:show]
before_action :check_cleanup_feature_flag!, only: :cleanup
def show
render_show
......@@ -20,8 +21,26 @@ def create_deploy_token
render_show
end
def cleanup
cleanup_params = params.require(:project).permit(:bfg_object_map)
result = Projects::UpdateService.new(project, current_user, cleanup_params).execute
if result[:status] == :success
RepositoryCleanupWorker.perform_async(project.id, current_user.id)
flash[:notice] = _('Repository cleanup has started. You will receive an email once the cleanup operation is complete.')
else
flash[:alert] = _('Failed to upload object map file')
end
redirect_to project_settings_repository_path(project)
end
private
def check_cleanup_feature_flag!
render_404 unless ::Feature.enabled?(:project_cleanup, project)
end
def render_show
@deploy_keys = DeployKeysPresenter.new(@project, current_user: current_user)
@deploy_tokens = @project.deploy_tokens.active
......
......@@ -257,6 +257,10 @@ def xcode_uri_to_repo(project = @project)
"xcode://clone?repo=#{CGI.escape(default_url_to_repo(project))}"
end
def link_to_bfg
link_to 'BFG', 'https://rtyley.github.io/bfg-repo-cleaner/', target: '_blank', rel: 'noopener noreferrer'
end
def legacy_render_context(params)
params[:legacy_render] ? { markdown_engine: :redcarpet } : {}
end
......
......@@ -24,6 +24,21 @@ def project_was_not_exported_email(current_user, project, errors)
subject: subject("Project export error"))
end
def repository_cleanup_success_email(project, user)
@project = project
@user = user
mail(to: user.notification_email, subject: subject("Project cleanup has completed"))
end
def repository_cleanup_failure_email(project, user, error)
@project = project
@user = user
@error = error
mail(to: user.notification_email, subject: subject("Project cleanup failure"))
end
def repository_push_email(project_id, opts = {})
@message =
Gitlab::Email::Message::RepositoryPush.new(self, project_id, opts)
......
......@@ -339,6 +339,7 @@ class Project < ActiveRecord::Base
presence: true,
inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } }
validates :variables, variable_duplicates: { scope: :environment_scope }
validates :bfg_object_map, file_size: { maximum: :max_attachment_size }
# Scopes
scope :pending_delete, -> { where(pending_delete: true) }
......@@ -412,6 +413,9 @@ class Project < ActiveRecord::Base
only_integer: true,
message: 'needs to be beetween 10 minutes and 1 month' }
# Used by Projects::CleanupService to hold a map of rewritten object IDs
mount_uploader :bfg_object_map, AttachmentUploader
# Returns a project, if it is not about to be removed.
#
# id - The ID of the project to retrieve.
......@@ -1973,6 +1977,10 @@ def snippets_visible?(user = nil)
Ability.allowed?(user, :read_project_snippet, self)
end
def max_attachment_size
Gitlab::CurrentSettings.max_attachment_size.megabytes.to_i
end
private
def use_hashed_storage
......
......@@ -466,6 +466,14 @@ def issue_due(issue)
end
end
def repository_cleanup_success(project, user)
mailer.send(:repository_cleanup_success_email, project, user).deliver_later
end
def repository_cleanup_failure(project, user, error)
mailer.send(:repository_cleanup_failure_email, project, user, error).deliver_later
end
protected
def new_resource_email(target, method)
......
# frozen_string_literal: true
module Projects
# The CleanupService removes data from the project repository following a
# BFG rewrite: https://rtyley.github.io/bfg-repo-cleaner/
#
# Before executing this service, all refs rewritten by BFG should have been
# pushed to the repository
class CleanupService < BaseService
NoUploadError = StandardError.new("Couldn't find uploaded object map")
include Gitlab::Utils::StrongMemoize
# Attempt to clean up the project following the push. Warning: this is
# destructive!
#
# path is the path of an upload of a BFG object map file. It contains a line
# per rewritten object, with the old and new SHAs space-separated. It can be
# used to update or remove content that references the objects that BFG has
# altered
#
# Currently, only the project repository is modified by this service, but we
# may wish to modify other data sources in the future.
def execute
apply_bfg_object_map!
# Remove older objects that are no longer referenced
GitGarbageCollectWorker.new.perform(project.id, :gc)
# The cache may now be inaccurate, and holding onto it could prevent
# bugs assuming the presence of some object from manifesting for some
# time. Better to feel the pain immediately.
project.repository.expire_all_method_caches
project.bfg_object_map.remove!
end
private
def apply_bfg_object_map!
raise NoUploadError unless project.bfg_object_map.exists?
project.bfg_object_map.open do |io|
repository_cleaner.apply_bfg_object_map(io)
end
end
def repository_cleaner
@repository_cleaner ||= Gitlab::Git::RepositoryCleaner.new(repository.raw)
end
end
end
......@@ -9,7 +9,7 @@
%p
= message
%p
= s_('403|Please contact your GitLab administrator to get the permission.')
= s_('403|Please contact your GitLab administrator to get permission.')
.action-container.js-go-back{ style: 'display: none' }
%a{ href: 'javascript:history.back()', class: 'btn btn-success' }
= s_('Go Back')
......
Repository cleanup failed on <%= @project.web_url %>
<%= @error %>
Repository cleanup succeeded on <%= @project.web_url %>
Repository size is now <%= "%.1f" % (@project.repository.size || 0) %> MiB
- return unless Feature.enabled?(:project_cleanup, @project)
- expanded = Rails.env.test?
%section.settings.no-animate#cleanup{ class: ('expanded' if expanded) }
.settings-header
%h4= _('Repository cleanup')
%button.btn.js-settings-toggle
= expanded ? _('Collapse') : _('Expand')
%p
= _("Clean up after running %{bfg} on the repository" % { bfg: link_to_bfg }).html_safe
= link_to icon('question-circle'),
help_page_path('user/project/repository/reducing_the_repo_size_using_git.md'),
target: '_blank', rel: 'noopener noreferrer'
.settings-content
- url = cleanup_namespace_project_settings_repository_path(@project.namespace, @project)
= form_for @project, url: url, method: :post, authenticity_token: true, html: { class: 'js-requires-input' } do |f|
%fieldset.prepend-top-0.append-bottom-10
.append-bottom-10
%h5.prepend-top-0
= _("Upload object map")
%button.btn.btn-default.js-choose-file{ type: "button" }
= _("Choose a file")
%span.prepend-left-default.js-filename
= _("No file selected")
= f.file_field :bfg_object_map, accept: 'text/plain', class: "hidden js-object-map-input", required: true
.form-text.text-muted
= _("The maximum file size allowed is %{max_attachment_size}mb") % { max_attachment_size: Gitlab::CurrentSettings.max_attachment_size }
= f.submit _('Start cleanup'), class: 'btn btn-success'
......@@ -53,7 +53,7 @@
= _("Project avatar in repository: %{link}").html_safe % { link: @project.avatar_in_git }
.prepend-top-5.append-bottom-10
%button.btn.js-choose-project-avatar-button{ type: 'button' }= _("Choose file...")
%span.file_name.prepend-left-default.js-avatar-filename= _("No file chosen")
%span.file_name.prepend-left-default.js-filename= _("No file chosen")
= f.file_field :avatar, class: "js-project-avatar-input hidden"
.form-text.text-muted= _("The maximum file size allowed is 200KB.")
- if @project.avatar?
......
......@@ -13,3 +13,4 @@
= render "projects/protected_tags/index"
= render @deploy_keys
= render "projects/deploy_tokens/index"
= render "projects/cleanup/show"
......@@ -133,3 +133,4 @@
- create_note_diff_file
- delete_diff_files
- detect_repository_languages
- repository_cleanup
# frozen_string_literal: true
class RepositoryCleanupWorker
include ApplicationWorker
sidekiq_options retry: 3
sidekiq_retries_exhausted do |msg, err|
next if err.is_a?(ActiveRecord::RecordNotFound)
args = msg['args'] + [msg['error_message']]
new.perform_failure(*args)
end
def perform(project_id, user_id)
project = Project.find(project_id)
user = User.find(user_id)
Projects::CleanupService.new(project, user).execute
notification_service.repository_cleanup_success(project, user)
end
def perform_failure(project_id, user_id, error)
project = Project.find(project_id)
user = User.find(user_id)
# Ensure the file is removed
project.bfg_object_map.remove!
notification_service.repository_cleanup_failure(project, user, error)
end
private
def notification_service
@notification_service ||= NotificationService.new
end
end
---
title: Use BFG object maps to clean projects
merge_request: 23189
author:
type: added
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment