Read about our upcoming Code of Conduct on this issue

Commit 67145764 authored by Nick Thomas's avatar Nick Thomas
Browse files

Use BFG object maps to clean projects

parent 01d258e93b46
......@@ -50,10 +50,11 @@ function hideOrShowHelpBlock(form) {
}
$(() => {
const $form = $('form.js-requires-input');
if ($form) {
$('form.js-requires-input').each((i, el) => {
const $form = $(el);
$form.requiresInput();
hideOrShowHelpBlock($form);
$('.select2.js-select-namespace').change(() => hideOrShowHelpBlock($form));
}
});
});
export default (buttonSelector, fileSelector) => {
const btn = document.querySelector(buttonSelector);
const fileInput = document.querySelector(fileSelector);
const form = btn.closest('form');
btn.addEventListener('click', () => {
fileInput.click();
});
fileInput.addEventListener('change', () => {
form.querySelector('.js-filename').textContent = fileInput.value.replace(/^.*[\\\/]/, ''); // eslint-disable-line no-useless-escape
});
};
......@@ -3,8 +3,8 @@ import initSettingsPanels from '~/settings_panels';
import setupProjectEdit from '~/project_edit';
import initConfirmDangerModal from '~/confirm_danger_modal';
import mountBadgeSettings from '~/pages/shared/mount_badge_settings';
import fileUpload from '~/lib/utils/file_upload';
import initProjectLoadingSpinner from '../shared/save_project_loader';
import projectAvatar from '../shared/project_avatar';
import initProjectPermissionsSettings from '../shared/permissions';
document.addEventListener('DOMContentLoaded', () => {
......@@ -12,7 +12,7 @@ document.addEventListener('DOMContentLoaded', () => {
setupProjectEdit();
// Initialize expandable settings panels
initSettingsPanels();
projectAvatar();
fileUpload('.js-choose-project-avatar-button', '.js-project-avatar-input');
initProjectPermissionsSettings();
initConfirmDangerModal();
mountBadgeSettings(PROJECT_BADGE);
......
......@@ -7,6 +7,7 @@ import initDeployKeys from '~/deploy_keys';
import ProtectedBranchCreate from '~/protected_branches/protected_branch_create';
import ProtectedBranchEditList from '~/protected_branches/protected_branch_edit_list';
import DueDateSelectors from '~/due_date_select';
import fileUpload from '~/lib/utils/file_upload';
export default () => {
new ProtectedTagCreate();
......@@ -16,4 +17,5 @@ export default () => {
new ProtectedBranchCreate();
new ProtectedBranchEditList();
new DueDateSelectors();
fileUpload('.js-choose-file', '.js-object-map-input');
};
import $ from 'jquery';
export default function projectAvatar() {
$('.js-choose-project-avatar-button').bind('click', function onClickAvatar() {
const form = $(this).closest('form');
return form.find('.js-project-avatar-input').click();
});
$('.js-project-avatar-input').bind('change', function onClickAvatarInput() {
const form = $(this).closest('form');
const filename = $(this)
.val()
.replace(/^.*[\\\/]/, ''); // eslint-disable-line no-useless-escape
return form.find('.js-avatar-filename').text(filename);
});
}
......@@ -5,6 +5,7 @@ module Settings
class RepositoryController < Projects::ApplicationController
before_action :authorize_admin_project!
before_action :remote_mirror, only: [:show]
before_action :check_cleanup_feature_flag!, only: :cleanup
def show
render_show
......@@ -20,8 +21,26 @@ def create_deploy_token
render_show
end
def cleanup
cleanup_params = params.require(:project).permit(:bfg_object_map)
result = Projects::UpdateService.new(project, current_user, cleanup_params).execute
if result[:status] == :success
RepositoryCleanupWorker.perform_async(project.id, current_user.id)
flash[:notice] = _('Repository cleanup has started. You will receive an email once the cleanup operation is complete.')
else
flash[:alert] = _('Failed to upload object map file')
end
redirect_to project_settings_repository_path(project)
end
private
def check_cleanup_feature_flag!
render_404 unless ::Feature.enabled?(:project_cleanup, project)
end
def render_show
@deploy_keys = DeployKeysPresenter.new(@project, current_user: current_user)
@deploy_tokens = @project.deploy_tokens.active
......
......@@ -257,6 +257,10 @@ def xcode_uri_to_repo(project = @project)
"xcode://clone?repo=#{CGI.escape(default_url_to_repo(project))}"
end
def link_to_bfg
link_to 'BFG', 'https://rtyley.github.io/bfg-repo-cleaner/', target: '_blank', rel: 'noopener noreferrer'
end
def legacy_render_context(params)
params[:legacy_render] ? { markdown_engine: :redcarpet } : {}
end
......
......@@ -24,6 +24,21 @@ def project_was_not_exported_email(current_user, project, errors)
subject: subject("Project export error"))
end
def repository_cleanup_success_email(project, user)
@project = project
@user = user
mail(to: user.notification_email, subject: subject("Project cleanup has completed"))
end
def repository_cleanup_failure_email(project, user, error)
@project = project
@user = user
@error = error
mail(to: user.notification_email, subject: subject("Project cleanup failure"))
end
def repository_push_email(project_id, opts = {})
@message =
Gitlab::Email::Message::RepositoryPush.new(self, project_id, opts)
......
......@@ -339,6 +339,7 @@ class Project < ActiveRecord::Base
presence: true,
inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } }
validates :variables, variable_duplicates: { scope: :environment_scope }
validates :bfg_object_map, file_size: { maximum: :max_attachment_size }
# Scopes
scope :pending_delete, -> { where(pending_delete: true) }
......@@ -412,6 +413,9 @@ class Project < ActiveRecord::Base
only_integer: true,
message: 'needs to be beetween 10 minutes and 1 month' }
# Used by Projects::CleanupService to hold a map of rewritten object IDs
mount_uploader :bfg_object_map, AttachmentUploader
# Returns a project, if it is not about to be removed.
#
# id - The ID of the project to retrieve.
......@@ -1973,6 +1977,10 @@ def snippets_visible?(user = nil)
Ability.allowed?(user, :read_project_snippet, self)
end
def max_attachment_size
Gitlab::CurrentSettings.max_attachment_size.megabytes.to_i
end
private
def use_hashed_storage
......
......@@ -466,6 +466,14 @@ def issue_due(issue)
end
end
def repository_cleanup_success(project, user)
mailer.send(:repository_cleanup_success_email, project, user).deliver_later
end
def repository_cleanup_failure(project, user, error)
mailer.send(:repository_cleanup_failure_email, project, user, error).deliver_later
end
protected
def new_resource_email(target, method)
......
# frozen_string_literal: true
module Projects
# The CleanupService removes data from the project repository following a
# BFG rewrite: https://rtyley.github.io/bfg-repo-cleaner/
#
# Before executing this service, all refs rewritten by BFG should have been
# pushed to the repository
class CleanupService < BaseService
NoUploadError = StandardError.new("Couldn't find uploaded object map")
include Gitlab::Utils::StrongMemoize
# Attempt to clean up the project following the push. Warning: this is
# destructive!
#
# path is the path of an upload of a BFG object map file. It contains a line
# per rewritten object, with the old and new SHAs space-separated. It can be
# used to update or remove content that references the objects that BFG has
# altered
#
# Currently, only the project repository is modified by this service, but we
# may wish to modify other data sources in the future.
def execute
apply_bfg_object_map!
# Remove older objects that are no longer referenced
GitGarbageCollectWorker.new.perform(project.id, :gc)
# The cache may now be inaccurate, and holding onto it could prevent
# bugs assuming the presence of some object from manifesting for some
# time. Better to feel the pain immediately.
project.repository.expire_all_method_caches
project.bfg_object_map.remove!
end
private
def apply_bfg_object_map!
raise NoUploadError unless project.bfg_object_map.exists?
project.bfg_object_map.open do |io|
repository_cleaner.apply_bfg_object_map(io)
end
end
def repository_cleaner
@repository_cleaner ||= Gitlab::Git::RepositoryCleaner.new(repository.raw)
end
end
end
......@@ -9,7 +9,7 @@
%p
= message
%p
= s_('403|Please contact your GitLab administrator to get the permission.')
= s_('403|Please contact your GitLab administrator to get permission.')
.action-container.js-go-back{ style: 'display: none' }
%a{ href: 'javascript:history.back()', class: 'btn btn-success' }
= s_('Go Back')
......
Repository cleanup failed on <%= @project.web_url %>
<%= @error %>
Repository cleanup succeeded on <%= @project.web_url %>
Repository size is now <%= "%.1f" % (@project.repository.size || 0) %> MiB
- return unless Feature.enabled?(:project_cleanup, @project)
- expanded = Rails.env.test?
%section.settings.no-animate#cleanup{ class: ('expanded' if expanded) }
.settings-header
%h4= _('Repository cleanup')
%button.btn.js-settings-toggle
= expanded ? _('Collapse') : _('Expand')
%p
= _("Clean up after running %{bfg} on the repository" % { bfg: link_to_bfg }).html_safe
= link_to icon('question-circle'),
help_page_path('user/project/repository/reducing_the_repo_size_using_git.md'),
target: '_blank', rel: 'noopener noreferrer'
.settings-content
- url = cleanup_namespace_project_settings_repository_path(@project.namespace, @project)
= form_for @project, url: url, method: :post, authenticity_token: true, html: { class: 'js-requires-input' } do |f|
%fieldset.prepend-top-0.append-bottom-10
.append-bottom-10
%h5.prepend-top-0
= _("Upload object map")
%button.btn.btn-default.js-choose-file{ type: "button" }
= _("Choose a file")
%span.prepend-left-default.js-filename
= _("No file selected")
= f.file_field :bfg_object_map, accept: 'text/plain', class: "hidden js-object-map-input", required: true
.form-text.text-muted
= _("The maximum file size allowed is %{max_attachment_size}mb") % { max_attachment_size: Gitlab::CurrentSettings.max_attachment_size }
= f.submit _('Start cleanup'), class: 'btn btn-success'
......@@ -53,7 +53,7 @@
= _("Project avatar in repository: %{link}").html_safe % { link: @project.avatar_in_git }
.prepend-top-5.append-bottom-10
%button.btn.js-choose-project-avatar-button{ type: 'button' }= _("Choose file...")
%span.file_name.prepend-left-default.js-avatar-filename= _("No file chosen")
%span.file_name.prepend-left-default.js-filename= _("No file chosen")
= f.file_field :avatar, class: "js-project-avatar-input hidden"
.form-text.text-muted= _("The maximum file size allowed is 200KB.")
- if @project.avatar?
......
......@@ -13,3 +13,4 @@
= render "projects/protected_tags/index"
= render @deploy_keys
= render "projects/deploy_tokens/index"
= render "projects/cleanup/show"
......@@ -133,3 +133,4 @@
- create_note_diff_file
- delete_diff_files
- detect_repository_languages
- repository_cleanup
# frozen_string_literal: true
class RepositoryCleanupWorker
include ApplicationWorker
sidekiq_options retry: 3
sidekiq_retries_exhausted do |msg, err|
next if err.is_a?(ActiveRecord::RecordNotFound)
args = msg['args'] + [msg['error_message']]
new.perform_failure(*args)
end
def perform(project_id, user_id)
project = Project.find(project_id)
user = User.find(user_id)
Projects::CleanupService.new(project, user).execute
notification_service.repository_cleanup_success(project, user)
end
def perform_failure(project_id, user_id, error)
project = Project.find(project_id)
user = User.find(user_id)
# Ensure the file is removed
project.bfg_object_map.remove!
notification_service.repository_cleanup_failure(project, user, error)
end
private
def notification_service
@notification_service ||= NotificationService.new
end
end
---
title: Use BFG object maps to clean projects
merge_request: 23189
author:
type: added
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment