Skip to content
GitLab
Menu
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
heptapod
heptapod
Commits
06825a72e2ed
Commit
06825a72
authored
Nov 19, 2018
by
Nick Thomas
Browse files
Use BFG object maps to clean projects
parent
01d258e93b46
Changes
40
Hide whitespace changes
Inline
Side-by-side
app/assets/javascripts/behaviors/requires_input.js
View file @
06825a72
...
...
@@ -50,8 +50,9 @@
}
$
(()
=>
{
const
$form
=
$
(
'
form.js-requires-input
'
);
if
(
$form
)
{
$
(
'
form.js-requires-input
'
).
each
((
i
,
el
)
=>
{
const
$form
=
$
(
el
);
$form
.
requiresInput
();
hideOrShowHelpBlock
(
$form
);
$
(
'
.select2.js-select-namespace
'
).
change
(()
=>
hideOrShowHelpBlock
(
$form
));
...
...
@@ -55,5 +56,5 @@
$form
.
requiresInput
();
hideOrShowHelpBlock
(
$form
);
$
(
'
.select2.js-select-namespace
'
).
change
(()
=>
hideOrShowHelpBlock
(
$form
));
}
}
);
});
app/assets/javascripts/lib/utils/file_upload.js
0 → 100644
View file @
06825a72
export
default
(
buttonSelector
,
fileSelector
)
=>
{
const
btn
=
document
.
querySelector
(
buttonSelector
);
const
fileInput
=
document
.
querySelector
(
fileSelector
);
const
form
=
btn
.
closest
(
'
form
'
);
btn
.
addEventListener
(
'
click
'
,
()
=>
{
fileInput
.
click
();
});
fileInput
.
addEventListener
(
'
change
'
,
()
=>
{
form
.
querySelector
(
'
.js-filename
'
).
textContent
=
fileInput
.
value
.
replace
(
/^.*
[\\\/]
/
,
''
);
// eslint-disable-line no-useless-escape
});
};
app/assets/javascripts/pages/projects/edit/index.js
View file @
06825a72
...
...
@@ -3,4 +3,5 @@
import
setupProjectEdit
from
'
~/project_edit
'
;
import
initConfirmDangerModal
from
'
~/confirm_danger_modal
'
;
import
mountBadgeSettings
from
'
~/pages/shared/mount_badge_settings
'
;
import
fileUpload
from
'
~/lib/utils/file_upload
'
;
import
initProjectLoadingSpinner
from
'
../shared/save_project_loader
'
;
...
...
@@ -6,5 +7,4 @@
import
initProjectLoadingSpinner
from
'
../shared/save_project_loader
'
;
import
projectAvatar
from
'
../shared/project_avatar
'
;
import
initProjectPermissionsSettings
from
'
../shared/permissions
'
;
document
.
addEventListener
(
'
DOMContentLoaded
'
,
()
=>
{
...
...
@@ -12,7 +12,7 @@
setupProjectEdit
();
// Initialize expandable settings panels
initSettingsPanels
();
projectAvatar
(
);
fileUpload
(
'
.js-choose-project-avatar-button
'
,
'
.js-project-avatar-input
'
);
initProjectPermissionsSettings
();
initConfirmDangerModal
();
mountBadgeSettings
(
PROJECT_BADGE
);
...
...
app/assets/javascripts/pages/projects/settings/repository/form.js
View file @
06825a72
...
...
@@ -7,6 +7,7 @@
import
ProtectedBranchCreate
from
'
~/protected_branches/protected_branch_create
'
;
import
ProtectedBranchEditList
from
'
~/protected_branches/protected_branch_edit_list
'
;
import
DueDateSelectors
from
'
~/due_date_select
'
;
import
fileUpload
from
'
~/lib/utils/file_upload
'
;
export
default
()
=>
{
new
ProtectedTagCreate
();
...
...
@@ -16,4 +17,5 @@
new
ProtectedBranchCreate
();
new
ProtectedBranchEditList
();
new
DueDateSelectors
();
fileUpload
(
'
.js-choose-file
'
,
'
.js-object-map-input
'
);
};
app/assets/javascripts/pages/projects/shared/project_avatar.js
deleted
100644 → 0
View file @
01d258e9
import
$
from
'
jquery
'
;
export
default
function
projectAvatar
()
{
$
(
'
.js-choose-project-avatar-button
'
).
bind
(
'
click
'
,
function
onClickAvatar
()
{
const
form
=
$
(
this
).
closest
(
'
form
'
);
return
form
.
find
(
'
.js-project-avatar-input
'
).
click
();
});
$
(
'
.js-project-avatar-input
'
).
bind
(
'
change
'
,
function
onClickAvatarInput
()
{
const
form
=
$
(
this
).
closest
(
'
form
'
);
const
filename
=
$
(
this
)
.
val
()
.
replace
(
/^.*
[\\\/]
/
,
''
);
// eslint-disable-line no-useless-escape
return
form
.
find
(
'
.js-avatar-filename
'
).
text
(
filename
);
});
}
app/controllers/projects/settings/repository_controller.rb
View file @
06825a72
...
...
@@ -5,6 +5,7 @@
class
RepositoryController
<
Projects
::
ApplicationController
before_action
:authorize_admin_project!
before_action
:remote_mirror
,
only:
[
:show
]
before_action
:check_cleanup_feature_flag!
,
only: :cleanup
def
show
render_show
...
...
@@ -20,5 +21,19 @@
render_show
end
def
cleanup
cleanup_params
=
params
.
require
(
:project
).
permit
(
:bfg_object_map
)
result
=
Projects
::
UpdateService
.
new
(
project
,
current_user
,
cleanup_params
).
execute
if
result
[
:status
]
==
:success
RepositoryCleanupWorker
.
perform_async
(
project
.
id
,
current_user
.
id
)
flash
[
:notice
]
=
_
(
'Repository cleanup has started. You will receive an email once the cleanup operation is complete.'
)
else
flash
[
:alert
]
=
_
(
'Failed to upload object map file'
)
end
redirect_to
project_settings_repository_path
(
project
)
end
private
...
...
@@ -23,5 +38,9 @@
private
def
check_cleanup_feature_flag!
render_404
unless
::
Feature
.
enabled?
(
:project_cleanup
,
project
)
end
def
render_show
@deploy_keys
=
DeployKeysPresenter
.
new
(
@project
,
current_user:
current_user
)
@deploy_tokens
=
@project
.
deploy_tokens
.
active
...
...
app/helpers/projects_helper.rb
View file @
06825a72
...
...
@@ -257,6 +257,10 @@
"xcode://clone?repo=
#{
CGI
.
escape
(
default_url_to_repo
(
project
))
}
"
end
def
link_to_bfg
link_to
'BFG'
,
'https://rtyley.github.io/bfg-repo-cleaner/'
,
target:
'_blank'
,
rel:
'noopener noreferrer'
end
def
legacy_render_context
(
params
)
params
[
:legacy_render
]
?
{
markdown_engine: :redcarpet
}
:
{}
end
...
...
app/mailers/emails/projects.rb
View file @
06825a72
...
...
@@ -24,6 +24,21 @@
subject:
subject
(
"Project export error"
))
end
def
repository_cleanup_success_email
(
project
,
user
)
@project
=
project
@user
=
user
mail
(
to:
user
.
notification_email
,
subject:
subject
(
"Project cleanup has completed"
))
end
def
repository_cleanup_failure_email
(
project
,
user
,
error
)
@project
=
project
@user
=
user
@error
=
error
mail
(
to:
user
.
notification_email
,
subject:
subject
(
"Project cleanup failure"
))
end
def
repository_push_email
(
project_id
,
opts
=
{})
@message
=
Gitlab
::
Email
::
Message
::
RepositoryPush
.
new
(
self
,
project_id
,
opts
)
...
...
app/models/project.rb
View file @
06825a72
...
...
@@ -339,6 +339,7 @@
presence:
true
,
inclusion:
{
in:
->
(
_object
)
{
Gitlab
.
config
.
repositories
.
storages
.
keys
}
}
validates
:variables
,
variable_duplicates:
{
scope: :environment_scope
}
validates
:bfg_object_map
,
file_size:
{
maximum: :max_attachment_size
}
# Scopes
scope
:pending_delete
,
->
{
where
(
pending_delete:
true
)
}
...
...
@@ -412,6 +413,9 @@
only_integer:
true
,
message:
'needs to be beetween 10 minutes and 1 month'
}
# Used by Projects::CleanupService to hold a map of rewritten object IDs
mount_uploader
:bfg_object_map
,
AttachmentUploader
# Returns a project, if it is not about to be removed.
#
# id - The ID of the project to retrieve.
...
...
@@ -1973,6 +1977,10 @@
Ability
.
allowed?
(
user
,
:read_project_snippet
,
self
)
end
def
max_attachment_size
Gitlab
::
CurrentSettings
.
max_attachment_size
.
megabytes
.
to_i
end
private
def
use_hashed_storage
...
...
app/services/notification_service.rb
View file @
06825a72
...
...
@@ -466,6 +466,14 @@
end
end
def
repository_cleanup_success
(
project
,
user
)
mailer
.
send
(
:repository_cleanup_success_email
,
project
,
user
).
deliver_later
end
def
repository_cleanup_failure
(
project
,
user
,
error
)
mailer
.
send
(
:repository_cleanup_failure_email
,
project
,
user
,
error
).
deliver_later
end
protected
def
new_resource_email
(
target
,
method
)
...
...
app/services/projects/cleanup_service.rb
0 → 100644
View file @
06825a72
# frozen_string_literal: true
module
Projects
# The CleanupService removes data from the project repository following a
# BFG rewrite: https://rtyley.github.io/bfg-repo-cleaner/
#
# Before executing this service, all refs rewritten by BFG should have been
# pushed to the repository
class
CleanupService
<
BaseService
NoUploadError
=
StandardError
.
new
(
"Couldn't find uploaded object map"
)
include
Gitlab
::
Utils
::
StrongMemoize
# Attempt to clean up the project following the push. Warning: this is
# destructive!
#
# path is the path of an upload of a BFG object map file. It contains a line
# per rewritten object, with the old and new SHAs space-separated. It can be
# used to update or remove content that references the objects that BFG has
# altered
#
# Currently, only the project repository is modified by this service, but we
# may wish to modify other data sources in the future.
def
execute
apply_bfg_object_map!
# Remove older objects that are no longer referenced
GitGarbageCollectWorker
.
new
.
perform
(
project
.
id
,
:gc
)
# The cache may now be inaccurate, and holding onto it could prevent
# bugs assuming the presence of some object from manifesting for some
# time. Better to feel the pain immediately.
project
.
repository
.
expire_all_method_caches
project
.
bfg_object_map
.
remove!
end
private
def
apply_bfg_object_map!
raise
NoUploadError
unless
project
.
bfg_object_map
.
exists?
project
.
bfg_object_map
.
open
do
|
io
|
repository_cleaner
.
apply_bfg_object_map
(
io
)
end
end
def
repository_cleaner
@repository_cleaner
||=
Gitlab
::
Git
::
RepositoryCleaner
.
new
(
repository
.
raw
)
end
end
end
app/views/errors/access_denied.html.haml
View file @
06825a72
...
...
@@ -9,7 +9,7 @@
%p
=
message
%p
=
s_
(
'403|Please contact your GitLab administrator to get
the
permission.'
)
=
s_
(
'403|Please contact your GitLab administrator to get permission.'
)
.action-container.js-go-back
{
style:
'display: none'
}
%a
{
href:
'javascript:history.back()'
,
class:
'btn btn-success'
}
=
s_
(
'Go Back'
)
...
...
app/views/notify/repository_cleanup_failure_email.text.erb
0 → 100644
View file @
06825a72
Repository cleanup failed on
<%=
@project
.
web_url
%>
<%=
@error
%>
app/views/notify/repository_cleanup_success_email.text.erb
0 → 100644
View file @
06825a72
Repository cleanup succeeded on
<%=
@project
.
web_url
%>
Repository size is now
<%=
"%.1f"
%
(
@project
.
repository
.
size
||
0
)
%>
MiB
app/views/projects/cleanup/_show.html.haml
0 → 100644
View file @
06825a72
-
return
unless
Feature
.
enabled?
(
:project_cleanup
,
@project
)
-
expanded
=
Rails
.
env
.
test?
%section
.settings.no-animate
#cleanup
{
class:
(
'expanded'
if
expanded
)
}
.settings-header
%h4
=
_
(
'Repository cleanup'
)
%button
.btn.js-settings-toggle
=
expanded
?
_
(
'Collapse'
)
:
_
(
'Expand'
)
%p
=
_
(
"Clean up after running %{bfg} on the repository"
%
{
bfg:
link_to_bfg
}).
html_safe
=
link_to
icon
(
'question-circle'
),
help_page_path
(
'user/project/repository/reducing_the_repo_size_using_git.md'
),
target:
'_blank'
,
rel:
'noopener noreferrer'
.settings-content
-
url
=
cleanup_namespace_project_settings_repository_path
(
@project
.
namespace
,
@project
)
=
form_for
@project
,
url:
url
,
method: :post
,
authenticity_token:
true
,
html:
{
class:
'js-requires-input'
}
do
|
f
|
%fieldset
.prepend-top-0.append-bottom-10
.append-bottom-10
%h5
.prepend-top-0
=
_
(
"Upload object map"
)
%button
.btn.btn-default.js-choose-file
{
type:
"button"
}
=
_
(
"Choose a file"
)
%span
.prepend-left-default.js-filename
=
_
(
"No file selected"
)
=
f
.
file_field
:bfg_object_map
,
accept:
'text/plain'
,
class:
"hidden js-object-map-input"
,
required:
true
.form-text.text-muted
=
_
(
"The maximum file size allowed is %{max_attachment_size}mb"
)
%
{
max_attachment_size:
Gitlab
::
CurrentSettings
.
max_attachment_size
}
=
f
.
submit
_
(
'Start cleanup'
),
class:
'btn btn-success'
app/views/projects/edit.html.haml
View file @
06825a72
...
...
@@ -53,7 +53,7 @@
=
_
(
"Project avatar in repository: %{link}"
).
html_safe
%
{
link:
@project
.
avatar_in_git
}
.prepend-top-5.append-bottom-10
%button
.btn.js-choose-project-avatar-button
{
type:
'button'
}=
_
(
"Choose file..."
)
%span
.file_name.prepend-left-default.js-
avatar-
filename
=
_
(
"No file chosen"
)
%span
.file_name.prepend-left-default.js-filename
=
_
(
"No file chosen"
)
=
f
.
file_field
:avatar
,
class:
"js-project-avatar-input hidden"
.form-text.text-muted
=
_
(
"The maximum file size allowed is 200KB."
)
-
if
@project
.
avatar?
...
...
app/views/projects/settings/repository/show.html.haml
View file @
06825a72
...
...
@@ -13,3 +13,4 @@
=
render
"projects/protected_tags/index"
=
render
@deploy_keys
=
render
"projects/deploy_tokens/index"
=
render
"projects/cleanup/show"
app/workers/all_queues.yml
View file @
06825a72
...
...
@@ -133,3 +133,4 @@
-
create_note_diff_file
-
delete_diff_files
-
detect_repository_languages
-
repository_cleanup
app/workers/repository_cleanup_worker.rb
0 → 100644
View file @
06825a72
# frozen_string_literal: true
class
RepositoryCleanupWorker
include
ApplicationWorker
sidekiq_options
retry:
3
sidekiq_retries_exhausted
do
|
msg
,
err
|
next
if
err
.
is_a?
(
ActiveRecord
::
RecordNotFound
)
args
=
msg
[
'args'
]
+
[
msg
[
'error_message'
]]
new
.
perform_failure
(
*
args
)
end
def
perform
(
project_id
,
user_id
)
project
=
Project
.
find
(
project_id
)
user
=
User
.
find
(
user_id
)
Projects
::
CleanupService
.
new
(
project
,
user
).
execute
notification_service
.
repository_cleanup_success
(
project
,
user
)
end
def
perform_failure
(
project_id
,
user_id
,
error
)
project
=
Project
.
find
(
project_id
)
user
=
User
.
find
(
user_id
)
# Ensure the file is removed
project
.
bfg_object_map
.
remove!
notification_service
.
repository_cleanup_failure
(
project
,
user
,
error
)
end
private
def
notification_service
@notification_service
||=
NotificationService
.
new
end
end
changelogs/unreleased/19376-post-bfg-cleanup.yml
0 → 100644
View file @
06825a72
---
title
:
Use BFG object maps to clean projects
merge_request
:
23189
author
:
type
:
added
Prev
1
2
Next
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment