Skip to content
GitLab
Menu
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
heptapod
heptapod
Commits
64bff8ce6d26
Commit
64bff8ce
authored
May 03, 2018
by
Tiago Botelho
Browse files
Backports every CE related change from ee-5484 to CE
parent
8ce3dd144d5a
Changes
33
Hide whitespace changes
Inline
Side-by-side
app/controllers/projects/mirrors_controller.rb
0 → 100644
View file @
64bff8ce
class
Projects::MirrorsController
<
Projects
::
ApplicationController
include
RepositorySettingsRedirect
# Authorize
before_action
:authorize_admin_mirror!
before_action
:remote_mirror
,
only:
[
:update
]
layout
"project_settings"
def
show
redirect_to_repository_settings
(
project
)
end
def
update
if
project
.
update_attributes
(
mirror_params
)
flash
[
:notice
]
=
'Mirroring settings were successfully updated.'
else
flash
[
:alert
]
=
project
.
errors
.
full_messages
.
join
(
', '
).
html_safe
end
respond_to
do
|
format
|
format
.
html
{
redirect_to_repository_settings
(
project
)
}
format
.
json
do
if
project
.
errors
.
present?
render
json:
project
.
errors
,
status: :unprocessable_entity
else
render
json:
ProjectMirrorSerializer
.
new
.
represent
(
project
)
end
end
end
end
def
update_now
if
params
[
:sync_remote
]
project
.
update_remote_mirrors
flash
[
:notice
]
=
"The remote repository is being updated..."
end
redirect_to_repository_settings
(
project
)
end
private
def
remote_mirror
@remote_mirror
=
project
.
remote_mirrors
.
first_or_initialize
end
def
mirror_params_attributes
[
remote_mirrors_attributes:
%i[
url
id
enabled
only_protected_branches
]
]
end
def
mirror_params
params
.
require
(
:project
).
permit
(
mirror_params_attributes
)
end
end
app/controllers/projects/settings/repository_controller.rb
View file @
64bff8ce
...
...
@@ -2,6 +2,7 @@
module
Settings
class
RepositoryController
<
Projects
::
ApplicationController
before_action
:authorize_admin_project!
before_action
:remote_mirror
,
only:
[
:show
]
def
show
render_show
...
...
@@ -25,6 +26,7 @@
define_deploy_token
define_protected_refs
remote_mirror
render
'show'
end
...
...
@@ -41,6 +43,12 @@
load_gon_index
end
def
remote_mirror
return
unless
project
.
feature_available?
(
:repository_mirrors
)
@remote_mirror
=
project
.
remote_mirrors
.
first_or_initialize
end
def
access_levels_options
{
create_access_levels:
levels_for_dropdown
,
...
...
app/models/project.rb
View file @
64bff8ce
...
...
@@ -64,6 +64,9 @@
default_value_for
:only_allow_merge_if_all_discussions_are_resolved
,
false
add_authentication_token_field
:runners_token
before_validation
:mark_remote_mirrors_for_removal
before_save
:ensure_runners_token
after_save
:update_project_statistics
,
if: :namespace_id_changed?
...
...
@@ -241,8 +244,10 @@
has_many
:project_badges
,
class_name:
'ProjectBadge'
has_one
:ci_cd_settings
,
class_name:
'ProjectCiCdSetting'
,
inverse_of: :project
,
autosave:
true
,
dependent: :destroy
# rubocop:disable Cop/ActiveRecordDependent
has_many
:remote_mirrors
,
inverse_of: :project
accepts_nested_attributes_for
:variables
,
allow_destroy:
true
accepts_nested_attributes_for
:project_feature
,
update_only:
true
accepts_nested_attributes_for
:import_data
accepts_nested_attributes_for
:auto_devops
,
update_only:
true
...
...
@@ -244,8 +249,12 @@
accepts_nested_attributes_for
:variables
,
allow_destroy:
true
accepts_nested_attributes_for
:project_feature
,
update_only:
true
accepts_nested_attributes_for
:import_data
accepts_nested_attributes_for
:auto_devops
,
update_only:
true
accepts_nested_attributes_for
:remote_mirrors
,
allow_destroy:
true
,
reject_if:
->
(
attrs
)
{
attrs
[
:id
].
blank?
&&
attrs
[
:url
].
blank?
}
delegate
:name
,
to: :owner
,
allow_nil:
true
,
prefix:
true
delegate
:members
,
to: :team
,
prefix:
true
delegate
:add_user
,
:add_users
,
to: :team
...
...
@@ -335,6 +344,7 @@
scope
:with_issues_enabled
,
->
{
with_feature_enabled
(
:issues
)
}
scope
:with_issues_available_for_user
,
->
(
current_user
)
{
with_feature_available_for_user
(
:issues
,
current_user
)
}
scope
:with_merge_requests_enabled
,
->
{
with_feature_enabled
(
:merge_requests
)
}
scope
:with_remote_mirrors
,
->
{
joins
(
:remote_mirrors
).
where
(
remote_mirrors:
{
enabled:
true
}).
distinct
}
scope
:with_group_runners_enabled
,
->
do
joins
(
:ci_cd_settings
)
...
...
@@ -754,6 +764,37 @@
import_type
==
'gitea'
end
def
has_remote_mirror?
remote_mirror_available?
&&
remote_mirrors
.
enabled
.
exists?
end
def
updating_remote_mirror?
remote_mirrors
.
enabled
.
started
.
exists?
end
def
update_remote_mirrors
return
unless
remote_mirror_available?
remote_mirrors
.
enabled
.
each
(
&
:sync
)
end
def
mark_stuck_remote_mirrors_as_failed!
remote_mirrors
.
stuck
.
update_all
(
update_status: :failed
,
last_error:
'The remote mirror took to long to complete.'
,
last_update_at:
Time
.
now
)
end
def
mark_remote_mirrors_for_removal
remote_mirrors
.
each
(
&
:mark_for_delete_if_blank_url
)
end
def
remote_mirror_available?
remote_mirror_available_overridden
||
::
Gitlab
::
CurrentSettings
.
mirror_available
end
def
check_limit
unless
creator
.
can_create_project?
||
namespace
.
kind
==
'group'
projects_limit
=
creator
.
projects_limit
...
...
app/models/remote_mirror.rb
0 → 100644
View file @
64bff8ce
class
RemoteMirror
<
ActiveRecord
::
Base
include
AfterCommitQueue
PROTECTED_BACKOFF_DELAY
=
1
.
minute
UNPROTECTED_BACKOFF_DELAY
=
5
.
minutes
attr_encrypted
:credentials
,
key:
Gitlab
::
Application
.
secrets
.
db_key_base
,
marshal:
true
,
encode:
true
,
mode: :per_attribute_iv_and_salt
,
insecure_mode:
true
,
algorithm:
'aes-256-cbc'
default_value_for
:only_protected_branches
,
true
belongs_to
:project
,
inverse_of: :remote_mirrors
validates
:url
,
presence:
true
,
url:
{
protocols:
%w(ssh git http https)
,
allow_blank:
true
}
validates
:url
,
addressable_url:
true
,
if: :url_changed?
before_save
:set_new_remote_name
,
if: :mirror_url_changed?
after_save
:set_override_remote_mirror_available
,
unless:
->
{
Gitlab
::
CurrentSettings
.
current_application_settings
.
mirror_available
}
after_save
:refresh_remote
,
if: :mirror_url_changed?
after_update
:reset_fields
,
if: :mirror_url_changed?
after_commit
:remove_remote
,
on: :destroy
scope
:enabled
,
->
{
where
(
enabled:
true
)
}
scope
:started
,
->
{
with_update_status
(
:started
)
}
scope
:stuck
,
->
{
started
.
where
(
'last_update_at < ? OR (last_update_at IS NULL AND updated_at < ?)'
,
1
.
day
.
ago
,
1
.
day
.
ago
)
}
state_machine
:update_status
,
initial: :none
do
event
:update_start
do
transition
[
:none
,
:finished
,
:failed
]
=>
:started
end
event
:update_finish
do
transition
started: :finished
end
event
:update_fail
do
transition
started: :failed
end
state
:started
state
:finished
state
:failed
after_transition
any
=>
:started
do
|
remote_mirror
,
_
|
Gitlab
::
Metrics
.
add_event
(
:remote_mirrors_running
,
path:
remote_mirror
.
project
.
full_path
)
remote_mirror
.
update
(
last_update_started_at:
Time
.
now
)
end
after_transition
started: :finished
do
|
remote_mirror
,
_
|
Gitlab
::
Metrics
.
add_event
(
:remote_mirrors_finished
,
path:
remote_mirror
.
project
.
full_path
)
timestamp
=
Time
.
now
remote_mirror
.
update_attributes!
(
last_update_at:
timestamp
,
last_successful_update_at:
timestamp
,
last_error:
nil
)
end
after_transition
started: :failed
do
|
remote_mirror
,
_
|
Gitlab
::
Metrics
.
add_event
(
:remote_mirrors_failed
,
path:
remote_mirror
.
project
.
full_path
)
remote_mirror
.
update
(
last_update_at:
Time
.
now
)
end
end
def
remote_name
super
||
fallback_remote_name
end
def
update_failed?
update_status
==
'failed'
end
def
update_in_progress?
update_status
==
'started'
end
def
update_repository
(
options
)
raw
.
update
(
options
)
end
def
sync
return
unless
enabled?
return
if
Gitlab
::
Geo
.
secondary?
if
recently_scheduled?
RepositoryUpdateRemoteMirrorWorker
.
perform_in
(
backoff_delay
,
self
.
id
,
Time
.
now
)
else
RepositoryUpdateRemoteMirrorWorker
.
perform_async
(
self
.
id
,
Time
.
now
)
end
end
def
enabled
return
false
unless
project
&&
super
return
false
unless
project
.
remote_mirror_available?
return
false
unless
project
.
repository_exists?
return
false
if
project
.
pending_delete?
true
end
alias_method
:enabled?
,
:enabled
def
updated_since?
(
timestamp
)
last_update_started_at
&&
last_update_started_at
>
timestamp
&&
!
update_failed?
end
def
mark_for_delete_if_blank_url
mark_for_destruction
if
url
.
blank?
end
def
mark_as_failed
(
error_message
)
update_fail
update_column
(
:last_error
,
Gitlab
::
UrlSanitizer
.
sanitize
(
error_message
))
end
def
url
=
(
value
)
super
(
value
)
&&
return
unless
Gitlab
::
UrlSanitizer
.
valid?
(
value
)
mirror_url
=
Gitlab
::
UrlSanitizer
.
new
(
value
)
self
.
credentials
=
mirror_url
.
credentials
super
(
mirror_url
.
sanitized_url
)
end
def
url
if
super
Gitlab
::
UrlSanitizer
.
new
(
super
,
credentials:
credentials
).
full_url
end
rescue
super
end
def
safe_url
return
if
url
.
nil?
result
=
URI
.
parse
(
url
)
result
.
password
=
'*****'
if
result
.
password
result
.
user
=
'*****'
if
result
.
user
&&
result
.
user
!=
"git"
# tokens or other data may be saved as user
result
.
to_s
end
private
def
raw
@raw
||=
Gitlab
::
Git
::
RemoteMirror
.
new
(
project
.
repository
.
raw
,
remote_name
)
end
def
fallback_remote_name
return
unless
id
"remote_mirror_
#{
id
}
"
end
def
recently_scheduled?
return
false
unless
self
.
last_update_started_at
self
.
last_update_started_at
>=
Time
.
now
-
backoff_delay
end
def
backoff_delay
if
self
.
only_protected_branches
PROTECTED_BACKOFF_DELAY
else
UNPROTECTED_BACKOFF_DELAY
end
end
def
reset_fields
update_columns
(
last_error:
nil
,
last_update_at:
nil
,
last_successful_update_at:
nil
,
update_status:
'finished'
)
end
def
set_override_remote_mirror_available
enabled
=
read_attribute
(
:enabled
)
project
.
update
(
remote_mirror_available_overridden:
enabled
)
end
def
set_new_remote_name
self
.
remote_name
=
"remote_mirror_
#{
SecureRandom
.
hex
}
"
end
def
refresh_remote
return
unless
project
# Before adding a new remote we have to delete the data from
# the previous remote name
prev_remote_name
=
remote_name_was
||
fallback_remote_name
run_after_commit
do
project
.
repository
.
async_remove_remote
(
prev_remote_name
)
end
project
.
repository
.
add_remote
(
remote_name
,
url
)
end
def
remove_remote
return
unless
project
# could be pending to delete so don't need to touch the git repository
project
.
repository
.
async_remove_remote
(
remote_name
)
end
def
mirror_url_changed?
url_changed?
||
encrypted_credentials_changed?
end
end
app/models/repository.rb
View file @
64bff8ce
...
...
@@ -861,6 +861,20 @@
gitlab_shell
.
fetch_remote
(
raw_repository
,
remote
,
ssh_auth:
ssh_auth
,
forced:
forced
,
no_tags:
no_tags
,
prune:
prune
)
end
def
async_remove_remote
(
remote_name
)
return
unless
remote_name
job_id
=
RepositoryRemoveRemoteWorker
.
perform_async
(
project
.
id
,
remote_name
)
if
job_id
Rails
.
logger
.
info
(
"Remove remote job scheduled for
#{
project
.
id
}
with remote name:
#{
remote_name
}
job ID
#{
job_id
}
."
)
else
Rails
.
logger
.
info
(
"Remove remote job failed to create for
#{
project
.
id
}
with remote name
#{
remote_name
}
."
)
end
job_id
end
def
fetch_source_branch!
(
source_repository
,
source_branch
,
local_ref
)
raw_repository
.
fetch_source_branch!
(
source_repository
.
raw_repository
,
source_branch
,
local_ref
)
end
...
...
app/serializers/project_mirror_entity.rb
0 → 100644
View file @
64bff8ce
class
ProjectMirrorEntity
<
Grape
::
Entity
prepend
::
EE
::
ProjectMirrorEntity
expose
:id
expose
:remote_mirrors_attributes
do
|
project
|
next
[]
unless
project
.
remote_mirrors
.
present?
project
.
remote_mirrors
.
map
do
|
remote
|
remote
.
as_json
(
only:
%i[id url enabled]
)
end
end
end
app/services/git_push_service.rb
View file @
64bff8ce
...
...
@@ -55,6 +55,7 @@
execute_related_hooks
perform_housekeeping
update_remote_mirrors
update_caches
update_signatures
...
...
@@ -119,6 +120,13 @@
protected
def
update_remote_mirrors
return
unless
@project
.
has_remote_mirror?
@project
.
mark_stuck_remote_mirrors_as_failed!
@project
.
update_remote_mirrors
end
def
execute_related_hooks
# Update merge requests that may be affected by this push. A new branch
# could cause the last commit of a merge request to change.
...
...
app/services/projects/update_remote_mirror_service.rb
0 → 100644
View file @
64bff8ce
module
Projects
class
UpdateRemoteMirrorService
<
BaseService
attr_reader
:errors
def
execute
(
remote_mirror
)
@errors
=
[]
return
success
unless
remote_mirror
.
enabled?
begin
repository
.
fetch_remote
(
remote_mirror
.
remote_name
,
no_tags:
true
)
opts
=
{}
if
remote_mirror
.
only_protected_branches?
opts
[
:only_branches_matching
]
=
project
.
protected_branches
.
select
(
:name
).
map
(
&
:name
)
end
remote_mirror
.
update_repository
(
opts
)
rescue
=>
e
errors
<<
e
.
message
.
strip
end
if
errors
.
present?
error
(
errors
.
join
(
"
\n\n
"
))
else
success
end
end
end
end
app/views/projects/mirrors/_push.html.haml
0 → 100644
View file @
64bff8ce
-
expanded
=
Rails
.
env
.
test?
%section
.settings.no-animate
{
class:
(
'expanded'
if
expanded
)
}
.settings-header
%h4
Push to a remote repository
%button
.btn.js-settings-toggle
=
expanded
?
'Collapse'
:
'Expand'
%p
Set up the remote repository that you want to update with the content of the current repository
every time someone pushes to it.
=
link_to
'Read more'
,
help_page_path
(
'workflow/repository_mirroring'
,
anchor:
'pushing-to-a-remote-repository'
),
target:
'_blank'
.settings-content
=
form_for
@project
,
url:
project_mirror_path
(
@project
)
do
|
f
|
%div
=
form_errors
(
@project
)
=
render
"shared/remote_mirror_update_button"
,
remote_mirror:
@remote_mirror
-
if
@remote_mirror
.
last_error
.
present?
.panel.panel-danger
.panel-heading
-
if
@remote_mirror
.
last_update_at
The remote repository failed to update
#{
time_ago_with_tooltip
(
@remote_mirror
.
last_update_at
)
}
.
-
else
The remote repository failed to update.
-
if
@remote_mirror
.
last_successful_update_at
Last successful update
#{
time_ago_with_tooltip
(
@remote_mirror
.
last_successful_update_at
)
}
.
.panel-body
%pre
:preserve
#{
h
(
@remote_mirror
.
last_error
.
strip
)
}
=
f
.
fields_for
:remote_mirrors
,
@remote_mirror
do
|
rm_form
|
.form-group
=
rm_form
.
check_box
:enabled
,
class:
"pull-left"
.prepend-left-20
=
rm_form
.
label
:enabled
,
"Remote mirror repository"
,
class:
"label-light append-bottom-0"
%p
.light.append-bottom-0
Automatically update the remote mirror's branches, tags, and commits from this repository every time someone pushes to it.
.form-group.has-feedback
=
rm_form
.
label
:url
,
"Git repository URL"
,
class:
"label-light"
=
rm_form
.
text_field
:url
,
class:
"form-control"
,
placeholder:
'https://username:password@gitlab.company.com/group/project.git'
=
render
"projects/mirrors/instructions"
.form-group
=
rm_form
.
check_box
:only_protected_branches
,
class:
'pull-left'
.prepend-left-20
=
rm_form
.
label
:only_protected_branches
,
class:
'label-light'
=
link_to
icon
(
'question-circle'
),
help_page_path
(
'user/project/protected_branches'
)
=
f
.
submit
'Save changes'
,
class:
'btn btn-create'
,
name:
'update_remote_mirror'
app/views/projects/mirrors/_show.html.haml
0 → 100644
View file @
64bff8ce
-
if
can?
(
current_user
,
:admin_mirror
,
@project
)
=
render
'projects/mirrors/push'
app/views/shared/_remote_mirror_update_button.html.haml
0 → 100644
View file @
64bff8ce
-
if
@project
.
has_remote_mirror?
.append-bottom-default
-
if
remote_mirror
.
update_in_progress?
%span
.btn.disabled
=
icon
(
"refresh spin"
)
Updating
…
-
else
=
link_to
update_now_project_mirror_path
(
@project
,
sync_remote:
true
),
method: :post
,
class:
"btn"
do
=
icon
(
"refresh"
)
Update Now
-
if
@remote_mirror
.
last_successful_update_at
%p
.inline.prepend-left-10
Successfully updated
#{
time_ago_with_tooltip
(
@remote_mirror
.
last_successful_update_at
)
}
.
app/workers/all_queues.yml
View file @
64bff8ce
...
...
@@ -112,3 +112,4 @@
-
update_user_activity
-
upload_checksum
-
web_hook
-
repository_update_remote_mirror
app/workers/repository_remove_remote_worker.rb
0 → 100644
View file @
64bff8ce
class
RepositoryRemoveRemoteWorker
include
ApplicationWorker
include
ExclusiveLeaseGuard
LEASE_TIMEOUT
=
1
.
hour
attr_reader
:project
,
:remote_name
def
perform
(
project_id
,
remote_name
)
@remote_name
=
remote_name
@project
=
Project
.
find_by_id
(
project_id
)
return
unless
@project
logger
.
info
(
"Removing remote
#{
remote_name
}
from project
#{
project
.
id
}
"
)
try_obtain_lease
do
remove_remote
=
@project
.
repository
.
remove_remote
(
remote_name
)
if
remove_remote
logger
.
info
(
"Remote
#{
remote_name
}
was successfully removed from project
#{
project
.
id
}
"
)
else
logger
.
error
(
"Could not remove remote
#{
remote_name
}
from project
#{
project
.
id
}
"
)
end
end
end
def
lease_timeout
LEASE_TIMEOUT
end
def
lease_key
"remove_remote_
#{
project
.
id
}
_
#{
remote_name
}
"
end
end
app/workers/repository_update_remote_mirror_worker.rb
0 → 100644
View file @
64bff8ce
class
RepositoryUpdateRemoteMirrorWorker
UpdateAlreadyInProgressError
=
Class
.
new
(
StandardError
)
UpdateError
=
Class
.
new
(
StandardError
)
include
ApplicationWorker
include
Gitlab
::
ShellAdapter
sidekiq_options
retry:
3
,
dead:
false
sidekiq_retry_in
{
|
count
|
30
*
count
}
sidekiq_retries_exhausted
do
|
msg
,
_
|
Sidekiq
.
logger
.
warn
"Failed
#{
msg
[
'class'
]
}
with
#{
msg
[
'args'
]
}
:
#{
msg
[
'error_message'
]
}
"
end
def
perform
(
remote_mirror_id
,
scheduled_time
)
remote_mirror
=
RemoteMirror
.
find
(
remote_mirror_id
)
return
if
remote_mirror
.
updated_since?
(
scheduled_time
)
raise
UpdateAlreadyInProgressError
if
remote_mirror
.
update_in_progress?