Skip to content
GitLab
Menu
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
heptapod
heptapod
Commits
cfa3d1e2b867
Commit
cfa3d1e2
authored
Jun 01, 2017
by
Tiago Botelho
Browse files
backports changed import logic from pull mirroring feature into CE
parent
cc8a0c211283
Changes
15
Hide whitespace changes
Inline
Side-by-side
app/assets/javascripts/importer_status.js
View file @
cfa3d1e2
...
...
@@ -56,6 +56,8 @@
if
(
job
.
import_status
===
'
finished
'
)
{
job_item
.
removeClass
(
"
active
"
).
addClass
(
"
success
"
);
return
status_field
.
html
(
'
<span><i class="fa fa-check"></i> done</span>
'
);
}
else
if
(
job
.
import_status
===
'
scheduled
'
)
{
return
status_field
.
html
(
"
<i class='fa fa-spinner fa-spin'></i> scheduled
"
);
}
else
if
(
job
.
import_status
===
'
started
'
)
{
return
status_field
.
html
(
"
<i class='fa fa-spinner fa-spin'></i> started
"
);
}
else
{
...
...
app/controllers/projects/imports_controller.rb
View file @
cfa3d1e2
...
...
@@ -14,14 +14,7 @@
@project
.
import_url
=
params
[
:project
][
:import_url
]
if
@project
.
save
@project
.
reload
if
@project
.
import_failed?
@project
.
import_retry
else
@project
.
import_start
@project
.
add_import_job
end
@project
.
reload
.
import_schedule
end
redirect_to
namespace_project_import_path
(
@project
.
namespace
,
@project
)
...
...
app/models/application_setting.rb
View file @
cfa3d1e2
...
...
@@ -199,7 +199,7 @@
ApplicationSetting
.
define_attribute_methods
end
def
self
.
defaults
_ce
def
self
.
defaults
{
after_sign_up_text:
nil
,
akismet_enabled:
false
,
...
...
@@ -250,10 +250,6 @@
}
end
def
self
.
defaults
defaults_ce
end
def
self
.
create_from_defaults
create
(
defaults
)
end
...
...
app/models/project.rb
View file @
cfa3d1e2
...
...
@@ -165,7 +165,7 @@
has_many
:todos
,
dependent: :destroy
has_many
:notification_settings
,
dependent: :destroy
,
as: :source
has_one
:import_data
,
dependent: :de
stroy
,
class_name:
"ProjectImportData"
has_one
:import_data
,
dependent: :de
lete
,
class_name:
"ProjectImportData"
has_one
:project_feature
,
dependent: :destroy
has_one
:statistics
,
class_name:
'ProjectStatistics'
,
dependent: :delete
has_many
:container_repositories
,
dependent: :destroy
...
...
@@ -298,4 +298,12 @@
scope
:excluding_project
,
->
(
project
)
{
where
.
not
(
id:
project
)
}
state_machine
:import_status
,
initial: :none
do
event
:import_schedule
do
transition
[
:none
,
:finished
,
:failed
]
=>
:scheduled
end
event
:force_import_start
do
transition
[
:none
,
:finished
,
:failed
]
=>
:started
end
event
:import_start
do
...
...
@@ -301,5 +309,5 @@
event
:import_start
do
transition
[
:none
,
:finished
]
=>
:started
transition
scheduled:
:started
end
event
:import_finish
do
...
...
@@ -307,10 +315,10 @@
end
event
:import_fail
do
transition
started
:
:failed
transition
[
:scheduled
,
:
started
]
=>
:failed
end
event
:import_retry
do
transition
failed: :started
end
...
...
@@ -311,10 +319,11 @@
end
event
:import_retry
do
transition
failed: :started
end
state
:scheduled
state
:started
state
:finished
state
:failed
...
...
@@ -317,8 +326,12 @@
state
:started
state
:finished
state
:failed
after_transition
any
=>
:finished
,
do: :reset_cache_and_import_attrs
after_transition
[
:none
,
:finished
,
:failed
]
=>
:scheduled
do
|
project
,
_
|
project
.
run_after_commit
{
add_import_job
}
end
after_transition
started: :finished
,
do: :reset_cache_and_import_attrs
end
class
<<
self
...
...
@@ -530,6 +543,10 @@
end
def
import_in_progress?
import_started?
||
import_scheduled?
end
def
import_started?
import?
&&
import_status
==
'started'
end
...
...
@@ -533,6 +550,10 @@
import?
&&
import_status
==
'started'
end
def
import_scheduled?
import_status
==
'scheduled'
end
def
import_failed?
import_status
==
'failed'
end
...
...
app/services/projects/create_service.rb
View file @
cfa3d1e2
...
...
@@ -48,8 +48,6 @@
save_project_and_import_data
(
import_data
)
@project
.
import_start
if
@project
.
import?
after_create_actions
if
@project
.
persisted?
if
@project
.
errors
.
empty?
...
...
@@ -53,7 +51,7 @@
after_create_actions
if
@project
.
persisted?
if
@project
.
errors
.
empty?
@project
.
add_
import_
job
if
@project
.
import?
@project
.
import_
schedule
if
@project
.
import?
else
fail
(
error:
@project
.
errors
.
full_messages
.
join
(
', '
))
end
...
...
@@ -57,6 +55,7 @@
else
fail
(
error:
@project
.
errors
.
full_messages
.
join
(
', '
))
end
@project
rescue
ActiveRecord
::
RecordInvalid
=>
e
message
=
"Unable to save
#{
e
.
record
.
type
}
:
#{
e
.
record
.
errors
.
full_messages
.
join
(
", "
)
}
"
...
...
app/workers/repository_fork_worker.rb
View file @
cfa3d1e2
class
RepositoryForkWorker
ForkError
=
Class
.
new
(
StandardError
)
include
Sidekiq
::
Worker
include
Gitlab
::
ShellAdapter
include
DedicatedSidekiqQueue
...
...
@@ -8,12 +10,8 @@
source_path:
source_path
,
target_path:
target_path
)
project
=
Project
.
find_by_id
(
project_id
)
unless
project
.
present?
logger
.
error
(
"Project
#{
project_id
}
no longer exists!"
)
return
end
project
=
Project
.
find
(
project_id
)
project
.
import_start
result
=
gitlab_shell
.
fork_repository
(
forked_from_repository_storage_path
,
source_path
,
project
.
repository_storage_path
,
target_path
)
...
...
@@ -17,10 +15,6 @@
result
=
gitlab_shell
.
fork_repository
(
forked_from_repository_storage_path
,
source_path
,
project
.
repository_storage_path
,
target_path
)
unless
result
logger
.
error
(
"Unable to fork project
#{
project_id
}
for repository
#{
source_path
}
->
#{
target_path
}
"
)
project
.
mark_import_as_failed
(
'The project could not be forked.'
)
return
end
raise
ForkError
,
"Unable to fork project
#{
project_id
}
for repository
#{
source_path
}
->
#{
target_path
}
"
unless
result
project
.
repository
.
after_import
...
...
@@ -25,10 +19,5 @@
project
.
repository
.
after_import
unless
project
.
valid_repo?
logger
.
error
(
"Project
#{
project_id
}
had an invalid repository after fork"
)
project
.
mark_import_as_failed
(
'The forked repository is invalid.'
)
return
end
raise
ForkError
,
"Project
#{
project_id
}
had an invalid repository after fork"
unless
project
.
valid_repo?
project
.
import_finish
...
...
@@ -33,4 +22,19 @@
project
.
import_finish
rescue
ForkError
=>
ex
fail_fork
(
project
,
ex
.
message
)
raise
rescue
=>
ex
return
unless
project
fail_fork
(
project
,
ex
.
message
)
raise
ForkError
,
"
#{
ex
.
class
}
#{
ex
.
message
}
"
end
private
def
fail_fork
(
project
,
message
)
Rails
.
logger
.
error
(
message
)
project
.
mark_import_as_failed
(
message
)
end
end
app/workers/repository_import_worker.rb
View file @
cfa3d1e2
class
RepositoryImportWorker
ImportError
=
Class
.
new
(
StandardError
)
include
Sidekiq
::
Worker
include
DedicatedSidekiqQueue
...
...
@@ -10,6 +12,8 @@
@project
=
Project
.
find
(
project_id
)
@current_user
=
@project
.
creator
project
.
import_start
Gitlab
::
Metrics
.
add_event
(
:import_repository
,
import_url:
@project
.
import_url
,
path:
@project
.
path_with_namespace
)
...
...
@@ -17,11 +21,7 @@
project
.
update_columns
(
import_jid:
self
.
jid
,
import_error:
nil
)
result
=
Projects
::
ImportService
.
new
(
project
,
current_user
).
execute
if
result
[
:status
]
==
:error
project
.
mark_import_as_failed
(
result
[
:message
])
return
end
raise
ImportError
,
result
[
:message
]
if
result
[
:status
]
==
:error
project
.
repository
.
after_import
project
.
import_finish
...
...
@@ -25,5 +25,19 @@
project
.
repository
.
after_import
project
.
import_finish
rescue
ImportError
=>
ex
fail_import
(
project
,
ex
.
message
)
raise
rescue
=>
ex
return
unless
project
fail_import
(
project
,
ex
.
message
)
raise
ImportError
,
"
#{
ex
.
class
}
#{
ex
.
message
}
"
end
private
def
fail_import
(
project
,
message
)
project
.
mark_import_as_failed
(
message
)
end
end
lib/tasks/import.rake
View file @
cfa3d1e2
...
...
@@ -37,7 +37,7 @@
end
def
import!
@project
.
import_start
@project
.
force_
import_start
timings
=
Benchmark
.
measure
do
Github
::
Import
.
new
(
@project
,
@options
).
execute
...
...
spec/factories/forked_project_links.rb
View file @
cfa3d1e2
...
...
@@ -7,5 +7,9 @@
link
.
forked_from_project
.
reload
link
.
forked_to_project
.
reload
end
trait
:forked_to_empty_project
do
association
:forked_to_project
,
factory: :empty_project
end
end
end
spec/factories/projects.rb
View file @
cfa3d1e2
...
...
@@ -24,6 +24,22 @@
visibility_level
Gitlab
::
VisibilityLevel
::
PRIVATE
end
trait
:import_scheduled
do
import_status
:scheduled
end
trait
:import_started
do
import_status
:started
end
trait
:import_finished
do
import_status
:finished
end
trait
:import_failed
do
import_status
:failed
end
trait
:archived
do
archived
true
end
...
...
spec/features/merge_requests/diffs_spec.rb
View file @
cfa3d1e2
...
...
@@ -68,6 +68,10 @@
let
(
:merge_request
)
{
create
(
:merge_request_with_diffs
,
source_project:
forked_project
,
target_project:
project
,
author:
author_user
)
}
let
(
:changelog_id
)
{
Digest
::
SHA1
.
hexdigest
(
"CHANGELOG"
)
}
before
do
forked_project
.
repository
.
after_import
end
context
'as author'
do
it
'shows direct edit link'
do
login_as
(
author_user
)
...
...
@@ -71,6 +75,7 @@
context
'as author'
do
it
'shows direct edit link'
do
login_as
(
author_user
)
visit
diffs_namespace_project_merge_request_path
(
project
.
namespace
,
project
,
merge_request
)
# Throws `Capybara::Poltergeist::InvalidSelector` if we try to use `#hash` syntax
...
...
@@ -81,6 +86,7 @@
context
'as user who needs to fork'
do
it
'shows fork/cancel confirmation'
do
login_as
(
user
)
visit
diffs_namespace_project_merge_request_path
(
project
.
namespace
,
project
,
merge_request
)
# Throws `Capybara::Poltergeist::InvalidSelector` if we try to use `#hash` syntax
...
...
spec/models/project_spec.rb
View file @
cfa3d1e2
...
...
@@ -50,7 +50,7 @@
it
{
is_expected
.
to
have_one
(
:external_wiki_service
).
dependent
(
:destroy
)
}
it
{
is_expected
.
to
have_one
(
:project_feature
).
dependent
(
:destroy
)
}
it
{
is_expected
.
to
have_one
(
:statistics
).
class_name
(
'ProjectStatistics'
).
dependent
(
:delete
)
}
it
{
is_expected
.
to
have_one
(
:import_data
).
class_name
(
'ProjectImportData'
).
dependent
(
:de
stroy
)
}
it
{
is_expected
.
to
have_one
(
:import_data
).
class_name
(
'ProjectImportData'
).
dependent
(
:de
lete
)
}
it
{
is_expected
.
to
have_one
(
:last_event
).
class_name
(
'Event'
)
}
it
{
is_expected
.
to
have_one
(
:forked_from_project
).
through
(
:forked_project_link
)
}
it
{
is_expected
.
to
have_many
(
:commit_statuses
)
}
...
...
@@ -1446,11 +1446,10 @@
end
describe
'Project import job'
do
let
(
:project
)
{
create
(
:empty_project
)
}
let
(
:mirror
)
{
false
}
let
(
:project
)
{
create
(
:empty_project
,
import_url:
generate
(
:url
))
}
before
do
allow_any_instance_of
(
Gitlab
::
Shell
).
to
receive
(
:import_repository
)
.
with
(
project
.
repository_storage_path
,
project
.
path_with_namespace
,
project
.
import_url
)
.
and_return
(
true
)
...
...
@@ -1451,11 +1450,9 @@
before
do
allow_any_instance_of
(
Gitlab
::
Shell
).
to
receive
(
:import_repository
)
.
with
(
project
.
repository_storage_path
,
project
.
path_with_namespace
,
project
.
import_url
)
.
and_return
(
true
)
allow
(
project
).
to
receive
(
:repository_exists?
).
and_return
(
true
)
expect_any_instance_of
(
Repository
).
to
receive
(
:after_import
)
.
and_call_original
end
...
...
@@ -1463,8 +1460,7 @@
it
'imports a project'
do
expect_any_instance_of
(
RepositoryImportWorker
).
to
receive
(
:perform
).
and_call_original
project
.
import_start
project
.
add_import_job
project
.
import_schedule
expect
(
project
.
reload
.
import_status
).
to
eq
(
'finished'
)
end
...
...
@@ -1551,7 +1547,7 @@
describe
'#add_import_job'
do
context
'forked'
do
let
(
:forked_project_link
)
{
create
(
:forked_project_link
)
}
let
(
:forked_project_link
)
{
create
(
:forked_project_link
,
:forked_to_empty_project
)
}
let
(
:forked_from_project
)
{
forked_project_link
.
forked_from_project
}
let
(
:project
)
{
forked_project_link
.
forked_to_project
}
...
...
@@ -1565,5 +1561,6 @@
end
context
'not forked'
do
let
(
:project
)
{
create
(
:empty_project
)
}
it
'schedules a RepositoryImportWorker job'
do
project
=
create
(
:empty_project
,
import_url:
generate
(
:url
))
...
...
@@ -1569,5 +1566,4 @@
it
'schedules a RepositoryImportWorker job'
do
expect
(
RepositoryImportWorker
).
to
receive
(
:perform_async
).
with
(
project
.
id
)
project
.
add_import_job
...
...
spec/services/projects/create_service_spec.rb
View file @
cfa3d1e2
...
...
@@ -161,9 +161,5 @@
end
context
'when a bad service template is created'
do
before
do
create
(
:service
,
type:
'DroneCiService'
,
project:
nil
,
template:
true
,
active:
true
)
end
it
'reports an error in the imported project'
do
opts
[
:import_url
]
=
'http://www.gitlab.com/gitlab-org/gitlab-ce'
...
...
@@ -168,4 +164,6 @@
it
'reports an error in the imported project'
do
opts
[
:import_url
]
=
'http://www.gitlab.com/gitlab-org/gitlab-ce'
create
(
:service
,
type:
'DroneCiService'
,
project:
nil
,
template:
true
,
active:
true
)
project
=
create_project
(
user
,
opts
)
...
...
@@ -170,6 +168,6 @@
project
=
create_project
(
user
,
opts
)
expect
(
project
.
errors
.
full_messages_for
(
:base
).
first
).
to
match
/Unable to save project. Error: Unable to save DroneCiService/
expect
(
project
.
errors
.
full_messages_for
(
:base
).
first
).
to
match
(
/Unable to save project. Error: Unable to save DroneCiService/
)
expect
(
project
.
services
.
count
).
to
eq
0
end
end
...
...
spec/workers/repository_fork_worker_spec.rb
View file @
cfa3d1e2
require
'spec_helper'
describe
RepositoryForkWorker
do
let
(
:project
)
{
create
(
:project
,
:repository
)
}
let
(
:project
)
{
create
(
:project
,
:repository
,
:import_scheduled
)
}
let
(
:fork_project
)
{
create
(
:project
,
:repository
,
forked_from_project:
project
)
}
let
(
:shell
)
{
Gitlab
::
Shell
.
new
}
...
...
@@ -46,5 +46,9 @@
end
it
"handles bad fork"
do
source_path
=
project
.
full_path
target_path
=
fork_project
.
namespace
.
full_path
error_message
=
"Unable to fork project
#{
project
.
id
}
for repository
#{
source_path
}
->
#{
target_path
}
"
expect
(
shell
).
to
receive
(
:fork_repository
).
and_return
(
false
)
...
...
@@ -49,4 +53,7 @@
expect
(
shell
).
to
receive
(
:fork_repository
).
and_return
(
false
)
expect
(
subject
.
logger
).
to
receive
(
:error
)
expect
do
subject
.
perform
(
project
.
id
,
'/test/path'
,
source_path
,
target_path
)
end
.
to
raise_error
(
RepositoryForkWorker
::
ForkError
,
error_message
)
end
...
...
@@ -52,9 +59,14 @@
subject
.
perform
(
project
.
id
,
'/test/path'
,
project
.
full_path
,
fork_project
.
namespace
.
full_path
)
it
'handles unexpected error'
do
source_path
=
project
.
full_path
target_path
=
fork_project
.
namespace
.
full_path
allow_any_instance_of
(
Gitlab
::
Shell
).
to
receive
(
:fork_repository
).
and_raise
(
RuntimeError
)
expect
do
subject
.
perform
(
project
.
id
,
'/test/path'
,
source_path
,
target_path
)
end
.
to
raise_error
(
RepositoryForkWorker
::
ForkError
)
expect
(
project
.
reload
.
import_status
).
to
eq
(
'failed'
)
end
end
end
spec/workers/repository_import_worker_spec.rb
View file @
cfa3d1e2
require
'spec_helper'
describe
RepositoryImportWorker
do
let
(
:project
)
{
create
(
:empty_project
)
}
let
(
:project
)
{
create
(
:empty_project
,
:import_scheduled
)
}
subject
{
described_class
.
new
}
...
...
@@ -21,7 +21,7 @@
context
'when the import has failed'
do
it
'hide the credentials that were used in the import URL'
do
error
=
%q{remote: Not Found fatal: repository 'https://user:pass@test.com/root/repoC.git/' not found }
expect_any_instance_of
(
Projects
::
ImportService
).
to
receive
(
:execute
).
and_return
({
status: :error
,
message:
error
})
expect_any_instance_of
(
Projects
::
ImportService
).
to
receive
(
:execute
).
and_return
({
status: :error
,
message:
error
})
allow
(
subject
).
to
receive
(
:jid
).
and_return
(
'123'
)
...
...
@@ -26,4 +26,9 @@
allow
(
subject
).
to
receive
(
:jid
).
and_return
(
'123'
)
subject
.
perform
(
project
.
id
)
expect
do
subject
.
perform
(
project
.
id
)
end
.
to
raise_error
(
RepositoryImportWorker
::
ImportError
,
error
)
expect
(
project
.
reload
.
import_jid
).
not_to
be_nil
end
end
...
...
@@ -29,6 +34,12 @@
expect
(
project
.
reload
.
import_error
).
to
include
(
"https://*****:*****@test.com/root/repoC.git/"
)
expect
(
project
.
reload
.
import_jid
).
not_to
be_nil
context
'with unexpected error'
do
it
'marks import as failed'
do
allow_any_instance_of
(
Projects
::
ImportService
).
to
receive
(
:execute
).
and_raise
(
RuntimeError
)
expect
do
subject
.
perform
(
project
.
id
)
end
.
to
raise_error
(
RepositoryImportWorker
::
ImportError
)
expect
(
project
.
reload
.
import_status
).
to
eq
(
'failed'
)
end
end
end
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment