Commit cfa3d1e2 authored by Tiago Botelho's avatar Tiago Botelho
Browse files

backports changed import logic from pull mirroring feature into CE

parent cc8a0c211283
......@@ -56,6 +56,8 @@
if (job.import_status === 'finished') {
job_item.removeClass("active").addClass("success");
return status_field.html('<span><i class="fa fa-check"></i> done</span>');
} else if (job.import_status === 'scheduled') {
return status_field.html("<i class='fa fa-spinner fa-spin'></i> scheduled");
} else if (job.import_status === 'started') {
return status_field.html("<i class='fa fa-spinner fa-spin'></i> started");
} else {
......
......@@ -14,14 +14,7 @@
@project.import_url = params[:project][:import_url]
if @project.save
@project.reload
if @project.import_failed?
@project.import_retry
else
@project.import_start
@project.add_import_job
end
@project.reload.import_schedule
end
redirect_to namespace_project_import_path(@project.namespace, @project)
......
......@@ -199,7 +199,7 @@
ApplicationSetting.define_attribute_methods
end
def self.defaults_ce
def self.defaults
{
after_sign_up_text: nil,
akismet_enabled: false,
......@@ -250,10 +250,6 @@
}
end
def self.defaults
defaults_ce
end
def self.create_from_defaults
create(defaults)
end
......
......@@ -165,7 +165,7 @@
has_many :todos, dependent: :destroy
has_many :notification_settings, dependent: :destroy, as: :source
has_one :import_data, dependent: :destroy, class_name: "ProjectImportData"
has_one :import_data, dependent: :delete, class_name: "ProjectImportData"
has_one :project_feature, dependent: :destroy
has_one :statistics, class_name: 'ProjectStatistics', dependent: :delete
has_many :container_repositories, dependent: :destroy
......@@ -298,4 +298,12 @@
scope :excluding_project, ->(project) { where.not(id: project) }
state_machine :import_status, initial: :none do
event :import_schedule do
transition [:none, :finished, :failed] => :scheduled
end
event :force_import_start do
transition [:none, :finished, :failed] => :started
end
event :import_start do
......@@ -301,5 +309,5 @@
event :import_start do
transition [:none, :finished] => :started
transition scheduled: :started
end
event :import_finish do
......@@ -307,10 +315,10 @@
end
event :import_fail do
transition started: :failed
transition [:scheduled, :started] => :failed
end
event :import_retry do
transition failed: :started
end
......@@ -311,10 +319,11 @@
end
event :import_retry do
transition failed: :started
end
state :scheduled
state :started
state :finished
state :failed
......@@ -317,8 +326,12 @@
state :started
state :finished
state :failed
after_transition any => :finished, do: :reset_cache_and_import_attrs
after_transition [:none, :finished, :failed] => :scheduled do |project, _|
project.run_after_commit { add_import_job }
end
after_transition started: :finished, do: :reset_cache_and_import_attrs
end
class << self
......@@ -530,6 +543,10 @@
end
def import_in_progress?
import_started? || import_scheduled?
end
def import_started?
import? && import_status == 'started'
end
......@@ -533,6 +550,10 @@
import? && import_status == 'started'
end
def import_scheduled?
import_status == 'scheduled'
end
def import_failed?
import_status == 'failed'
end
......
......@@ -48,8 +48,6 @@
save_project_and_import_data(import_data)
@project.import_start if @project.import?
after_create_actions if @project.persisted?
if @project.errors.empty?
......@@ -53,7 +51,7 @@
after_create_actions if @project.persisted?
if @project.errors.empty?
@project.add_import_job if @project.import?
@project.import_schedule if @project.import?
else
fail(error: @project.errors.full_messages.join(', '))
end
......@@ -57,6 +55,7 @@
else
fail(error: @project.errors.full_messages.join(', '))
end
@project
rescue ActiveRecord::RecordInvalid => e
message = "Unable to save #{e.record.type}: #{e.record.errors.full_messages.join(", ")} "
......
class RepositoryForkWorker
ForkError = Class.new(StandardError)
include Sidekiq::Worker
include Gitlab::ShellAdapter
include DedicatedSidekiqQueue
......@@ -8,12 +10,8 @@
source_path: source_path,
target_path: target_path)
project = Project.find_by_id(project_id)
unless project.present?
logger.error("Project #{project_id} no longer exists!")
return
end
project = Project.find(project_id)
project.import_start
result = gitlab_shell.fork_repository(forked_from_repository_storage_path, source_path,
project.repository_storage_path, target_path)
......@@ -17,10 +15,6 @@
result = gitlab_shell.fork_repository(forked_from_repository_storage_path, source_path,
project.repository_storage_path, target_path)
unless result
logger.error("Unable to fork project #{project_id} for repository #{source_path} -> #{target_path}")
project.mark_import_as_failed('The project could not be forked.')
return
end
raise ForkError, "Unable to fork project #{project_id} for repository #{source_path} -> #{target_path}" unless result
project.repository.after_import
......@@ -25,10 +19,5 @@
project.repository.after_import
unless project.valid_repo?
logger.error("Project #{project_id} had an invalid repository after fork")
project.mark_import_as_failed('The forked repository is invalid.')
return
end
raise ForkError, "Project #{project_id} had an invalid repository after fork" unless project.valid_repo?
project.import_finish
......@@ -33,4 +22,19 @@
project.import_finish
rescue ForkError => ex
fail_fork(project, ex.message)
raise
rescue => ex
return unless project
fail_fork(project, ex.message)
raise ForkError, "#{ex.class} #{ex.message}"
end
private
def fail_fork(project, message)
Rails.logger.error(message)
project.mark_import_as_failed(message)
end
end
class RepositoryImportWorker
ImportError = Class.new(StandardError)
include Sidekiq::Worker
include DedicatedSidekiqQueue
......@@ -10,6 +12,8 @@
@project = Project.find(project_id)
@current_user = @project.creator
project.import_start
Gitlab::Metrics.add_event(:import_repository,
import_url: @project.import_url,
path: @project.path_with_namespace)
......@@ -17,11 +21,7 @@
project.update_columns(import_jid: self.jid, import_error: nil)
result = Projects::ImportService.new(project, current_user).execute
if result[:status] == :error
project.mark_import_as_failed(result[:message])
return
end
raise ImportError, result[:message] if result[:status] == :error
project.repository.after_import
project.import_finish
......@@ -25,5 +25,19 @@
project.repository.after_import
project.import_finish
rescue ImportError => ex
fail_import(project, ex.message)
raise
rescue => ex
return unless project
fail_import(project, ex.message)
raise ImportError, "#{ex.class} #{ex.message}"
end
private
def fail_import(project, message)
project.mark_import_as_failed(message)
end
end
......@@ -37,7 +37,7 @@
end
def import!
@project.import_start
@project.force_import_start
timings = Benchmark.measure do
Github::Import.new(@project, @options).execute
......
......@@ -7,5 +7,9 @@
link.forked_from_project.reload
link.forked_to_project.reload
end
trait :forked_to_empty_project do
association :forked_to_project, factory: :empty_project
end
end
end
......@@ -24,6 +24,22 @@
visibility_level Gitlab::VisibilityLevel::PRIVATE
end
trait :import_scheduled do
import_status :scheduled
end
trait :import_started do
import_status :started
end
trait :import_finished do
import_status :finished
end
trait :import_failed do
import_status :failed
end
trait :archived do
archived true
end
......
......@@ -68,6 +68,10 @@
let(:merge_request) { create(:merge_request_with_diffs, source_project: forked_project, target_project: project, author: author_user) }
let(:changelog_id) { Digest::SHA1.hexdigest("CHANGELOG") }
before do
forked_project.repository.after_import
end
context 'as author' do
it 'shows direct edit link' do
login_as(author_user)
......@@ -71,6 +75,7 @@
context 'as author' do
it 'shows direct edit link' do
login_as(author_user)
visit diffs_namespace_project_merge_request_path(project.namespace, project, merge_request)
# Throws `Capybara::Poltergeist::InvalidSelector` if we try to use `#hash` syntax
......@@ -81,6 +86,7 @@
context 'as user who needs to fork' do
it 'shows fork/cancel confirmation' do
login_as(user)
visit diffs_namespace_project_merge_request_path(project.namespace, project, merge_request)
# Throws `Capybara::Poltergeist::InvalidSelector` if we try to use `#hash` syntax
......
......@@ -50,7 +50,7 @@
it { is_expected.to have_one(:external_wiki_service).dependent(:destroy) }
it { is_expected.to have_one(:project_feature).dependent(:destroy) }
it { is_expected.to have_one(:statistics).class_name('ProjectStatistics').dependent(:delete) }
it { is_expected.to have_one(:import_data).class_name('ProjectImportData').dependent(:destroy) }
it { is_expected.to have_one(:import_data).class_name('ProjectImportData').dependent(:delete) }
it { is_expected.to have_one(:last_event).class_name('Event') }
it { is_expected.to have_one(:forked_from_project).through(:forked_project_link) }
it { is_expected.to have_many(:commit_statuses) }
......@@ -1446,11 +1446,10 @@
end
describe 'Project import job' do
let(:project) { create(:empty_project) }
let(:mirror) { false }
let(:project) { create(:empty_project, import_url: generate(:url)) }
before do
allow_any_instance_of(Gitlab::Shell).to receive(:import_repository)
.with(project.repository_storage_path, project.path_with_namespace, project.import_url)
.and_return(true)
......@@ -1451,11 +1450,9 @@
before do
allow_any_instance_of(Gitlab::Shell).to receive(:import_repository)
.with(project.repository_storage_path, project.path_with_namespace, project.import_url)
.and_return(true)
allow(project).to receive(:repository_exists?).and_return(true)
expect_any_instance_of(Repository).to receive(:after_import)
.and_call_original
end
......@@ -1463,8 +1460,7 @@
it 'imports a project' do
expect_any_instance_of(RepositoryImportWorker).to receive(:perform).and_call_original
project.import_start
project.add_import_job
project.import_schedule
expect(project.reload.import_status).to eq('finished')
end
......@@ -1551,7 +1547,7 @@
describe '#add_import_job' do
context 'forked' do
let(:forked_project_link) { create(:forked_project_link) }
let(:forked_project_link) { create(:forked_project_link, :forked_to_empty_project) }
let(:forked_from_project) { forked_project_link.forked_from_project }
let(:project) { forked_project_link.forked_to_project }
......@@ -1565,5 +1561,6 @@
end
context 'not forked' do
let(:project) { create(:empty_project) }
it 'schedules a RepositoryImportWorker job' do
project = create(:empty_project, import_url: generate(:url))
......@@ -1569,5 +1566,4 @@
it 'schedules a RepositoryImportWorker job' do
expect(RepositoryImportWorker).to receive(:perform_async).with(project.id)
project.add_import_job
......
......@@ -161,9 +161,5 @@
end
context 'when a bad service template is created' do
before do
create(:service, type: 'DroneCiService', project: nil, template: true, active: true)
end
it 'reports an error in the imported project' do
opts[:import_url] = 'http://www.gitlab.com/gitlab-org/gitlab-ce'
......@@ -168,4 +164,6 @@
it 'reports an error in the imported project' do
opts[:import_url] = 'http://www.gitlab.com/gitlab-org/gitlab-ce'
create(:service, type: 'DroneCiService', project: nil, template: true, active: true)
project = create_project(user, opts)
......@@ -170,6 +168,6 @@
project = create_project(user, opts)
expect(project.errors.full_messages_for(:base).first).to match /Unable to save project. Error: Unable to save DroneCiService/
expect(project.errors.full_messages_for(:base).first).to match(/Unable to save project. Error: Unable to save DroneCiService/)
expect(project.services.count).to eq 0
end
end
......
require 'spec_helper'
describe RepositoryForkWorker do
let(:project) { create(:project, :repository) }
let(:project) { create(:project, :repository, :import_scheduled) }
let(:fork_project) { create(:project, :repository, forked_from_project: project) }
let(:shell) { Gitlab::Shell.new }
......@@ -46,5 +46,9 @@
end
it "handles bad fork" do
source_path = project.full_path
target_path = fork_project.namespace.full_path
error_message = "Unable to fork project #{project.id} for repository #{source_path} -> #{target_path}"
expect(shell).to receive(:fork_repository).and_return(false)
......@@ -49,4 +53,7 @@
expect(shell).to receive(:fork_repository).and_return(false)
expect(subject.logger).to receive(:error)
expect do
subject.perform(project.id, '/test/path', source_path, target_path)
end.to raise_error(RepositoryForkWorker::ForkError, error_message)
end
......@@ -52,9 +59,14 @@
subject.perform(
project.id,
'/test/path',
project.full_path,
fork_project.namespace.full_path)
it 'handles unexpected error' do
source_path = project.full_path
target_path = fork_project.namespace.full_path
allow_any_instance_of(Gitlab::Shell).to receive(:fork_repository).and_raise(RuntimeError)
expect do
subject.perform(project.id, '/test/path', source_path, target_path)
end.to raise_error(RepositoryForkWorker::ForkError)
expect(project.reload.import_status).to eq('failed')
end
end
end
require 'spec_helper'
describe RepositoryImportWorker do
let(:project) { create(:empty_project) }
let(:project) { create(:empty_project, :import_scheduled) }
subject { described_class.new }
......@@ -21,7 +21,7 @@
context 'when the import has failed' do
it 'hide the credentials that were used in the import URL' do
error = %q{remote: Not Found fatal: repository 'https://user:pass@test.com/root/repoC.git/' not found }
expect_any_instance_of(Projects::ImportService).to receive(:execute).
and_return({ status: :error, message: error })
expect_any_instance_of(Projects::ImportService).to receive(:execute).and_return({ status: :error, message: error })
allow(subject).to receive(:jid).and_return('123')
......@@ -26,4 +26,9 @@
allow(subject).to receive(:jid).and_return('123')
subject.perform(project.id)
expect do
subject.perform(project.id)
end.to raise_error(RepositoryImportWorker::ImportError, error)
expect(project.reload.import_jid).not_to be_nil
end
end
......@@ -29,6 +34,12 @@
expect(project.reload.import_error).to include("https://*****:*****@test.com/root/repoC.git/")
expect(project.reload.import_jid).not_to be_nil
context 'with unexpected error' do
it 'marks import as failed' do
allow_any_instance_of(Projects::ImportService).to receive(:execute).and_raise(RuntimeError)
expect do
subject.perform(project.id)
end.to raise_error(RepositoryImportWorker::ImportError)
expect(project.reload.import_status).to eq('failed')
end
end
end
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment