Commit de5f0e73 authored by Tiago Botelho's avatar Tiago Botelho
Browse files

Backports every CE related change from ee-44542 to CE

parent 3c3c3c07938a
class Import::BaseController < ApplicationController class Import::BaseController < ApplicationController
private private
def find_already_added_projects(import_type)
current_user.created_projects.where(import_type: import_type).includes(:import_state)
end
def find_jobs(import_type)
current_user.created_projects
.includes(:import_state)
.where(import_type: import_type)
.to_json(only: [:id], methods: [:import_status])
end
def find_or_create_namespace(names, owner) def find_or_create_namespace(names, owner)
names = params[:target_namespace].presence || names names = params[:target_namespace].presence || names
......
...@@ -22,16 +22,14 @@ def status ...@@ -22,16 +22,14 @@ def status
@repos, @incompatible_repos = repos.partition { |repo| repo.valid? } @repos, @incompatible_repos = repos.partition { |repo| repo.valid? }
@already_added_projects = current_user.created_projects.where(import_type: 'bitbucket') @already_added_projects = find_already_added_projects('bitbucket')
already_added_projects_names = @already_added_projects.pluck(:import_source) already_added_projects_names = @already_added_projects.pluck(:import_source)
@repos.to_a.reject! { |repo| already_added_projects_names.include?(repo.full_name) } @repos.to_a.reject! { |repo| already_added_projects_names.include?(repo.full_name) }
end end
def jobs def jobs
render json: current_user.created_projects render json: find_jobs('bitbucket')
.where(import_type: 'bitbucket')
.to_json(only: [:id, :import_status])
end end
def create def create
......
...@@ -46,15 +46,14 @@ def status ...@@ -46,15 +46,14 @@ def status
@repos = client.repos @repos = client.repos
@already_added_projects = current_user.created_projects.where(import_type: 'fogbugz') @already_added_projects = find_already_added_projects('fogbugz')
already_added_projects_names = @already_added_projects.pluck(:import_source) already_added_projects_names = @already_added_projects.pluck(:import_source)
@repos.reject! { |repo| already_added_projects_names.include? repo.name } @repos.reject! { |repo| already_added_projects_names.include? repo.name }
end end
def jobs def jobs
jobs = current_user.created_projects.where(import_type: 'fogbugz').to_json(only: [:id, :import_status]) render json: find_jobs('fogbugz')
render json: jobs
end end
def create def create
......
...@@ -24,15 +24,14 @@ def personal_access_token ...@@ -24,15 +24,14 @@ def personal_access_token
def status def status
@repos = client.repos @repos = client.repos
@already_added_projects = current_user.created_projects.where(import_type: provider) @already_added_projects = find_already_added_projects(provider)
already_added_projects_names = @already_added_projects.pluck(:import_source) already_added_projects_names = @already_added_projects.pluck(:import_source)
@repos.reject! { |repo| already_added_projects_names.include? repo.full_name } @repos.reject! { |repo| already_added_projects_names.include? repo.full_name }
end end
def jobs def jobs
jobs = current_user.created_projects.where(import_type: provider).to_json(only: [:id, :import_status]) render json: find_jobs(provider)
render json: jobs
end end
def create def create
......
...@@ -12,15 +12,14 @@ def callback ...@@ -12,15 +12,14 @@ def callback
def status def status
@repos = client.projects @repos = client.projects
@already_added_projects = current_user.created_projects.where(import_type: "gitlab") @already_added_projects = find_already_added_projects('gitlab')
already_added_projects_names = @already_added_projects.pluck(:import_source) already_added_projects_names = @already_added_projects.pluck(:import_source)
@repos = @repos.to_a.reject { |repo| already_added_projects_names.include? repo["path_with_namespace"] } @repos = @repos.to_a.reject { |repo| already_added_projects_names.include? repo["path_with_namespace"] }
end end
def jobs def jobs
jobs = current_user.created_projects.where(import_type: "gitlab").to_json(only: [:id, :import_status]) render json: find_jobs('gitlab')
render json: jobs
end end
def create def create
......
...@@ -73,15 +73,14 @@ def status ...@@ -73,15 +73,14 @@ def status
@repos = client.repos @repos = client.repos
@incompatible_repos = client.incompatible_repos @incompatible_repos = client.incompatible_repos
@already_added_projects = current_user.created_projects.where(import_type: "google_code") @already_added_projects = find_already_added_projects('google_code')
already_added_projects_names = @already_added_projects.pluck(:import_source) already_added_projects_names = @already_added_projects.pluck(:import_source)
@repos.reject! { |repo| already_added_projects_names.include? repo.name } @repos.reject! { |repo| already_added_projects_names.include? repo.name }
end end
def jobs def jobs
jobs = current_user.created_projects.where(import_type: "google_code").to_json(only: [:id, :import_status]) render json: find_jobs('google_code')
render json: jobs
end end
def create def create
......
...@@ -67,6 +67,9 @@ class Project < ActiveRecord::Base ...@@ -67,6 +67,9 @@ class Project < ActiveRecord::Base
before_save :ensure_runners_token before_save :ensure_runners_token
after_save :update_project_statistics, if: :namespace_id_changed? after_save :update_project_statistics, if: :namespace_id_changed?
after_save :create_import_state, if: ->(project) { project.import? && project.import_state.nil? }
after_create :create_project_feature, unless: :project_feature after_create :create_project_feature, unless: :project_feature
after_create :create_ci_cd_settings, after_create :create_ci_cd_settings,
...@@ -157,6 +160,8 @@ class Project < ActiveRecord::Base ...@@ -157,6 +160,8 @@ class Project < ActiveRecord::Base
has_one :fork_network_member has_one :fork_network_member
has_one :fork_network, through: :fork_network_member has_one :fork_network, through: :fork_network_member
has_one :import_state, autosave: true, class_name: 'ProjectImportState', inverse_of: :project
# Merge Requests for target project should be removed with it # Merge Requests for target project should be removed with it
has_many :merge_requests, foreign_key: 'target_project_id' has_many :merge_requests, foreign_key: 'target_project_id'
has_many :source_of_merge_requests, foreign_key: 'source_project_id', class_name: 'MergeRequest' has_many :source_of_merge_requests, foreign_key: 'source_project_id', class_name: 'MergeRequest'
...@@ -385,55 +390,9 @@ def self.with_feature_available_for_user(feature, user) ...@@ -385,55 +390,9 @@ def self.with_feature_available_for_user(feature, user)
scope :abandoned, -> { where('projects.last_activity_at < ?', 6.months.ago) } scope :abandoned, -> { where('projects.last_activity_at < ?', 6.months.ago) }
scope :excluding_project, ->(project) { where.not(id: project) } scope :excluding_project, ->(project) { where.not(id: project) }
scope :import_started, -> { where(import_status: 'started') }
state_machine :import_status, initial: :none do
event :import_schedule do
transition [:none, :finished, :failed] => :scheduled
end
event :force_import_start do
transition [:none, :finished, :failed] => :started
end
event :import_start do
transition scheduled: :started
end
event :import_finish do
transition started: :finished
end
event :import_fail do
transition [:scheduled, :started] => :failed
end
event :import_retry do
transition failed: :started
end
state :scheduled
state :started
state :finished
state :failed
after_transition [:none, :finished, :failed] => :scheduled do |project, _|
project.run_after_commit do
job_id = add_import_job
update(import_jid: job_id) if job_id
end
end
after_transition started: :finished do |project, _| scope :joins_import_state, -> { joins("LEFT JOIN project_mirror_data import_state ON import_state.project_id = projects.id") }
project.reset_cache_and_import_attrs scope :import_started, -> { joins_import_state.where("import_state.status = 'started' OR projects.import_status = 'started'") }
if Gitlab::ImportSources.importer_names.include?(project.import_type) && project.repo_exists?
project.run_after_commit do
Projects::AfterImportService.new(project).execute
end
end
end
end
class << self class << self
# Searches for a list of projects based on the query given in `query`. # Searches for a list of projects based on the query given in `query`.
...@@ -663,10 +622,6 @@ def import? ...@@ -663,10 +622,6 @@ def import?
external_import? || forked? || gitlab_project_import? || bare_repository_import? external_import? || forked? || gitlab_project_import? || bare_repository_import?
end end
def no_import?
import_status == 'none'
end
def external_import? def external_import?
import_url.present? import_url.present?
end end
...@@ -679,6 +634,93 @@ def import_in_progress? ...@@ -679,6 +634,93 @@ def import_in_progress?
import_started? || import_scheduled? import_started? || import_scheduled?
end end
def import_state_args
{
status: self[:import_status],
jid: self[:import_jid],
last_error: self[:import_error]
}
end
def ensure_import_state
return if self[:import_status] == 'none' || self[:import_status].nil?
return unless import_state.nil?
create_import_state(import_state_args)
update_column(:import_status, 'none')
end
def import_schedule
ensure_import_state
import_state&.schedule
end
def force_import_start
ensure_import_state
import_state&.force_start
end
def import_start
ensure_import_state
import_state&.start
end
def import_fail
ensure_import_state
import_state&.fail_op
end
def import_finish
ensure_import_state
import_state&.finish
end
def import_jid=(new_jid)
ensure_import_state
import_state&.jid = new_jid
end
def import_jid
ensure_import_state
import_state&.jid
end
def import_error=(new_error)
ensure_import_state
import_state&.last_error = new_error
end
def import_error
ensure_import_state
import_state&.last_error
end
def import_status=(new_status)
ensure_import_state
import_state&.status = new_status
end
def import_status
ensure_import_state
import_state&.status || 'none'
end
def no_import?
import_status == 'none'
end
def import_started? def import_started?
# import? does SQL work so only run it if it looks like there's an import running # import? does SQL work so only run it if it looks like there's an import running
import_status == 'started' && import? import_status == 'started' && import?
...@@ -1480,7 +1522,7 @@ def write_repository_config(gl_full_path: full_path) ...@@ -1480,7 +1522,7 @@ def write_repository_config(gl_full_path: full_path)
def rename_repo_notify! def rename_repo_notify!
# When we import a project overwriting the original project, there # When we import a project overwriting the original project, there
# is a move operation. In that case we don't want to send the instructions. # is a move operation. In that case we don't want to send the instructions.
send_move_instructions(full_path_was) unless started? send_move_instructions(full_path_was) unless import_started?
expires_full_path_cache expires_full_path_cache
self.old_path_with_namespace = full_path_was self.old_path_with_namespace = full_path_was
...@@ -1534,7 +1576,8 @@ def remove_import_jid ...@@ -1534,7 +1576,8 @@ def remove_import_jid
return unless import_jid return unless import_jid
Gitlab::SidekiqStatus.unset(import_jid) Gitlab::SidekiqStatus.unset(import_jid)
update_column(:import_jid, nil)
import_state.update_column(:jid, nil)
end end
def running_or_pending_build_count(force: false) def running_or_pending_build_count(force: false)
...@@ -1553,7 +1596,8 @@ def mark_import_as_failed(error_message) ...@@ -1553,7 +1596,8 @@ def mark_import_as_failed(error_message)
sanitized_message = Gitlab::UrlSanitizer.sanitize(error_message) sanitized_message = Gitlab::UrlSanitizer.sanitize(error_message)
import_fail import_fail
update_column(:import_error, sanitized_message)
import_state.update_column(:last_error, sanitized_message)
rescue ActiveRecord::ActiveRecordError => e rescue ActiveRecord::ActiveRecordError => e
Rails.logger.error("Error setting import status to failed: #{e.message}. Original error: #{sanitized_message}") Rails.logger.error("Error setting import status to failed: #{e.message}. Original error: #{sanitized_message}")
ensure ensure
......
class ProjectImportState < ActiveRecord::Base
include AfterCommitQueue
self.table_name = "project_mirror_data"
belongs_to :project, inverse_of: :import_state
validates :project, presence: true
state_machine :status, initial: :none do
event :schedule do
transition [:none, :finished, :failed] => :scheduled
end
event :force_start do
transition [:none, :finished, :failed] => :started
end
event :start do
transition scheduled: :started
end
event :finish do
transition started: :finished
end
event :fail_op do
transition [:scheduled, :started] => :failed
end
state :scheduled
state :started
state :finished
state :failed
after_transition [:none, :finished, :failed] => :scheduled do |state, _|
state.run_after_commit do
job_id = project.add_import_job
update(jid: job_id) if job_id
end
end
after_transition started: :finished do |state, _|
project = state.project
project.reset_cache_and_import_attrs
if Gitlab::ImportSources.importer_names.include?(project.import_type) && project.repo_exists?
state.run_after_commit do
Projects::AfterImportService.new(project).execute
end
end
end
end
end
...@@ -142,7 +142,7 @@ def fail(error:) ...@@ -142,7 +142,7 @@ def fail(error:)
if @project if @project
@project.errors.add(:base, message) @project.errors.add(:base, message)
@project.mark_import_as_failed(message) if @project.import? @project.mark_import_as_failed(message) if @project.persisted? && @project.import?
end end
@project @project
......
...@@ -63,11 +63,10 @@ def wait_for_jobs(waiters) ...@@ -63,11 +63,10 @@ def wait_for_jobs(waiters)
end end
def find_project(id) def find_project(id)
# We only care about the import JID so we can refresh it. We also only # TODO: Only select the JID
# want the project if it hasn't been marked as failed yet. It's possible # This is due to the fact that the JID could be present in either the project record or
# the import gets marked as stuck when jobs of the current stage failed # its associated import_state record
# somehow. Project.import_started.find_by(id: id)
Project.select(:import_jid).import_started.find_by(id: id)
end end
end end
end end
......
...@@ -31,7 +31,10 @@ def perform(project_id, check_job_id) ...@@ -31,7 +31,10 @@ def perform(project_id, check_job_id)
end end
def find_project(id) def find_project(id)
Project.select(:import_jid).import_started.find_by(id: id) # TODO: Only select the JID
# This is due to the fact that the JID could be present in either the project record or
# its associated import_state record
Project.import_started.find_by(id: id)
end end
end end
end end
......
...@@ -22,7 +22,8 @@ def mark_projects_without_jid_as_failed! ...@@ -22,7 +22,8 @@ def mark_projects_without_jid_as_failed!
end end
def mark_projects_with_jid_as_failed! def mark_projects_with_jid_as_failed!
jids_and_ids = enqueued_projects_with_jid.pluck(:import_jid, :id).to_h # TODO: Rollback this change to use SQL through #pluck
jids_and_ids = enqueued_projects_with_jid.map { |project| [project.import_jid, project.id] }.to_h
# Find the jobs that aren't currently running or that exceeded the threshold. # Find the jobs that aren't currently running or that exceeded the threshold.
completed_jids = Gitlab::SidekiqStatus.completed_jids(jids_and_ids.keys) completed_jids = Gitlab::SidekiqStatus.completed_jids(jids_and_ids.keys)
...@@ -42,15 +43,15 @@ def mark_projects_with_jid_as_failed! ...@@ -42,15 +43,15 @@ def mark_projects_with_jid_as_failed!
end end
def enqueued_projects def enqueued_projects
Project.with_import_status(:scheduled, :started) Project.joins_import_state.where("(import_state.status = 'scheduled' OR import_state.status = 'started') OR (projects.import_status = 'scheduled' OR projects.import_status = 'started')")
end end
def enqueued_projects_with_jid def enqueued_projects_with_jid
enqueued_projects.where.not(import_jid: nil) enqueued_projects.where.not("import_state.jid IS NULL AND projects.import_jid IS NULL")
end end
def enqueued_projects_without_jid def enqueued_projects_without_jid
enqueued_projects.where(import_jid: nil) enqueued_projects.where("import_state.jid IS NULL AND projects.import_jid IS NULL")
end end
def error_message def error_message
......
class CreateProjectMirrorData < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def up
return if table_exists?(:project_mirror_data)
create_table :project_mirror_data do |t|
t.references :project, index: true, foreign_key: { on_delete: :cascade }
t.string :status
t.string :jid
t.text :last_error
end
end
def down
drop_table(:project_mirror_data) if table_exists?(:project_mirror_data)
end
end
class AddIndexesToProjectMirrorData < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_concurrent_index :project_mirror_data, :jid
add_concurrent_index :project_mirror_data, :status
end
def down
remove_index :project_mirror_data, :jid if index_exists? :project_mirror_data, :jid
remove_index :project_mirror_data, :status if index_exists? :project_mirror_data, :status
end
end
class MigrateImportAttributesDataFromProjectsToProjectMirrorData < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
UP_MIGRATION = 'PopulateImportState'.freeze
DOWN_MIGRATION = 'RollbackImportStateData'.freeze