Commit 1b317374 authored by Tiago Botelho's avatar Tiago Botelho
Browse files

Removes all the irrelevant import related code and columns

Clears the import related columns and code from the Project
model over to the ProjectImportState model
parent 2c51eecfe5be
...@@ -13,10 +13,8 @@ def new ...@@ -13,10 +13,8 @@ def new
end end
def create def create
@project.import_url = params[:project][:import_url] if @project.update(safe_import_params)
@project.import_state.reload.schedule
if @project.save
@project.reload.import_schedule
end end
redirect_to project_import_path(@project) redirect_to project_import_path(@project)
...@@ -24,7 +22,7 @@ def create ...@@ -24,7 +22,7 @@ def create
def show def show
if @project.import_finished? if @project.import_finished?
if continue_params if continue_params && continue_params[:to]
redirect_to continue_params[:to], notice: continue_params[:notice] redirect_to continue_params[:to], notice: continue_params[:notice]
else else
redirect_to project_path(@project), notice: finished_notice redirect_to project_path(@project), notice: finished_notice
...@@ -67,4 +65,12 @@ def redirect_if_no_import ...@@ -67,4 +65,12 @@ def redirect_if_no_import
redirect_to project_path(@project) redirect_to project_path(@project)
end end
end end
def import_params
params.require(:project).permit(:import_url)
end
def safe_import_params
import_params
end
end end
...@@ -30,6 +30,7 @@ class Project < ActiveRecord::Base ...@@ -30,6 +30,7 @@ class Project < ActiveRecord::Base
include FeatureGate include FeatureGate
include OptionallySearch include OptionallySearch
include FromUnion include FromUnion
include IgnorableColumn
extend Gitlab::Cache::RequestCache extend Gitlab::Cache::RequestCache
extend Gitlab::ConfigHelper extend Gitlab::ConfigHelper
...@@ -55,6 +56,8 @@ class Project < ActiveRecord::Base ...@@ -55,6 +56,8 @@ class Project < ActiveRecord::Base
VALID_MIRROR_PORTS = [22, 80, 443].freeze VALID_MIRROR_PORTS = [22, 80, 443].freeze
VALID_MIRROR_PROTOCOLS = %w(http https ssh git).freeze VALID_MIRROR_PROTOCOLS = %w(http https ssh git).freeze
ignore_column :import_status, :import_jid, :import_error
cache_markdown_field :description, pipeline: :description cache_markdown_field :description, pipeline: :description
delegate :feature_available?, :builds_enabled?, :wiki_enabled?, delegate :feature_available?, :builds_enabled?, :wiki_enabled?,
...@@ -63,6 +66,12 @@ class Project < ActiveRecord::Base ...@@ -63,6 +66,12 @@ class Project < ActiveRecord::Base
delegate :base_dir, :disk_path, :ensure_storage_path_exists, to: :storage delegate :base_dir, :disk_path, :ensure_storage_path_exists, to: :storage
delegate :scheduled?, :started?, :in_progress?,
:failed?, :finished?,
prefix: :import, to: :import_state, allow_nil: true
delegate :no_import?, to: :import_state, allow_nil: true
default_value_for :archived, false default_value_for :archived, false
default_value_for :visibility_level, gitlab_config_features.visibility_level default_value_for :visibility_level, gitlab_config_features.visibility_level
default_value_for :resolve_outdated_diff_discussions, false default_value_for :resolve_outdated_diff_discussions, false
...@@ -454,8 +463,8 @@ def self.with_feature_available_for_user(feature, user) ...@@ -454,8 +463,8 @@ def self.with_feature_available_for_user(feature, user)
scope :excluding_project, ->(project) { where.not(id: project) } scope :excluding_project, ->(project) { where.not(id: project) }
scope :joins_import_state, -> { joins("LEFT JOIN project_mirror_data import_state ON import_state.project_id = projects.id") } # We require an alias to the project_mirror_data_table in order to use import_state in our queries
scope :import_started, -> { joins_import_state.where("import_state.status = 'started' OR projects.import_status = 'started'") } scope :joins_import_state, -> { joins("INNER JOIN project_mirror_data import_state ON import_state.project_id = projects.id") }
scope :for_group, -> (group) { where(group: group) } scope :for_group, -> (group) { where(group: group) }
class << self class << self
...@@ -631,6 +640,14 @@ def saved? ...@@ -631,6 +640,14 @@ def saved?
id && persisted? id && persisted?
end end
def import_status
import_state&.status || 'none'
end
def human_import_status_name
import_state&.human_status_name || 'none'
end
def add_import_job def add_import_job
job_id = job_id =
if forked? if forked?
...@@ -662,7 +679,7 @@ def reset_cache_and_import_attrs ...@@ -662,7 +679,7 @@ def reset_cache_and_import_attrs
ProjectCacheWorker.perform_async(self.id) ProjectCacheWorker.perform_async(self.id)
end end
update(import_error: nil) import_state.update(last_error: nil)
remove_import_data remove_import_data
end end
...@@ -724,130 +741,6 @@ def external_import? ...@@ -724,130 +741,6 @@ def external_import?
import_url.present? import_url.present?
end end
def imported?
import_finished?
end
def import_in_progress?
import_started? || import_scheduled?
end
def import_state_args
{
status: self[:import_status],
jid: self[:import_jid],
last_error: self[:import_error]
}
end
def ensure_import_state(force: false)
return if !force && (self[:import_status] == 'none' || self[:import_status].nil?)
return unless import_state.nil?
if persisted?
create_import_state(import_state_args)
update_column(:import_status, 'none')
else
build_import_state(import_state_args)
self[:import_status] = 'none'
end
end
def human_import_status_name
ensure_import_state
import_state.human_status_name
end
def import_schedule
ensure_import_state(force: true)
import_state.schedule
end
def force_import_start
ensure_import_state(force: true)
import_state.force_start
end
def import_start
ensure_import_state(force: true)
import_state.start
end
def import_fail
ensure_import_state(force: true)
import_state.fail_op
end
def import_finish
ensure_import_state(force: true)
import_state.finish
end
def import_jid=(new_jid)
ensure_import_state(force: true)
import_state.jid = new_jid
end
def import_jid
ensure_import_state
import_state&.jid
end
def import_error=(new_error)
ensure_import_state(force: true)
import_state.last_error = new_error
end
def import_error
ensure_import_state
import_state&.last_error
end
def import_status=(new_status)
ensure_import_state(force: true)
import_state.status = new_status
end
def import_status
ensure_import_state
import_state&.status || 'none'
end
def no_import?
import_status == 'none'
end
def import_started?
# import? does SQL work so only run it if it looks like there's an import running
import_status == 'started' && import?
end
def import_scheduled?
import_status == 'scheduled'
end
def import_failed?
import_status == 'failed'
end
def import_finished?
import_status == 'finished'
end
def safe_import_url def safe_import_url
Gitlab::UrlSanitizer.new(import_url).masked_url Gitlab::UrlSanitizer.new(import_url).masked_url
end end
...@@ -1646,8 +1539,8 @@ def write_repository_config(gl_full_path: full_path) ...@@ -1646,8 +1539,8 @@ def write_repository_config(gl_full_path: full_path)
def after_import def after_import
repository.after_import repository.after_import
wiki.repository.after_import wiki.repository.after_import
import_finish import_state.finish
remove_import_jid import_state.remove_jid
update_project_counter_caches update_project_counter_caches
after_create_default_branch after_create_default_branch
refresh_markdown_cache! refresh_markdown_cache!
...@@ -1687,32 +1580,11 @@ def after_create_default_branch ...@@ -1687,32 +1580,11 @@ def after_create_default_branch
end end
# rubocop: enable CodeReuse/ServiceClass # rubocop: enable CodeReuse/ServiceClass
def remove_import_jid
return unless import_jid
Gitlab::SidekiqStatus.unset(import_jid)
import_state.update_column(:jid, nil)
end
# Lazy loading of the `pipeline_status` attribute # Lazy loading of the `pipeline_status` attribute
def pipeline_status def pipeline_status
@pipeline_status ||= Gitlab::Cache::Ci::ProjectPipelineStatus.load_for_project(self) @pipeline_status ||= Gitlab::Cache::Ci::ProjectPipelineStatus.load_for_project(self)
end end
def mark_import_as_failed(error_message)
original_errors = errors.dup
sanitized_message = Gitlab::UrlSanitizer.sanitize(error_message)
import_fail
import_state.update_column(:last_error, sanitized_message)
rescue ActiveRecord::ActiveRecordError => e
Rails.logger.error("Error setting import status to failed: #{e.message}. Original error: #{sanitized_message}")
ensure
@errors = original_errors
end
def add_export_job(current_user:, after_export_strategy: nil, params: {}) def add_export_job(current_user:, after_export_strategy: nil, params: {})
job_id = ProjectExportWorker.perform_async(current_user.id, self.id, after_export_strategy, params) job_id = ProjectExportWorker.perform_async(current_user.id, self.id, after_export_strategy, params)
...@@ -1989,17 +1861,6 @@ def reference_counter(wiki: false) ...@@ -1989,17 +1861,6 @@ def reference_counter(wiki: false)
Gitlab::ReferenceCounter.new(gl_repository(is_wiki: wiki)) Gitlab::ReferenceCounter.new(gl_repository(is_wiki: wiki))
end end
# Refreshes the expiration time of the associated import job ID.
#
# This method can be used by asynchronous importers to refresh the status,
# preventing the StuckImportJobsWorker from marking the import as failed.
def refresh_import_jid_expiration
return unless import_jid
Gitlab::SidekiqStatus
.set(import_jid, StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION)
end
def badges def badges
return project_badges unless group return project_badges unless group
......
...@@ -69,4 +69,33 @@ def mark_as_failed(error_message) ...@@ -69,4 +69,33 @@ def mark_as_failed(error_message)
ensure ensure
@errors = original_errors @errors = original_errors
end end
alias_method :no_import?, :none?
def in_progress?
started? || scheduled?
end
def started?
# import? does SQL work so only run it if it looks like there's an import running
status == 'started' && project.import?
end
def remove_jid
return unless jid
Gitlab::SidekiqStatus.unset(jid)
update_column(:jid, nil)
end
# Refreshes the expiration time of the associated import job ID.
#
# This method can be used by asynchronous importers to refresh the status,
# preventing the StuckImportJobsWorker from marking the import as failed.
def refresh_jid_expiration
return unless jid
Gitlab::SidekiqStatus.set(jid, StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION)
end
end end
...@@ -148,7 +148,7 @@ def fail(error:) ...@@ -148,7 +148,7 @@ def fail(error:)
Rails.logger.error(log_message) Rails.logger.error(log_message)
if @project if @project
@project.mark_import_as_failed(message) if @project.persisted? && @project.import? @project.import_state.mark_as_failed(message) if @project.persisted? && @project.import?
end end
@project @project
...@@ -181,7 +181,7 @@ def set_project_name_from_path ...@@ -181,7 +181,7 @@ def set_project_name_from_path
def import_schedule def import_schedule
if @project.errors.empty? if @project.errors.empty?
@project.import_schedule if @project.import? && !@project.bare_repository_import? @project.import_state.schedule if @project.import? && !@project.bare_repository_import?
else else
fail(error: @project.errors.full_messages.join(', ')) fail(error: @project.errors.full_messages.join(', '))
end end
......
...@@ -37,11 +37,12 @@ ...@@ -37,11 +37,12 @@
%td %td
= link_to project.full_path, [project.namespace.becomes(Namespace), project] = link_to project.full_path, [project.namespace.becomes(Namespace), project]
%td.job-status %td.job-status
- if project.import_status == 'finished' - case project.import_status
- when 'finished'
%span %span
%i.fa.fa-check %i.fa.fa-check
= _('done') = _('done')
- elsif project.import_status == 'started' - when 'started'
%i.fa.fa-spinner.fa-spin %i.fa.fa-spinner.fa-spin
= _('started') = _('started')
- else - else
......
...@@ -38,9 +38,10 @@ ...@@ -38,9 +38,10 @@
%td %td
= link_to project.full_path, [project.namespace.becomes(Namespace), project] = link_to project.full_path, [project.namespace.becomes(Namespace), project]
%td.job-status %td.job-status
- if project.import_status == 'finished' - case project.import_status
- when 'finished'
= icon('check', text: 'Done') = icon('check', text: 'Done')
- elsif project.import_status == 'started' - when 'started'
= icon('spin', text: 'started') = icon('spin', text: 'started')
- else - else
= project.human_import_status_name = project.human_import_status_name
......
...@@ -34,11 +34,12 @@ ...@@ -34,11 +34,12 @@
%td %td
= link_to project.full_path, [project.namespace.becomes(Namespace), project] = link_to project.full_path, [project.namespace.becomes(Namespace), project]
%td.job-status %td.job-status
- if project.import_status == 'finished' - case project.import_status
- when 'finished'
%span %span
%i.fa.fa-check %i.fa.fa-check
= _("done") = _("done")
- elsif project.import_status == 'started' - when 'started'
%i.fa.fa-spinner.fa-spin %i.fa.fa-spinner.fa-spin
= _("started") = _("started")
- else - else
......
...@@ -30,11 +30,12 @@ ...@@ -30,11 +30,12 @@
%td %td
= link_to project.full_path, [project.namespace.becomes(Namespace), project] = link_to project.full_path, [project.namespace.becomes(Namespace), project]
%td.job-status %td.job-status
- if project.import_status == 'finished' - case project.import_status
- when 'finished'
%span %span
%i.fa.fa-check %i.fa.fa-check
= _('done') = _('done')
- elsif project.import_status == 'started' - when 'started'
%i.fa.fa-spinner.fa-spin %i.fa.fa-spinner.fa-spin
= _('started') = _('started')
- else - else
......
...@@ -39,11 +39,12 @@ ...@@ -39,11 +39,12 @@
%td %td
= link_to project.full_path, [project.namespace.becomes(Namespace), project] = link_to project.full_path, [project.namespace.becomes(Namespace), project]
%td.job-status %td.job-status
- if project.import_status == 'finished' - case project.import_status
- when 'finished'
%span %span
%i.fa.fa-check %i.fa.fa-check
= _("done") = _("done")
- elsif project.import_status == 'started' - when 'started'
%i.fa.fa-spinner.fa-spin %i.fa.fa-spinner.fa-spin
= _("started") = _("started")
- else - else
......
...@@ -10,7 +10,7 @@ ...@@ -10,7 +10,7 @@
.card-body .card-body
%pre %pre
:preserve :preserve
#{h(@project.import_error)} #{h(@project.import_state.last_error)}
= form_for @project, url: project_import_path(@project), method: :post do |f| = form_for @project, url: project_import_path(@project), method: :post do |f|
= render "shared/import_form", f: f = render "shared/import_form", f: f
......
...@@ -24,7 +24,7 @@ def try_import(client, project) ...@@ -24,7 +24,7 @@ def try_import(client, project)
def find_project(id) def find_project(id)
# If the project has been marked as failed we want to bail out # If the project has been marked as failed we want to bail out
# automatically. # automatically.
Project.import_started.find_by(id: id) Project.joins_import_state.where(import_state: { status: :started }).find_by(id: id)
end end
# rubocop: enable CodeReuse/ActiveRecord # rubocop: enable CodeReuse/ActiveRecord
end end
......
...@@ -18,7 +18,7 @@ module ProjectImportOptions ...@@ -18,7 +18,7 @@ module ProjectImportOptions
"import" "import"
end end
project.mark_import_as_failed("Every #{action} attempt has failed: #{job['error_message']}. Please try again.") project.import_state.mark_as_failed(_("Every %{action} attempt has failed: %{job_error_message}. Please try again.") % { action: action, job_error_message: job['error_message'] })
Sidekiq.logger.warn "Failed #{job['class']} with #{job['args']}: #{job['error_message']}" Sidekiq.logger.warn "Failed #{job['class']} with #{job['args']}: #{job['error_message']}"
end end
end end
......
...@@ -2,11 +2,11 @@ ...@@ -2,11 +2,11 @@
# Used in EE by mirroring # Used in EE by mirroring
module ProjectStartImport module ProjectStartImport
def start(project) def start(import_state)
if project.import_started? && project.import_jid == self.jid if import_state.started? && import_state.jid == self.jid
return true return true
end end
project.import_start import_state.start
end end
end end
...@@ -31,7 +31,7 @@ class AdvanceStageWorker ...@@ -31,7 +31,7 @@ class AdvanceStageWorker
# next_stage - The name of the next stage to start when all jobs have been # next_stage - The name of the next stage to start when all jobs have been
# completed. # completed.
def perform(project_id, waiters, next_stage) def perform(project_id, waiters, next_stage)
return unless (project = find_project(project_id)) return unless import_state = find_import_state(project_id)
new_waiters = wait_for_jobs(waiters) new_waiters = wait_for_jobs(waiters)
...@@ -41,7 +41,7 @@ def perform(project_id, waiters, next_stage) ...@@ -41,7 +41,7 @@ def perform(project_id, waiters, next_stage)
# the pressure on Redis. We _only_ do this once all jobs are done so # the pressure on Redis. We _only_ do this once all jobs are done so
# we don't get stuck forever if one or more jobs failed to notify the # we don't get stuck forever if one or more jobs failed to notify the
# JobWaiter. # JobWaiter.
project.refresh_import_jid_expiration