Read about our upcoming Code of Conduct on this issue

stuck_import_jobs_worker.rb 2.17 KB
Newer Older
1
class StuckImportJobsWorker
2
  include ApplicationWorker
3
4
  include CronjobQueue

5
  IMPORT_JOBS_EXPIRATION = 15.hours.to_i
6
7

  def perform
8
9
10
11
12
13
14
15
16
17
18
    projects_without_jid_count = mark_projects_without_jid_as_failed!
    projects_with_jid_count = mark_projects_with_jid_as_failed!

    Gitlab::Metrics.add_event(:stuck_import_jobs,
                             projects_without_jid_count: projects_without_jid_count,
                             projects_with_jid_count: projects_with_jid_count)
  end

  private

  def mark_projects_without_jid_as_failed!
19
    enqueued_projects_without_jid.each do |project|
20
21
22
23
24
      project.mark_import_as_failed(error_message)
    end.count
  end

  def mark_projects_with_jid_as_failed!
25
26
    # TODO: Rollback this change to use SQL through #pluck
    jids_and_ids = enqueued_projects_with_jid.map { |project| [project.import_jid, project.id] }.to_h
27

28
29
30
    # Find the jobs that aren't currently running or that exceeded the threshold.
    completed_jids = Gitlab::SidekiqStatus.completed_jids(jids_and_ids.keys)
    return unless completed_jids.any?
31

32
    completed_project_ids = jids_and_ids.values_at(*completed_jids)
33

34
35
36
    # We select the projects again, because they may have transitioned from
    # scheduled/started to finished/failed while we were looking up their Sidekiq status.
    completed_projects = enqueued_projects_with_jid.where(id: completed_project_ids)
37

38
    Rails.logger.info("Marked stuck import jobs as failed. JIDs: #{completed_projects.map(&:import_jid).join(', ')}")
39

40
41
42
    completed_projects.each do |project|
      project.mark_import_as_failed(error_message)
    end.count
43
  end
44

45
  def enqueued_projects
46
    Project.joins_import_state.where("(import_state.status = 'scheduled' OR import_state.status = 'started') OR (projects.import_status = 'scheduled' OR projects.import_status = 'started')")
47
48
  end

49
  def enqueued_projects_with_jid
50
    enqueued_projects.where.not("import_state.jid IS NULL AND projects.import_jid IS NULL")
51
  end
52

53
  def enqueued_projects_without_jid
54
    enqueued_projects.where("import_state.jid IS NULL AND projects.import_jid IS NULL")
55
56
57
  end

  def error_message
58
    "Import timed out. Import took longer than #{IMPORT_JOBS_EXPIRATION} seconds"
59
60
  end
end