Commit 532e48df authored by GitLab Bot's avatar GitLab Bot
Browse files

Add latest changes from gitlab-org/gitlab@master

parent 485d2ef03bba
......@@ -20,7 +20,7 @@ code_quality:
variables:
DOCKER_DRIVER: overlay2
DOCKER_TLS_CERTDIR: ""
CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/security-products/codequality:0.85.6"
CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/security-products/codequality:0.85.9"
script:
- |
if ! docker info &>/dev/null; then
......
......@@ -35,6 +35,7 @@ function renderMermaids($els) {
// mermaidAPI options
theme: 'neutral',
flowchart: {
useMaxWidth: true,
htmlLabels: false,
},
securityLevel: 'strict',
......
# frozen_string_literal: true
module Projects
class ExportJobFinder
InvalidExportJobStatusError = Class.new(StandardError)
attr_reader :project, :params
def initialize(project, params = {})
@project = project
@params = params
end
def execute
export_jobs = project.export_jobs
export_jobs = by_status(export_jobs)
export_jobs
end
private
def by_status(export_jobs)
return export_jobs unless params[:status]
raise InvalidExportJobStatusError, 'Invalid export job status' unless ProjectExportJob.state_machines[:status].states.map(&:name).include?(params[:status])
export_jobs.with_status(params[:status])
end
end
end
......@@ -10,6 +10,7 @@ class Bridge < Ci::Processable
include HasRef
InvalidBridgeTypeError = Class.new(StandardError)
InvalidTransitionError = Class.new(StandardError)
belongs_to :project
belongs_to :trigger_request
......
......@@ -186,6 +186,7 @@ class Project < ApplicationRecord
has_one :import_state, autosave: true, class_name: 'ProjectImportState', inverse_of: :project
has_one :import_export_upload, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
has_many :export_jobs, class_name: 'ProjectExportJob'
has_one :project_repository, inverse_of: :project
has_one :incident_management_setting, inverse_of: :project, class_name: 'IncidentManagement::ProjectIncidentManagementSetting'
has_one :error_tracking_setting, inverse_of: :project, class_name: 'ErrorTracking::ProjectErrorTrackingSetting'
......@@ -1850,10 +1851,12 @@ def export_path
end
def export_status
if export_in_progress?
if regeneration_in_progress?
:regeneration_in_progress
elsif export_enqueued?
:queued
elsif export_in_progress?
:started
elsif after_export_in_progress?
:after_export_action
elsif export_file_exists?
:finished
else
......@@ -1862,11 +1865,19 @@ def export_status
end
def export_in_progress?
import_export_shared.active_export_count > 0
strong_memoize(:export_in_progress) do
::Projects::ExportJobFinder.new(self, { status: :started }).execute.present?
end
end
def export_enqueued?
strong_memoize(:export_enqueued) do
::Projects::ExportJobFinder.new(self, { status: :queued }).execute.present?
end
end
def after_export_in_progress?
import_export_shared.after_export_in_progress?
def regeneration_in_progress?
(export_enqueued? || export_in_progress?) && export_file_exists?
end
def remove_exports
......
# frozen_string_literal: true
class ProjectExportJob < ApplicationRecord
belongs_to :project
validates :project, :jid, :status, presence: true
state_machine :status, initial: :queued do
event :start do
transition [:queued] => :started
end
event :finish do
transition [:started] => :finished
end
event :fail_op do
transition [:queued, :started] => :failed
end
state :queued, value: 0
state :started, value: 1
state :finished, value: 2
state :failed, value: 3
end
end
......@@ -52,6 +52,11 @@ def update_bridge_status!(bridge, pipeline)
subject.drop!(:downstream_pipeline_creation_failed)
end
end
rescue StateMachines::InvalidTransition => e
Gitlab::ErrorTracking.track_exception(
Ci::Bridge::InvalidTransitionError.new(e.message),
bridge_id: bridge.id,
downstream_pipeline_id: pipeline.id)
end
def ensure_preconditions!(target_ref)
......
......@@ -5,7 +5,14 @@ class PipelineBridgeStatusService < ::BaseService
def execute(pipeline)
return unless pipeline.bridge_triggered?
pipeline.source_bridge.inherit_status_from_downstream!(pipeline)
begin
pipeline.source_bridge.inherit_status_from_downstream!(pipeline)
rescue StateMachines::InvalidTransition => e
Gitlab::ErrorTracking.track_exception(
Ci::Bridge::InvalidTransitionError.new(e.message),
bridge_id: pipeline.source_bridge.id,
downstream_pipeline_id: pipeline.id)
end
end
end
end
......
......@@ -234,6 +234,13 @@
:resource_boundary: :cpu
:weight: 1
:idempotent:
- :name: cronjob:stuck_export_jobs
:feature_category: :importers
:has_external_dependencies:
:urgency: :default
:resource_boundary: :cpu
:weight: 1
:idempotent:
- :name: cronjob:stuck_import_jobs
:feature_category: :importers
:has_external_dependencies:
......
# frozen_string_literal: true
module ProjectExportOptions
extend ActiveSupport::Concern
EXPORT_RETRY_COUNT = 3
included do
sidekiq_options retry: EXPORT_RETRY_COUNT, status_expiration: StuckExportJobsWorker::EXPORT_JOBS_EXPIRATION
# We mark the project export as failed once we have exhausted all retries
sidekiq_retries_exhausted do |job|
project = Project.find(job['args'][1])
# rubocop: disable CodeReuse/ActiveRecord
job = project.export_jobs.find_by(jid: job["jid"])
# rubocop: enable CodeReuse/ActiveRecord
if job&.fail_op
Sidekiq.logger.info "Job #{job['jid']} for project #{project.id} has been set to failed state"
else
Sidekiq.logger.error "Failed to set Job #{job['jid']} for project #{project.id} to failed state"
end
end
end
end
......@@ -3,17 +3,24 @@
class ProjectExportWorker # rubocop:disable Scalability/IdempotentWorker
include ApplicationWorker
include ExceptionBacktrace
include ProjectExportOptions
sidekiq_options retry: 3
feature_category :importers
worker_resource_boundary :memory
def perform(current_user_id, project_id, after_export_strategy = {}, params = {})
current_user = User.find(current_user_id)
project = Project.find(project_id)
export_job = project.export_jobs.safe_find_or_create_by(jid: self.jid)
after_export = build!(after_export_strategy)
export_job&.start
::Projects::ImportExport::ExportService.new(project, current_user, params).execute(after_export)
export_job&.finish
rescue ActiveRecord::RecordNotFound, Gitlab::ImportExport::AfterExportStrategyBuilder::StrategyNotFoundError => e
logger.error("Failed to export project #{project_id}: #{e.message}")
end
private
......
# frozen_string_literal: true
# rubocop:disable Scalability/IdempotentWorker
class StuckExportJobsWorker
include ApplicationWorker
# rubocop:disable Scalability/CronWorkerContext
# This worker updates export states inline and does not schedule
# other jobs.
include CronjobQueue
# rubocop:enable Scalability/CronWorkerContext
feature_category :importers
worker_resource_boundary :cpu
EXPORT_JOBS_EXPIRATION = 6.hours.to_i
def perform
failed_jobs_count = mark_stuck_jobs_as_failed!
Gitlab::Metrics.add_event(:stuck_export_jobs,
failed_jobs_count: failed_jobs_count)
end
private
# rubocop: disable CodeReuse/ActiveRecord
def mark_stuck_jobs_as_failed!
jids_and_ids = enqueued_exports.pluck(:jid, :id).to_h
completed_jids = Gitlab::SidekiqStatus.completed_jids(jids_and_ids.keys)
return unless completed_jids.any?
completed_ids = jids_and_ids.values_at(*completed_jids)
# We select the export states again, because they may have transitioned from
# started to finished while we were looking up their Sidekiq status.
completed_jobs = enqueued_exports.where(id: completed_ids)
Sidekiq.logger.info(
message: 'Marked stuck export jobs as failed',
job_ids: completed_jobs.map(&:jid)
)
completed_jobs.each do |job|
job.fail_op
end.count
end
# rubocop: enable CodeReuse/ActiveRecord
def enqueued_exports
ProjectExportJob.with_status([:started, :queued])
end
end
# rubocop:enable Scalability/IdempotentWorker
---
title: Fix quick actions executing in multiline inline code when placed on its own line
merge_request: 24933
author: Pavlo Dudchenko
type: fixed
---
title: Update GitLab's codeclimate to 0.85.9
merge_request: 26712
author: Eddie Stubbington
type: other
---
title: Fix logic to determine project export state and add regeneration_in_progress state
merge_request: 23664
author:
type: fixed
---
title: Fix Mermaid flowchart width
merge_request: 26848
author: julien MILLAU
type: fixed
......@@ -453,6 +453,9 @@
Settings.cron_jobs['stuck_import_jobs_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['stuck_import_jobs_worker']['cron'] ||= '15 * * * *'
Settings.cron_jobs['stuck_import_jobs_worker']['job_class'] = 'StuckImportJobsWorker'
Settings.cron_jobs['stuck_export_jobs_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['stuck_export_jobs_worker']['cron'] ||= '30 * * * *'
Settings.cron_jobs['stuck_export_jobs_worker']['job_class'] = 'StuckExportJobsWorker'
Settings.cron_jobs['gitlab_usage_ping_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['gitlab_usage_ping_worker']['cron'] ||= nil # This is dynamically loaded in the sidekiq initializer
Settings.cron_jobs['gitlab_usage_ping_worker']['job_class'] = 'GitlabUsagePingWorker'
......
# frozen_string_literal: true
class CreateProjectExportJobs < ActiveRecord::Migration[6.0]
DOWNTIME = false
def change
create_table :project_export_jobs do |t|
t.references :project, index: false, null: false, foreign_key: { on_delete: :cascade }
t.timestamps_with_timezone null: false
t.integer :status, limit: 2, null: false, default: 0
t.string :jid, limit: 100, null: false, unique: true
t.index [:project_id, :jid]
t.index [:jid], unique: true
t.index [:status]
t.index [:project_id, :status]
end
end
end
......@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 2020_03_10_135823) do
ActiveRecord::Schema.define(version: 2020_03_11_165635) do
# These are extensions that must be enabled in order to support this database
enable_extension "pg_trgm"
......@@ -3242,6 +3242,18 @@
t.string "organization_name"
end
create_table "project_export_jobs", force: :cascade do |t|
t.bigint "project_id", null: false
t.datetime_with_timezone "created_at", null: false
t.datetime_with_timezone "updated_at", null: false
t.integer "status", limit: 2, default: 0, null: false
t.string "jid", limit: 100, null: false
t.index ["jid"], name: "index_project_export_jobs_on_jid", unique: true
t.index ["project_id", "jid"], name: "index_project_export_jobs_on_project_id_and_jid"
t.index ["project_id", "status"], name: "index_project_export_jobs_on_project_id_and_status"
t.index ["status"], name: "index_project_export_jobs_on_status"
end
create_table "project_feature_usages", primary_key: "project_id", id: :integer, default: nil, force: :cascade do |t|
t.datetime "jira_dvcs_cloud_last_sync_at"
t.datetime "jira_dvcs_server_last_sync_at"
......@@ -5017,6 +5029,7 @@
add_foreign_key "project_deploy_tokens", "deploy_tokens", on_delete: :cascade
add_foreign_key "project_deploy_tokens", "projects", on_delete: :cascade
add_foreign_key "project_error_tracking_settings", "projects", on_delete: :cascade
add_foreign_key "project_export_jobs", "projects", on_delete: :cascade
add_foreign_key "project_feature_usages", "projects", on_delete: :cascade
add_foreign_key "project_features", "projects", name: "fk_18513d9b92", on_delete: :cascade
add_foreign_key "project_group_links", "projects", name: "fk_daa8cee94c", on_delete: :cascade
......
......@@ -122,12 +122,12 @@ our AsciiDoc snippets, wikis and repos using delimited blocks:
- **Markdown**
~~~markdown
````markdown
```plantuml
Bob -> Alice : hello
Alice -> Bob : hi
```
~~~
````
- **AsciiDoc**
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment