Read about our upcoming Code of Conduct on this issue

pipeline.rb 12.1 KB
Newer Older
Douwe Maan's avatar
Douwe Maan committed
1
module Ci
2
  class Pipeline < ActiveRecord::Base
Douwe Maan's avatar
Douwe Maan committed
3
    extend Ci::Model
4
    include HasStatus
5
    include Importable
6
    include AfterCommitQueue
Rydkin Maxim's avatar
Rydkin Maxim committed
7
    include Presentable
Kamil Trzcinski's avatar
WIP    
Kamil Trzcinski committed
8

Kamil Trzciński's avatar
Kamil Trzciński committed
9
    belongs_to :project
10
    belongs_to :user
11
    belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline'
12
    belongs_to :pipeline_schedule, class_name: 'Ci::PipelineSchedule'
13

14
    has_many :stages
15
    has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id
16
    has_many :builds, foreign_key: :commit_id
17
    has_many :trigger_requests, dependent: :destroy, foreign_key: :commit_id # rubocop:disable Cop/ActiveRecordDependent
Shinya Maeda's avatar
init    
Shinya Maeda committed
18
    has_many :variables, class_name: 'Ci::PipelineVariable'
Felipe Artur's avatar
Felipe Artur committed
19
20
21

    # Merge requests for which the current pipeline is running against
    # the merge request's latest commit.
22
    has_many :merge_requests, foreign_key: "head_pipeline_id"
Douwe Maan's avatar
Douwe Maan committed
23

24
    has_many :pending_builds, -> { pending }, foreign_key: :commit_id, class_name: 'Ci::Build'
Shinya Maeda's avatar
Shinya Maeda committed
25
    has_many :retryable_builds, -> { latest.failed_or_canceled.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build'
26
    has_many :cancelable_statuses, -> { cancelable }, foreign_key: :commit_id, class_name: 'CommitStatus'
27
28
    has_many :manual_actions, -> { latest.manual_actions.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build'
    has_many :artifacts, -> { latest.with_artifacts_not_expired.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build'
29

30
31
    has_many :auto_canceled_pipelines, class_name: 'Ci::Pipeline', foreign_key: 'auto_canceled_by_id'
    has_many :auto_canceled_jobs, class_name: 'CommitStatus', foreign_key: 'auto_canceled_by_id'
32

Douwe Maan's avatar
Douwe Maan committed
33
34
    delegate :id, to: :project, prefix: true

35
    validates :source, exclusion: { in: %w(unknown), unless: :importing? }, on: :create
Douwe Maan's avatar
Douwe Maan committed
36
37
38
    validates :sha, presence: { unless: :importing? }
    validates :ref, presence: { unless: :importing? }
    validates :status, presence: { unless: :importing? }
39
    validate :valid_commit_sha, unless: :importing?
Douwe Maan's avatar
Douwe Maan committed
40

41
    after_create :keep_around_commits, unless: :importing?
Kamil Trzcinski's avatar
Kamil Trzcinski committed
42

43
44
45
46
47
48
49
50
51
52
    enum source: {
      unknown: nil,
      push: 1,
      web: 2,
      trigger: 3,
      schedule: 4,
      api: 5,
      external: 6
    }

53
    state_machine :status, initial: :created do
54
      event :enqueue do
Kamil Trzcinski's avatar
Kamil Trzcinski committed
55
        transition created: :pending
56
        transition [:success, :failed, :canceled, :skipped] => :running
57
58
59
      end

      event :run do
Kamil Trzcinski's avatar
Kamil Trzcinski committed
60
        transition any - [:running] => :running
61
62
      end

63
      event :skip do
Kamil Trzcinski's avatar
Kamil Trzcinski committed
64
        transition any - [:skipped] => :skipped
65
66
67
      end

      event :drop do
Kamil Trzcinski's avatar
Kamil Trzcinski committed
68
        transition any - [:failed] => :failed
69
70
      end

71
      event :succeed do
Kamil Trzcinski's avatar
Kamil Trzcinski committed
72
        transition any - [:success] => :success
73
74
75
      end

      event :cancel do
Kamil Trzcinski's avatar
Kamil Trzcinski committed
76
        transition any - [:canceled] => :canceled
77
78
      end

79
      event :block do
80
        transition any - [:manual] => :manual
81
82
      end

Kamil Trzcinski's avatar
Kamil Trzcinski committed
83
84
85
86
      # IMPORTANT
      # Do not add any operations to this state_machine
      # Create a separate worker for each new operation

87
      before_transition [:created, :pending] => :running do |pipeline|
88
        pipeline.started_at = Time.now
89
90
      end

91
      before_transition any => [:success, :failed, :canceled] do |pipeline|
92
        pipeline.finished_at = Time.now
93
94
95
        pipeline.update_duration
      end

96
97
98
99
      before_transition any => [:manual] do |pipeline|
        pipeline.update_duration
      end

Lin Jen-Shin's avatar
Lin Jen-Shin committed
100
      before_transition canceled: any - [:canceled] do |pipeline|
Lin Jen-Shin's avatar
Lin Jen-Shin committed
101
102
103
        pipeline.auto_canceled_by = nil
      end

104
      after_transition [:created, :pending] => :running do |pipeline|
105
        pipeline.run_after_commit { PipelineMetricsWorker.perform_async(pipeline.id) }
106
107
108
      end

      after_transition any => [:success] do |pipeline|
109
        pipeline.run_after_commit { PipelineMetricsWorker.perform_async(pipeline.id) }
110
111
      end

112
      after_transition [:created, :pending, :running] => :success do |pipeline|
113
        pipeline.run_after_commit { PipelineSuccessWorker.perform_async(pipeline.id) }
114
      end
115
116

      after_transition do |pipeline, transition|
117
118
119
        next if transition.loopback?

        pipeline.run_after_commit do
120
          PipelineHooksWorker.perform_async(pipeline.id)
121
          ExpirePipelineCacheWorker.perform_async(pipeline.id)
122
        end
123
      end
124

125
      after_transition any => [:success, :failed] do |pipeline|
126
        pipeline.run_after_commit do
127
          PipelineNotificationWorker.perform_async(pipeline.id)
128
        end
129
      end
130
131
    end

132
    # ref can't be HEAD or SHA, can only be branch/tag name
133
    scope :latest, ->(ref = nil) do
Douwe Maan's avatar
Douwe Maan committed
134
135
136
      max_id = unscope(:select)
        .select("max(#{quoted_table_name}.id)")
        .group(:ref, :sha)
137

138
139
140
141
142
      if ref
        where(ref: ref, id: max_id.where(ref: ref))
      else
        where(id: max_id)
      end
143
    end
144
    scope :internal, -> { where(source: internal_sources) }
145

146
147
148
149
    def self.latest_status(ref = nil)
      latest(ref).status
    end

150
    def self.latest_successful_for(ref)
151
      success.latest(ref).order(id: :desc).first
152
153
    end

154
155
156
157
158
159
    def self.latest_successful_for_refs(refs)
      success.latest(refs).order(id: :desc).each_with_object({}) do |pipeline, hash|
        hash[pipeline.ref] ||= pipeline
      end
    end

Douwe Maan's avatar
Douwe Maan committed
160
161
162
163
    def self.truncate_sha(sha)
      sha[0...8]
    end

164
    def self.total_duration
Lin Jen-Shin's avatar
Lin Jen-Shin committed
165
      where.not(duration: nil).sum(:duration)
166
167
    end

168
169
170
171
    def self.internal_sources
      sources.reject { |source| source == "external" }.values
    end

172
173
    def stages_count
      statuses.select(:stage).distinct.count
Kamil Trzcinski's avatar
Kamil Trzcinski committed
174
175
    end

176
    def stages_names
177
178
      statuses.order(:stage_idx).distinct
        .pluck(:stage, :stage_idx).map(&:first)
Kamil Trzcinski's avatar
Kamil Trzcinski committed
179
180
    end

181
    def legacy_stage(name)
182
      stage = Ci::LegacyStage.new(self, name: name)
183
184
185
186
      stage unless stage.statuses_count.zero?
    end

    def legacy_stages
187
188
      # TODO, this needs refactoring, see gitlab-ce#26481.

Douwe Maan's avatar
Douwe Maan committed
189
190
      stages_query = statuses
        .group('stage').select(:stage).order('max(stage_idx)')
191

Kamil Trzcinski's avatar
Kamil Trzcinski committed
192
193
      status_sql = statuses.latest.where('stage=sg.stage').status_sql

194
      warnings_sql = statuses.latest.select('COUNT(*)')
Douwe Maan's avatar
Douwe Maan committed
195
        .where('stage=sg.stage').failed_but_allowed.to_sql
196

Douwe Maan's avatar
Douwe Maan committed
197
198
      stages_with_statuses = CommitStatus.from(stages_query, :sg)
        .pluck('sg.stage', status_sql, "(#{warnings_sql})")
Kamil Trzcinski's avatar
Kamil Trzcinski committed
199
200

      stages_with_statuses.map do |stage|
201
        Ci::LegacyStage.new(self, Hash[%i[name status warnings].zip(stage)])
Kamil Trzcinski's avatar
Kamil Trzcinski committed
202
203
204
      end
    end

Douwe Maan's avatar
Douwe Maan committed
205
    def valid_commit_sha
206
      if self.sha == Gitlab::Git::BLANK_SHA
Douwe Maan's avatar
Douwe Maan committed
207
208
209
210
211
        self.errors.add(:sha, " cant be 00000000 (branch removal)")
      end
    end

    def git_author_name
212
      commit.try(:author_name)
Douwe Maan's avatar
Douwe Maan committed
213
214
215
    end

    def git_author_email
216
      commit.try(:author_email)
Douwe Maan's avatar
Douwe Maan committed
217
218
219
    end

    def git_commit_message
220
      commit.try(:message)
Douwe Maan's avatar
Douwe Maan committed
221
222
    end

223
224
225
226
    def git_commit_title
      commit.try(:title)
    end

Douwe Maan's avatar
Douwe Maan committed
227
    def short_sha
228
      Ci::Pipeline.truncate_sha(sha)
Douwe Maan's avatar
Douwe Maan committed
229
230
    end

231
    def commit
232
      @commit ||= project.commit(sha)
Douwe Maan's avatar
Douwe Maan committed
233
234
235
236
    rescue
      nil
    end

Kamil Trzcinski's avatar
Kamil Trzcinski committed
237
238
239
240
    def branch?
      !tag?
    end

241
    def stuck?
242
      pending_builds.any?(&:stuck?)
243
244
    end

Kamil Trzcinski's avatar
Kamil Trzcinski committed
245
    def retryable?
246
      retryable_builds.any?
Kamil Trzcinski's avatar
Kamil Trzcinski committed
247
248
    end

249
    def cancelable?
250
      cancelable_statuses.any?
251
252
    end

Lin Jen-Shin's avatar
Lin Jen-Shin committed
253
254
255
256
    def auto_canceled?
      canceled? && auto_canceled_by_id?
    end

Kamil Trzcinski's avatar
Kamil Trzcinski committed
257
    def cancel_running
258
      Gitlab::OptimisticLocking.retry_lock(cancelable_statuses) do |cancelable|
Lin Jen-Shin's avatar
Lin Jen-Shin committed
259
260
261
        cancelable.find_each do |job|
          yield(job) if block_given?
          job.cancel
262
        end
Lin Jen-Shin's avatar
Lin Jen-Shin committed
263
      end
Kamil Trzcinski's avatar
Kamil Trzcinski committed
264
265
    end

266
267
268
269
270
    def auto_cancel_running(pipeline)
      update(auto_canceled_by: pipeline)

      cancel_running do |job|
        job.auto_canceled_by = pipeline
271
      end
Kamil Trzcinski's avatar
Kamil Trzcinski committed
272
273
    end

274
    def retry_failed(current_user)
Douwe Maan's avatar
Douwe Maan committed
275
276
      Ci::RetryPipelineService.new(project, current_user)
        .execute(self)
Kamil Trzcinski's avatar
Kamil Trzcinski committed
277
278
    end

279
    def mark_as_processable_after_stage(stage_idx)
280
      builds.skipped.after_stage(stage_idx).find_each(&:process)
281
282
    end

Kamil Trzcinski's avatar
Kamil Trzcinski committed
283
284
285
286
287
288
289
    def latest?
      return false unless ref
      commit = project.commit(ref)
      return false unless commit
      commit.sha == sha
    end

Kamil Trzcinski's avatar
Kamil Trzcinski committed
290
291
    def retried
      @retried ||= (statuses.order(id: :desc) - statuses.latest)
Douwe Maan's avatar
Douwe Maan committed
292
293
294
    end

    def coverage
295
      coverage_array = statuses.latest.map(&:coverage).compact
296
297
      if coverage_array.size >= 1
        '%.2f' % (coverage_array.reduce(:+) / coverage_array.size)
Douwe Maan's avatar
Douwe Maan committed
298
299
300
      end
    end

301
    def stage_seeds
302
303
      return [] unless config_processor

304
      @stage_seeds ||= config_processor.stage_seeds(self)
305
306
    end

307
308
    def has_stage_seeds?
      stage_seeds.any?
309
310
    end

Connor Shea's avatar
Connor Shea committed
311
    def has_warnings?
312
      builds.latest.failed_but_allowed.any?
313
314
    end

Douwe Maan's avatar
Douwe Maan committed
315
    def config_processor
316
      return unless ci_yaml_file
317
318
319
      return @config_processor if defined?(@config_processor)

      @config_processor ||= begin
320
        Ci::GitlabCiYamlProcessor.new(ci_yaml_file, project.full_path)
321
      rescue Ci::GitlabCiYamlProcessor::ValidationError, Psych::SyntaxError => e
322
        self.yaml_errors = e.message
323
324
        nil
      rescue
325
        self.yaml_errors = 'Undefined error'
326
327
        nil
      end
Douwe Maan's avatar
Douwe Maan committed
328
329
    end

330
    def ci_yaml_file_path
331
      if project.ci_config_path.blank?
332
333
        '.gitlab-ci.yml'
      else
334
        project.ci_config_path
335
336
337
      end
    end

Kamil Trzcinski's avatar
Kamil Trzcinski committed
338
    def ci_yaml_file
339
340
      return @ci_yaml_file if defined?(@ci_yaml_file)

341
342
343
344
      @ci_yaml_file = (ci_yaml_from_repo || implied_ci_yaml_file).tap do |config|
        unless config
          self.yaml_errors = "Failed to load CI/CD config file for #{sha}"
        end
345
      end
Kamil Trzcinski's avatar
Kamil Trzcinski committed
346
347
    end

348
349
350
351
    def has_yaml_errors?
      yaml_errors.present?
    end

Kamil Trzcinski's avatar
Kamil Trzcinski committed
352
353
354
355
    def environments
      builds.where.not(environment: nil).success.pluck(:environment).uniq
    end

James Lopez's avatar
James Lopez committed
356
357
358
359
360
361
362
363
364
365
366
367
368
    # Manually set the notes for a Ci::Pipeline
    # There is no ActiveRecord relation between Ci::Pipeline and notes
    # as they are related to a commit sha. This method helps importing
    # them using the +Gitlab::ImportExport::RelationFactory+ class.
    def notes=(notes)
      notes.each do |note|
        note[:id] = nil
        note[:commit_id] = sha
        note[:noteable_id] = self['id']
        note.save!
      end
    end

369
370
371
372
    def notes
      Note.for_commit_id(sha)
    end

373
374
375
    def process!
      Ci::ProcessPipelineService.new(project, user).execute(self)
    end
376

377
    def update_status
378
      Gitlab::OptimisticLocking.retry_lock(self) do
Kamil Trzcinski's avatar
Kamil Trzcinski committed
379
        case latest_builds_status
380
381
382
383
384
385
        when 'pending' then enqueue
        when 'running' then run
        when 'success' then succeed
        when 'failed' then drop
        when 'canceled' then cancel
        when 'skipped' then skip
386
        when 'manual' then block
387
        end
388
      end
389
390
    end

391
392
    def predefined_variables
      [
Lin Jen-Shin's avatar
Lin Jen-Shin committed
393
        { key: 'CI_PIPELINE_ID', value: id.to_s, public: true },
394
395
        { key: 'CI_CONFIG_PATH', value: ci_yaml_file_path, public: true },
        { key: 'CI_PIPELINE_SOURCE', value: source.to_s, public: true }
396
397
398
      ]
    end

399
400
401
402
403
404
405
    def queued_duration
      return unless started_at

      seconds = (started_at - created_at).to_i
      seconds unless seconds.zero?
    end

406
    def update_duration
407
408
      return unless started_at

409
      self.duration = Gitlab::Ci::PipelineDuration.from_pipeline(self)
410
411
412
    end

    def execute_hooks
Kamil Trzcinski's avatar
Kamil Trzcinski committed
413
414
415
      data = pipeline_data
      project.execute_hooks(data, :pipeline_hooks)
      project.execute_services(data, :pipeline_hooks)
416
417
    end

418
419
    # All the merge requests for which the current pipeline runs/ran against
    def all_merge_requests
420
      @all_merge_requests ||= project.merge_requests.where(source_branch: ref)
421
422
    end

423
    def detailed_status(current_user)
Douwe Maan's avatar
Douwe Maan committed
424
425
426
      Gitlab::Ci::Status::Pipeline::Factory
        .new(self, current_user)
        .fabricate!
427
428
    end

429
430
    private

431
432
433
434
435
436
437
438
439
440
441
442
    def implied_ci_yaml_file
      if project.auto_devops_enabled?
        Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps').content
      end
    end

    def ci_yaml_from_repo
      project.repository.gitlab_ci_yml_for(sha, ci_yaml_file_path)
    rescue GRPC::NotFound, Rugged::ReferenceError, GRPC::Internal
      nil
    end

443
    def pipeline_data
444
      Gitlab::DataBuilder::Pipeline.build(self)
Kamil Trzcinski's avatar
Kamil Trzcinski committed
445
    end
446

447
    def latest_builds_status
448
449
450
      return 'failed' unless yaml_errors.blank?

      statuses.latest.status || 'skipped'
Kamil Trzcinski's avatar
Kamil Trzcinski committed
451
    end
452
453

    def keep_around_commits
454
      return unless project
455

456
457
458
      project.repository.keep_around(self.sha)
      project.repository.keep_around(self.before_sha)
    end
Douwe Maan's avatar
Douwe Maan committed
459
460
  end
end