Read about our upcoming Code of Conduct on this issue

register_job_service_spec.rb 20 KB
Newer Older
Kamil Trzcinski's avatar
Kamil Trzcinski committed
1
require 'spec_helper'
Valery Sizov's avatar
Valery Sizov committed
2
3

module Ci
4
  describe RegisterJobService do
5
6
7
8
9
10
11
    set(:group) { create(:group) }
    set(:project) { create(:project, group: group, shared_runners_enabled: false, group_runners_enabled: false) }
    set(:pipeline) { create(:ci_pipeline, project: project) }
    let!(:shared_runner) { create(:ci_runner, is_shared: true) }
    let!(:specific_runner) { create(:ci_runner, is_shared: false) }
    let!(:group_runner) { create(:ci_runner, groups: [group], runner_type: :group_type) }
    let!(:pending_job) { create(:ci_build, pipeline: pipeline) }
Valery Sizov's avatar
Valery Sizov committed
12
13

    before do
14
      specific_runner.assign_to(project)
Douwe Maan's avatar
Douwe Maan committed
15
16
    end

17
    describe '#execute' do
Valery Sizov's avatar
Valery Sizov committed
18
19
      context 'runner follow tag list' do
        it "picks build with the same tag" do
20
21
          pending_job.update(tag_list: ["linux"])
          specific_runner.update(tag_list: ["linux"])
22
          expect(execute(specific_runner)).to eq(pending_job)
Valery Sizov's avatar
Valery Sizov committed
23
24
25
        end

        it "does not pick build with different tag" do
26
27
          pending_job.update(tag_list: ["linux"])
          specific_runner.update(tag_list: ["win32"])
28
          expect(execute(specific_runner)).to be_falsey
Valery Sizov's avatar
Valery Sizov committed
29
30
31
        end

        it "picks build without tag" do
32
          expect(execute(specific_runner)).to eq(pending_job)
Valery Sizov's avatar
Valery Sizov committed
33
34
35
        end

        it "does not pick build with tag" do
36
          pending_job.update(tag_list: ["linux"])
37
          expect(execute(specific_runner)).to be_falsey
Valery Sizov's avatar
Valery Sizov committed
38
39
40
        end

        it "pick build without tag" do
41
          specific_runner.update(tag_list: ["win32"])
42
          expect(execute(specific_runner)).to eq(pending_job)
Valery Sizov's avatar
Valery Sizov committed
43
        end
Douwe Maan's avatar
Douwe Maan committed
44
45
      end

46
47
48
49
50
51
52
53
54
55
56
      context 'deleted projects' do
        before do
          project.update(pending_delete: true)
        end

        context 'for shared runners' do
          before do
            project.update(shared_runners_enabled: true)
          end

          it 'does not pick a build' do
57
            expect(execute(shared_runner)).to be_nil
58
59
60
61
62
          end
        end

        context 'for specific runner' do
          it 'does not pick a build' do
63
            expect(execute(specific_runner)).to be_nil
64
65
66
67
          end
        end
      end

Valery Sizov's avatar
Valery Sizov committed
68
69
      context 'allow shared runners' do
        before do
70
          project.update(shared_runners_enabled: true)
Valery Sizov's avatar
Valery Sizov committed
71
72
        end

Kamil Trzcinski's avatar
Kamil Trzcinski committed
73
        context 'for multiple builds' do
74
          let!(:project2) { create :project, shared_runners_enabled: true }
Kamil Trzcinski's avatar
Kamil Trzcinski committed
75
          let!(:pipeline2) { create :ci_pipeline, project: project2 }
76
          let!(:project3) { create :project, shared_runners_enabled: true }
Kamil Trzcinski's avatar
Kamil Trzcinski committed
77
          let!(:pipeline3) { create :ci_pipeline, project: project3 }
78
          let!(:build1_project1) { pending_job }
79
80
81
82
83
          let!(:build2_project1) { FactoryBot.create :ci_build, pipeline: pipeline }
          let!(:build3_project1) { FactoryBot.create :ci_build, pipeline: pipeline }
          let!(:build1_project2) { FactoryBot.create :ci_build, pipeline: pipeline2 }
          let!(:build2_project2) { FactoryBot.create :ci_build, pipeline: pipeline2 }
          let!(:build1_project3) { FactoryBot.create :ci_build, pipeline: pipeline3 }
Kamil Trzcinski's avatar
Kamil Trzcinski committed
84
85
86

          it 'prefers projects without builds first' do
            # it gets for one build from each of the projects
87
88
89
            expect(execute(shared_runner)).to eq(build1_project1)
            expect(execute(shared_runner)).to eq(build1_project2)
            expect(execute(shared_runner)).to eq(build1_project3)
Kamil Trzcinski's avatar
Kamil Trzcinski committed
90
91

            # then it gets a second build from each of the projects
92
93
            expect(execute(shared_runner)).to eq(build2_project1)
            expect(execute(shared_runner)).to eq(build2_project2)
Kamil Trzcinski's avatar
Kamil Trzcinski committed
94
95

            # in the end the third build
96
            expect(execute(shared_runner)).to eq(build3_project1)
Kamil Trzcinski's avatar
Kamil Trzcinski committed
97
98
99
100
          end

          it 'equalises number of running builds' do
            # after finishing the first build for project 1, get a second build from the same project
101
            expect(execute(shared_runner)).to eq(build1_project1)
Kamil Trzcinski's avatar
Kamil Trzcinski committed
102
            build1_project1.reload.success
103
            expect(execute(shared_runner)).to eq(build2_project1)
Kamil Trzcinski's avatar
Kamil Trzcinski committed
104

105
            expect(execute(shared_runner)).to eq(build1_project2)
Kamil Trzcinski's avatar
Kamil Trzcinski committed
106
            build1_project2.reload.success
107
108
109
            expect(execute(shared_runner)).to eq(build2_project2)
            expect(execute(shared_runner)).to eq(build1_project3)
            expect(execute(shared_runner)).to eq(build3_project1)
Kamil Trzcinski's avatar
Kamil Trzcinski committed
110
111
112
          end
        end

Valery Sizov's avatar
Valery Sizov committed
113
        context 'shared runner' do
114
          let(:build) { execute(shared_runner) }
Valery Sizov's avatar
Valery Sizov committed
115
116
117
118
119
120
121
122

          it { expect(build).to be_kind_of(Build) }
          it { expect(build).to be_valid }
          it { expect(build).to be_running }
          it { expect(build.runner).to eq(shared_runner) }
        end

        context 'specific runner' do
123
          let(:build) { execute(specific_runner) }
Valery Sizov's avatar
Valery Sizov committed
124
125
126
127
128
129

          it { expect(build).to be_kind_of(Build) }
          it { expect(build).to be_valid }
          it { expect(build).to be_running }
          it { expect(build.runner).to eq(specific_runner) }
        end
Douwe Maan's avatar
Douwe Maan committed
130
131
      end

Valery Sizov's avatar
Valery Sizov committed
132
      context 'disallow shared runners' do
Kamil Trzcinski's avatar
Kamil Trzcinski committed
133
        before do
134
          project.update(shared_runners_enabled: false)
Kamil Trzcinski's avatar
Kamil Trzcinski committed
135
136
        end

Valery Sizov's avatar
Valery Sizov committed
137
        context 'shared runner' do
138
          let(:build) { execute(shared_runner) }
Douwe Maan's avatar
Douwe Maan committed
139

Valery Sizov's avatar
Valery Sizov committed
140
141
          it { expect(build).to be_nil }
        end
Douwe Maan's avatar
Douwe Maan committed
142

Valery Sizov's avatar
Valery Sizov committed
143
        context 'specific runner' do
144
          let(:build) { execute(specific_runner) }
Douwe Maan's avatar
Douwe Maan committed
145

Valery Sizov's avatar
Valery Sizov committed
146
147
148
149
150
          it { expect(build).to be_kind_of(Build) }
          it { expect(build).to be_valid }
          it { expect(build).to be_running }
          it { expect(build.runner).to eq(specific_runner) }
        end
Douwe Maan's avatar
Douwe Maan committed
151
      end
Felipe Artur's avatar
Felipe Artur committed
152
153
154

      context 'disallow when builds are disabled' do
        before do
155
          project.update(shared_runners_enabled: true, group_runners_enabled: true)
Felipe Artur's avatar
Felipe Artur committed
156
157
158
159
          project.project_feature.update_attribute(:builds_access_level, ProjectFeature::DISABLED)
        end

        context 'and uses shared runner' do
160
          let(:build) { execute(shared_runner) }
Felipe Artur's avatar
Felipe Artur committed
161
162
163
164

          it { expect(build).to be_nil }
        end

165
166
167
168
169
170
171
        context 'and uses group runner' do
          let(:build) { execute(group_runner) }

          it { expect(build).to be_nil }
        end

        context 'and uses project runner' do
172
          let(:build) { execute(specific_runner) }
Felipe Artur's avatar
Felipe Artur committed
173
174
175
176

          it { expect(build).to be_nil }
        end
      end
177

178
179
      context 'allow group runners' do
        before do
180
          project.update!(group_runners_enabled: true)
181
182
183
184
185
186
187
        end

        context 'for multiple builds' do
          let!(:project2) { create :project, group_runners_enabled: true, group: group }
          let!(:pipeline2) { create :ci_pipeline, project: project2 }
          let!(:project3) { create :project, group_runners_enabled: true, group: group }
          let!(:pipeline3) { create :ci_pipeline, project: project3 }
Alexis Reigel's avatar
Alexis Reigel committed
188

189
190
191
192
193
194
195
          let!(:build1_project1) { pending_job }
          let!(:build2_project1) { create :ci_build, pipeline: pipeline }
          let!(:build3_project1) { create :ci_build, pipeline: pipeline }
          let!(:build1_project2) { create :ci_build, pipeline: pipeline2 }
          let!(:build2_project2) { create :ci_build, pipeline: pipeline2 }
          let!(:build1_project3) { create :ci_build, pipeline: pipeline3 }

Alexis Reigel's avatar
Alexis Reigel committed
196
197
198
199
200
201
202
203
          # these shouldn't influence the scheduling
          let!(:unrelated_group) { create :group }
          let!(:unrelated_project) { create :project, group_runners_enabled: true, group: unrelated_group }
          let!(:unrelated_pipeline) { create :ci_pipeline, project: unrelated_project }
          let!(:build1_unrelated_project) { create :ci_build, pipeline: unrelated_pipeline }
          let!(:unrelated_group_runner) { create :ci_runner, groups: [unrelated_group] }

          it 'does not consider builds from other group runners' do
204
            expect(described_class.new(group_runner).send(:builds_for_group_runner).count).to eq 6
Alexis Reigel's avatar
Alexis Reigel committed
205
206
            execute(group_runner)

207
            expect(described_class.new(group_runner).send(:builds_for_group_runner).count).to eq 5
Alexis Reigel's avatar
Alexis Reigel committed
208
209
            execute(group_runner)

210
            expect(described_class.new(group_runner).send(:builds_for_group_runner).count).to eq 4
Alexis Reigel's avatar
Alexis Reigel committed
211
212
            execute(group_runner)

213
            expect(described_class.new(group_runner).send(:builds_for_group_runner).count).to eq 3
Alexis Reigel's avatar
Alexis Reigel committed
214
215
            execute(group_runner)

216
            expect(described_class.new(group_runner).send(:builds_for_group_runner).count).to eq 2
Alexis Reigel's avatar
Alexis Reigel committed
217
218
            execute(group_runner)

219
            expect(described_class.new(group_runner).send(:builds_for_group_runner).count).to eq 1
Alexis Reigel's avatar
Alexis Reigel committed
220
221
            execute(group_runner)

222
            expect(described_class.new(group_runner).send(:builds_for_group_runner).count).to eq 0
Alexis Reigel's avatar
Alexis Reigel committed
223
            expect(execute(group_runner)).to be_nil
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
          end
        end

        context 'group runner' do
          let(:build) { execute(group_runner) }

          it { expect(build).to be_kind_of(Build) }
          it { expect(build).to be_valid }
          it { expect(build).to be_running }
          it { expect(build.runner).to eq(group_runner) }
        end
      end

      context 'disallow group runners' do
        before do
239
          project.update!(group_runners_enabled: false)
240
241
242
243
244
245
246
247
248
        end

        context 'group runner' do
          let(:build) { execute(group_runner) }

          it { expect(build).to be_nil }
        end
      end

249
250
      context 'when first build is stalled' do
        before do
251
          pending_job.update(lock_version: 0)
252
253
254
255
256
257
258
259
        end

        subject { described_class.new(specific_runner).execute }

        context 'with multiple builds are in queue' do
          let!(:other_build) { create :ci_build, pipeline: pipeline }

          before do
260
            allow_any_instance_of(Ci::RegisterJobService).to receive(:builds_for_project_runner)
261
              .and_return(Ci::Build.where(id: [pending_job, other_build]))
262
263
264
265
266
267
268
269
270
271
          end

          it "receives second build from the queue" do
            expect(subject).to be_valid
            expect(subject.build).to eq(other_build)
          end
        end

        context 'when single build is in queue' do
          before do
272
            allow_any_instance_of(Ci::RegisterJobService).to receive(:builds_for_project_runner)
273
              .and_return(Ci::Build.where(id: pending_job))
274
275
276
277
278
279
280
281
282
          end

          it "does not receive any valid result" do
            expect(subject).not_to be_valid
          end
        end

        context 'when there is no build in queue' do
          before do
283
            allow_any_instance_of(Ci::RegisterJobService).to receive(:builds_for_project_runner)
284
              .and_return(Ci::Build.none)
285
286
287
288
289
290
291
292
293
          end

          it "does not receive builds but result is valid" do
            expect(subject).to be_valid
            expect(subject.build).to be_nil
          end
        end
      end

294
      context 'when access_level of runner is not_protected' do
Shinya Maeda's avatar
Shinya Maeda committed
295
        let!(:specific_runner) { create(:ci_runner, :specific) }
296

Shinya Maeda's avatar
Shinya Maeda committed
297
        context 'when a job is protected' do
298
          let!(:pending_job) { create(:ci_build, :protected, pipeline: pipeline) }
Shinya Maeda's avatar
Shinya Maeda committed
299

300
301
          it 'picks the job' do
            expect(execute(specific_runner)).to eq(pending_job)
Shinya Maeda's avatar
Shinya Maeda committed
302
303
304
305
          end
        end

        context 'when a job is unprotected' do
306
          let!(:pending_job) { create(:ci_build, pipeline: pipeline) }
Shinya Maeda's avatar
Shinya Maeda committed
307

308
309
          it 'picks the job' do
            expect(execute(specific_runner)).to eq(pending_job)
Shinya Maeda's avatar
Shinya Maeda committed
310
311
312
313
          end
        end

        context 'when protected attribute of a job is nil' do
314
          let!(:pending_job) { create(:ci_build, pipeline: pipeline) }
Shinya Maeda's avatar
Shinya Maeda committed
315
316

          before do
317
            pending_job.update_attribute(:protected, nil)
Shinya Maeda's avatar
Shinya Maeda committed
318
319
          end

320
321
          it 'picks the job' do
            expect(execute(specific_runner)).to eq(pending_job)
Shinya Maeda's avatar
Shinya Maeda committed
322
323
324
325
          end
        end
      end

326
      context 'when access_level of runner is ref_protected' do
Shinya Maeda's avatar
Shinya Maeda committed
327
        let!(:specific_runner) { create(:ci_runner, :ref_protected, :specific) }
Shinya Maeda's avatar
Shinya Maeda committed
328
329

        context 'when a job is protected' do
330
          let!(:pending_job) { create(:ci_build, :protected, pipeline: pipeline) }
Shinya Maeda's avatar
Shinya Maeda committed
331

332
333
          it 'picks the job' do
            expect(execute(specific_runner)).to eq(pending_job)
Shinya Maeda's avatar
Shinya Maeda committed
334
335
336
337
          end
        end

        context 'when a job is unprotected' do
338
          let!(:pending_job) { create(:ci_build, pipeline: pipeline) }
Shinya Maeda's avatar
Shinya Maeda committed
339

340
          it 'does not pick the job' do
Shinya Maeda's avatar
Shinya Maeda committed
341
342
343
            expect(execute(specific_runner)).to be_nil
          end
        end
Shinya Maeda's avatar
Shinya Maeda committed
344
345

        context 'when protected attribute of a job is nil' do
346
          let!(:pending_job) { create(:ci_build, pipeline: pipeline) }
Shinya Maeda's avatar
Shinya Maeda committed
347
348

          before do
349
            pending_job.update_attribute(:protected, nil)
Shinya Maeda's avatar
Shinya Maeda committed
350
351
          end

352
          it 'does not pick the job' do
Shinya Maeda's avatar
Shinya Maeda committed
353
354
355
            expect(execute(specific_runner)).to be_nil
          end
        end
Shinya Maeda's avatar
Shinya Maeda committed
356
357
      end

358
      context 'when "dependencies" keyword is specified' do
359
360
361
362
363
364
        shared_examples 'not pick' do
          it 'does not pick the build and drops the build' do
            expect(subject).to be_nil
            expect(pending_job.reload).to be_failed
            expect(pending_job).to be_missing_dependency_failure
          end
Shinya Maeda's avatar
Shinya Maeda committed
365
366
        end

367
368
        shared_examples 'validation is active' do
          context 'when depended job has not been completed yet' do
369
            let!(:pre_stage_job) { create(:ci_build, :manual, pipeline: pipeline, name: 'test', stage_idx: 0) }
370

371
            it { expect(subject).to eq(pending_job) }
372
373
374
375
376
377
378
379
380
381
382
383
384
385
          end

          context 'when artifacts of depended job has been expired' do
            let!(:pre_stage_job) { create(:ci_build, :success, :expired, pipeline: pipeline, name: 'test', stage_idx: 0) }

            it_behaves_like 'not pick'
          end

          context 'when artifacts of depended job has been erased' do
            let!(:pre_stage_job) { create(:ci_build, :success, pipeline: pipeline, name: 'test', stage_idx: 0, erased_at: 1.minute.ago) }

            before do
              pre_stage_job.erase
            end
386

387
388
            it_behaves_like 'not pick'
          end
389
390

          context 'when job object is staled' do
391
            let!(:pre_stage_job) { create(:ci_build, :success, :expired, pipeline: pipeline, name: 'test', stage_idx: 0) }
392
393
394
395
396
397
398
399
400
401

            before do
              allow_any_instance_of(Ci::Build).to receive(:drop!)
                .and_raise(ActiveRecord::StaleObjectError.new(pending_job, :drop!))
            end

            it 'does not drop nor pick' do
              expect(subject).to be_nil
            end
          end
402
403
        end

404
405
        shared_examples 'validation is not active' do
          context 'when depended job has not been completed yet' do
406
            let!(:pre_stage_job) { create(:ci_build, :manual, pipeline: pipeline, name: 'test', stage_idx: 0) }
407

408
409
410
411
            it { expect(subject).to eq(pending_job) }
          end
          context 'when artifacts of depended job has been expired' do
            let!(:pre_stage_job) { create(:ci_build, :success, :expired, pipeline: pipeline, name: 'test', stage_idx: 0) }
412

413
            it { expect(subject).to eq(pending_job) }
414
          end
415

416
417
          context 'when artifacts of depended job has been erased' do
            let!(:pre_stage_job) { create(:ci_build, :success, pipeline: pipeline, name: 'test', stage_idx: 0, erased_at: 1.minute.ago) }
418

419
420
            before do
              pre_stage_job.erase
421
422
            end

423
424
425
            it { expect(subject).to eq(pending_job) }
          end
        end
426

427
428
429
430
431
432
433
434
435
436
437
438
        before do
          stub_feature_flags(ci_disable_validates_dependencies: false)
        end

        let!(:pre_stage_job) { create(:ci_build, :success, pipeline: pipeline, name: 'test', stage_idx: 0) }
        let!(:pending_job) { create(:ci_build, :pending, pipeline: pipeline, stage_idx: 1, options: { dependencies: ['test'] } ) }

        subject { execute(specific_runner) }

        context 'when validates for dependencies is enabled' do
          before do
            stub_feature_flags(ci_disable_validates_dependencies: false)
439
          end
440
441
442
443
444
445
446
447
448
449

          it_behaves_like 'validation is active'
        end

        context 'when validates for dependencies is disabled' do
          before do
            stub_feature_flags(ci_disable_validates_dependencies: true)
          end

          it_behaves_like 'validation is not active'
450
451
        end
      end
452
    end
453

454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
    describe '#register_success' do
      let!(:current_time) { Time.new(2018, 4, 5, 14, 0, 0) }
      let!(:attempt_counter) { double('Gitlab::Metrics::NullMetric') }
      let!(:job_queue_duration_seconds) { double('Gitlab::Metrics::NullMetric') }

      before do
        allow(Time).to receive(:now).and_return(current_time)

        # Stub defaults for any metrics other than the ones we're testing
        allow(Gitlab::Metrics).to receive(:counter)
                                    .with(any_args)
                                    .and_return(Gitlab::Metrics::NullMetric.instance)
        allow(Gitlab::Metrics).to receive(:histogram)
                                    .with(any_args)
                                    .and_return(Gitlab::Metrics::NullMetric.instance)

        # Stub tested metrics
        allow(Gitlab::Metrics).to receive(:counter)
                                    .with(:job_register_attempts_total, anything)
                                    .and_return(attempt_counter)
        allow(Gitlab::Metrics).to receive(:histogram)
                                    .with(:job_queue_duration_seconds, anything, anything, anything)
                                    .and_return(job_queue_duration_seconds)

        project.update(shared_runners_enabled: true)
        pending_job.update(created_at: current_time - 3600, queued_at: current_time - 1800)
480
      end
481

482
      shared_examples 'attempt counter collector' do
483
484
485
486
487
488
        it 'increments attempt counter' do
          allow(job_queue_duration_seconds).to receive(:observe)
          expect(attempt_counter).to receive(:increment)

          execute(runner)
        end
489
      end
490

491
      shared_examples 'jobs queueing time histogram collector' do
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
        it 'counts job queuing time histogram with expected labels' do
          allow(attempt_counter).to receive(:increment)
          expect(job_queue_duration_seconds).to receive(:observe)
            .with({ shared_runner: expected_shared_runner,
                    jobs_running_for_project: expected_jobs_running_for_project_first_job }, 1800)

          execute(runner)
        end

        context 'when project already has running jobs' do
          let!(:build2) { create( :ci_build, :running, pipeline: pipeline, runner: shared_runner) }
          let!(:build3) { create( :ci_build, :running, pipeline: pipeline, runner: shared_runner) }

          it 'counts job queuing time histogram with expected labels' do
            allow(attempt_counter).to receive(:increment)
            expect(job_queue_duration_seconds).to receive(:observe)
              .with({ shared_runner: expected_shared_runner,
                      jobs_running_for_project: expected_jobs_running_for_project_third_job }, 1800)

            execute(runner)
          end
        end
      end

516
517
518
519
520
      shared_examples 'metrics collector' do
        it_behaves_like 'attempt counter collector'
        it_behaves_like 'jobs queueing time histogram collector'
      end

521
522
523
524
525
526
527
      context 'when shared runner is used' do
        let(:runner) { shared_runner }
        let(:expected_shared_runner) { true }
        let(:expected_jobs_running_for_project_first_job) { 0 }
        let(:expected_jobs_running_for_project_third_job) { 2 }

        it_behaves_like 'metrics collector'
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542

        context 'when pending job with queued_at=nil is used' do
          before do
            pending_job.update(queued_at: nil)
          end

          it_behaves_like 'attempt counter collector'

          it "doesn't count job queuing time histogram" do
            allow(attempt_counter).to receive(:increment)
            expect(job_queue_duration_seconds).not_to receive(:observe)

            execute(runner)
          end
        end
543
544
545
546
547
548
549
550
551
552
553
554
555
556
      end

      context 'when specific runner is used' do
        let(:runner) { specific_runner }
        let(:expected_shared_runner) { false }
        let(:expected_jobs_running_for_project_first_job) { '+Inf' }
        let(:expected_jobs_running_for_project_third_job) { '+Inf' }

        it_behaves_like 'metrics collector'
      end
    end

    def execute(runner)
      described_class.new(runner).execute.build
Douwe Maan's avatar
Douwe Maan committed
557
558
559
    end
  end
end