Read about our upcoming Code of Conduct on this issue

14_pipelines.rb 9.16 KB
Newer Older
1
2
require './spec/support/sidekiq'

3
class Gitlab::Seeder::Pipelines
4
  STAGES = %w[build test security deploy notify]
5
  BUILDS = [
Kamil Trzcinski's avatar
Kamil Trzcinski committed
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
    # build stage
    { name: 'build:linux', stage: 'build', status: :success,
      queued_at: 10.hour.ago, started_at: 9.hour.ago, finished_at: 8.hour.ago },
    { name: 'build:osx', stage: 'build', status: :success,
      queued_at: 10.hour.ago, started_at: 10.hour.ago, finished_at: 9.hour.ago },

    # test stage
    { name: 'rspec:linux 0 3', stage: 'test', status: :success,
      queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago },
    { name: 'rspec:linux 1 3', stage: 'test', status: :success,
      queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago },
    { name: 'rspec:linux 2 3', stage: 'test', status: :success,
      queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago },
    { name: 'rspec:windows 0 3', stage: 'test', status: :success,
      queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago },
    { name: 'rspec:windows 1 3', stage: 'test', status: :success,
      queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago },
    { name: 'rspec:windows 2 3', stage: 'test', status: :success,
      queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago },
    { name: 'rspec:windows 2 3', stage: 'test', status: :success,
      queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago },
    { name: 'rspec:osx', stage: 'test', status_event: :success,
      queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago },
    { name: 'spinach:linux', stage: 'test', status: :success,
      queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago },
    { name: 'spinach:osx', stage: 'test', status: :failed, allow_failure: true,
      queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago },

34
35
36
37
38
39
40
41
42
43
    # security stage
    { name: 'dast', stage: 'security', status: :success,
      queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago },
    { name: 'sast', stage: 'security', status: :success,
      queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago },
    { name: 'dependency_scanning', stage: 'security', status: :success,
      queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago },
    { name: 'container_scanning', stage: 'security', status: :success,
      queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago },

Kamil Trzcinski's avatar
Kamil Trzcinski committed
44
45
46
47
48
49
50
51
52
53
    # deploy stage
    { name: 'staging', stage: 'deploy', environment: 'staging', status_event: :success,
      options: { environment: { action: 'start', on_stop: 'stop staging' } },
      queued_at: 7.hour.ago, started_at: 6.hour.ago, finished_at: 4.hour.ago },
    { name: 'stop staging', stage: 'deploy', environment: 'staging',
      when: 'manual', status: :skipped },
    { name: 'production', stage: 'deploy', environment: 'production',
      when: 'manual', status: :skipped },

    # notify stage
54
    { name: 'slack', stage: 'notify', when: 'manual', status: :success },
55
  ]
Kamil Trzcinski's avatar
Kamil Trzcinski committed
56
57
58
59
  EXTERNAL_JOBS = [
    { name: 'jenkins', stage: 'test', status: :success,
      queued_at: 7.hour.ago, started_at: 6.hour.ago, finished_at: 4.hour.ago },
  ]
Z.J. van de Weg's avatar
Z.J. van de Weg committed
60

Grzegorz Bizon's avatar
Grzegorz Bizon committed
61
62
63
  def initialize(project)
    @project = project
  end
Grzegorz Bizon's avatar
Grzegorz Bizon committed
64

Grzegorz Bizon's avatar
Grzegorz Bizon committed
65
  def seed!
66
    pipelines.each do |pipeline|
67
68
69
70
      BUILDS.each { |opts| build_create!(pipeline, opts) }
      EXTERNAL_JOBS.each { |opts| commit_status_create!(pipeline, opts) }
      pipeline.update_duration
      pipeline.update_status
Grzegorz Bizon's avatar
Grzegorz Bizon committed
71
    end
72
  end
Grzegorz Bizon's avatar
Grzegorz Bizon committed
73

74
75
  private

76
  def pipelines
77
    create_master_pipelines + create_merge_request_pipelines
78
79
  end

80
  def create_master_pipelines
81
    @project.repository.commits('master', limit: 4).map do |commit|
82
83
      create_pipeline!(@project, 'master', commit)
    end
Grzegorz Bizon's avatar
Grzegorz Bizon committed
84
85
86
87
  rescue
    []
  end

88
  def create_merge_request_pipelines
89
    pipelines = @project.merge_requests.first(3).map do |merge_request|
90
91
92
      project = merge_request.source_project
      branch = merge_request.source_branch

93
      merge_request.commits.last(4).map do |commit|
94
95
96
        create_pipeline!(project, branch, commit).tap do |pipeline|
          merge_request.update!(head_pipeline_id: pipeline.id)
        end
97
98
99
100
      end
    end

    pipelines.flatten
101
102
103
104
105
  rescue
    []
  end


106
  def create_pipeline!(project, ref, commit)
107
    project.ci_pipelines.create!(sha: commit.id, ref: ref, source: :push)
108
109
  end

110
  def build_create!(pipeline, opts = {})
111
112
    attributes = job_attributes(pipeline, opts)
      .merge(commands: '$ build command')
113

114
    Ci::Build.create!(attributes).tap do |build|
115
116
117
118
      # We need to set build trace and artifacts after saving a build
      # (id required), that is why we need `#tap` method instead of passing
      # block directly to `Ci::Build#create!`.

119
      setup_artifacts(build)
120
      setup_test_reports(build)
121
122
123
124
125
      if build.ref == build.project.default_branch
        setup_security_reports_file(build)
      else
        setup_security_reports_legacy_archive(build)
      end
126
      setup_build_log(build)
127
128
129
130

      build.project.environments.
        find_or_create_by(name: build.expanded_environment_name)

131
      build.save!
132
133
    end
  end
134

135
  def setup_artifacts(build)
136
    return unless build.stage == "build"
137
138

    artifacts_cache_file(artifacts_archive_path) do |file|
139
      build.job_artifacts.build(project: build.project, file_type: :archive, file_format: :zip, file: file)
140
141
142
    end

    artifacts_cache_file(artifacts_metadata_path) do |file|
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
      build.job_artifacts.build(project: build.project, file_type: :metadata, file_format: :gzip, file: file)
    end
  end

  def setup_test_reports(build)
    return unless build.stage == "test" && build.name == "rspec:osx"

    if build.ref == build.project.default_branch
      artifacts_cache_file(test_reports_pass_path) do |file|
        build.job_artifacts.build(project: build.project, file_type: :junit, file_format: :gzip, file: file)
      end
    else
      artifacts_cache_file(test_reports_failed_path) do |file|
        build.job_artifacts.build(project: build.project, file_type: :junit, file_format: :gzip, file: file)
      end
158
159
160
    end
  end

161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
  def setup_security_reports_file(build)
    return unless build.stage == "security"

    # we have two sources: master and feature-branch
    branch_name = build.ref == build.project.default_branch ?
      'master' : 'feature-branch'

    artifacts_cache_file(security_reports_path(branch_name, build.name)) do |file|
      build.job_artifacts.build(
        project: build.project,
        file_type: build.name,
        file_format: :raw,
        file: file)
    end
  end

  def setup_security_reports_legacy_archive(build)
    return unless build.stage == "security"

    # we have two sources: master and feature-branch
    branch_name = build.ref == build.project.default_branch ?
      'master' : 'feature-branch'

    artifacts_cache_file(security_reports_archive_path(branch_name)) do |file|
      build.job_artifacts.build(
        project: build.project,
        file_type: :archive,
        file_format: :zip,
        file: file)
    end

    # assign dummy metadata
    artifacts_cache_file(artifacts_metadata_path) do |file|
      build.job_artifacts.build(
        project: build.project,
        file_type: :metadata,
        file_format: :gzip,
        file: file)
    end

    build.options = {
      artifacts: {
        paths: [
          Ci::JobArtifact::DEFAULT_FILE_NAMES.fetch(build.name.to_sym)
        ]
      }
    }
  end

210
211
  def setup_build_log(build)
    if %w(running success failed).include?(build.status)
212
      build.trace.set(FFaker::Lorem.paragraphs(6).join("\n\n"))
213
214
    end
  end
Z.J. van de Weg's avatar
Z.J. van de Weg committed
215

216
  def commit_status_create!(pipeline, opts = {})
217
218
    attributes = job_attributes(pipeline, opts)

Z.J. van de Weg's avatar
Z.J. van de Weg committed
219
    GenericCommitStatus.create!(attributes)
220
  end
Z.J. van de Weg's avatar
Z.J. van de Weg committed
221

222
  def job_attributes(pipeline, opts)
223
    { name: 'test build', stage: 'test', stage_idx: stage_index(opts[:stage]),
224
      ref: pipeline.ref, tag: false, user: build_user, project: @project, pipeline: pipeline,
225
226
227
      created_at: Time.now, updated_at: Time.now
    }.merge(opts)
  end
228

Grzegorz Bizon's avatar
Grzegorz Bizon committed
229
230
231
232
233
  def build_user
    @project.team.users.sample
  end

  def build_status
234
    Ci::Build::AVAILABLE_STATUSES.sample
Grzegorz Bizon's avatar
Grzegorz Bizon committed
235
236
  end

237
238
239
240
  def stage_index(stage)
    STAGES.index(stage) || 0
  end

241
  def artifacts_archive_path
242
    Rails.root + 'spec/fixtures/ci_build_artifacts.zip'
Grzegorz Bizon's avatar
Grzegorz Bizon committed
243
244
  end

245
246
247
248
  def artifacts_metadata_path
    Rails.root + 'spec/fixtures/ci_build_artifacts_metadata.gz'
  end

249
250
251
252
253
254
255
256
  def test_reports_pass_path
    Rails.root + 'spec/fixtures/junit/junit_ant.xml.gz'
  end

  def test_reports_failed_path
    Rails.root + 'spec/fixtures/junit/junit.xml.gz'
  end

257
258
259
260
261
262
263
264
265
  def security_reports_archive_path(branch)
    Rails.root.join('spec', 'fixtures', 'security-reports', branch + '.zip')
  end

  def security_reports_path(branch, name)
    file_name = Ci::JobArtifact::DEFAULT_FILE_NAMES.fetch(name.to_sym)
    Rails.root.join('spec', 'fixtures', 'security-reports', branch, file_name)
  end

266
  def artifacts_cache_file(file_path)
267
268
    file = Tempfile.new("artifacts")
    file.close
269

270
271
272
    FileUtils.copy(file_path, file.path)

    yield(UploadedFile.new(file.path, filename: File.basename(file_path)))
Grzegorz Bizon's avatar
Grzegorz Bizon committed
273
274
275
276
  end
end

Gitlab::Seeder.quiet do
277
  Project.all.sample(5).each do |project|
278
    project_builds = Gitlab::Seeder::Pipelines.new(project)
Grzegorz Bizon's avatar
Grzegorz Bizon committed
279
280
    project_builds.seed!
  end
281
end