Read about our upcoming Code of Conduct on this issue

Commit 9b928b85 authored by Stan Hu's avatar Stan Hu
Browse files

Reduce complexity of CI files


Signed-off-by: default avatarRémy Coutable <remy@rymai.me>
parent caaacb76488f
image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.6.3-golang-1.11-git-2.22-chrome-73.0-node-12.x-yarn-1.16-postgresql-9.6-graphicsmagick-1.3.33"
stages:
- build
- prepare
- quick-test
- test
- review
- qa
- post-test
- pages
variables:
RAILS_ENV: "test"
NODE_ENV: "test"
......@@ -11,25 +21,9 @@ variables:
FLAKY_RSPEC_SUITE_REPORT_PATH: rspec_flaky/report-suite.json
BUILD_ASSETS_IMAGE: "false"
before_script:
- date
- source scripts/utils.sh
- source scripts/prepare_build.sh
- date
after_script:
- date
stages:
- build
- prepare
- merge
- test
- review
- qa
- post-test
- pages
include:
- local: .gitlab/ci/global.gitlab-ci.yml
- local: .gitlab/ci/cng.gitlab-ci.yml
......
cloud-native-image:
image: ruby:2.6-alpine
before_script: []
dependencies: []
stage: post-test
allow_failure: true
variables:
GIT_DEPTH: "1"
cache: {}
when: manual
script:
- install_gitlab_gem
- CNG_PROJECT_PATH="gitlab-org/build/CNG" BUILD_TRIGGER_TOKEN=$CI_JOB_TOKEN ./scripts/trigger-build cng
only:
- tags@gitlab-org/gitlab-ce
- tags@gitlab-org/gitlab-ee
refs:
- tags@gitlab-org/gitlab-ce
- tags@gitlab-org/gitlab-ee
.review-docs: &review-docs
extends: .single-script-job-dedicated-runner
.review-docs:
extends:
- .default-tags
- .default-retry
image: ruby:2.6-alpine
stage: review
dependencies: []
variables:
SCRIPT_NAME: trigger-build-docs
GIT_STRATEGY: none
environment:
name: review-docs/$CI_COMMIT_REF_SLUG
# DOCS_REVIEW_APPS_DOMAIN and DOCS_GITLAB_REPO_SUFFIX are CI variables
# Discussion: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/14236/diffs#note_40140693
url: http://$CI_ENVIRONMENT_SLUG.$DOCS_REVIEW_APPS_DOMAIN/$DOCS_GITLAB_REPO_SUFFIX
on_stop: review-docs-cleanup
before_script:
# We don't clone the repo by using GIT_STRATEGY: none and only download the
# single script we need here so it's much faster than cloning.
- apk add --update openssl
- wget $CI_PROJECT_URL/raw/$CI_COMMIT_SHA/scripts/trigger-build-docs
- chmod 755 trigger-build-docs
# Trigger a manual docs build in gitlab-docs only on non docs-only branches.
# Useful to preview the docs changes live.
review-docs-deploy-manual:
extends:
- .review-docs
- .no-docs-and-no-qa
stage: review
- .except-docs-qa
script:
- gem install gitlab --no-document
- ./$SCRIPT_NAME deploy
- ./trigger-build-docs deploy
when: manual
only:
- branches@gitlab-org/gitlab-ce
......@@ -27,39 +37,40 @@ review-docs-deploy-manual:
# Always trigger a docs build in gitlab-docs only on docs-only branches.
# Useful to preview the docs changes live.
review-docs-deploy:
<<: *review-docs
stage: review
extends:
- .review-docs
- .except-qa
script:
- gem install gitlab --no-document
- ./$SCRIPT_NAME deploy
- ./trigger-build-docs deploy
only:
- /(^docs[\/-].+|.+-docs$)/@gitlab-org/gitlab-ce
- /(^docs[\/-].+|.+-docs$)/@gitlab-org/gitlab-ee
except:
- /(^qa[\/-].*|.*-qa$)/
# Cleanup remote environment of gitlab-docs
review-docs-cleanup:
<<: *review-docs
stage: review
extends:
- .review-docs
- .except-qa
environment:
name: review-docs/$CI_COMMIT_REF_SLUG
action: stop
script:
- gem install gitlab --no-document
- ./$SCRIPT_NAME cleanup
- ./trigger-build-docs cleanup
when: manual
only:
- branches@gitlab-org/gitlab-ce
- branches@gitlab-org/gitlab-ee
docs lint:
extends: .dedicated-runner
extends:
- .default-tags
- .default-retry
- .except-qa
image: "registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-docs-lint"
stage: test
cache: {}
dependencies: []
before_script: []
script:
- scripts/lint-doc.sh
- mv doc/ /tmp/gitlab-docs/content/$DOCS_GITLAB_REPO_SUFFIX
......@@ -72,5 +83,3 @@ docs lint:
- bundle exec nanoc check internal_links
# Check the internal anchor links
- bundle exec nanoc check internal_anchors
except:
- /(^qa[\/-].*|.*-qa$)/
.assets-compile-cache: &assets-compile-cache
.assets-compile-cache:
cache:
key: "assets-compile:vendor_ruby:.yarn-cache:tmp_cache_assets_sprockets:v6"
paths:
- vendor/ruby/
- .yarn-cache/
- tmp/cache/assets/sprockets
.use-pg: &use-pg
services:
- name: postgres:9.6.14
command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
- name: redis:alpine
.gitlab:assets:compile-metadata:
<<: *assets-compile-cache
extends: .dedicated-no-docs-pull-cache-job
extends:
- .default-tags
- .default-retry
- .assets-compile-cache
- .default-before_script
- .except-docs
image: dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.6.3-git-2.22-chrome-73.0-node-12.x-yarn-1.16-graphicsmagick-1.3.33-docker-18.06.1
dependencies:
- setup-test-env
dependencies: ["setup-test-env"]
services:
- docker:19.03.0-dind
variables:
......@@ -30,6 +26,14 @@
NODE_OPTIONS: --max_old_space_size=3584
DOCKER_DRIVER: overlay2
DOCKER_HOST: tcp://docker:2375
cache:
key: "assets-compile:production:vendor_ruby:.yarn-cache:tmp_cache_assets_sprockets:v6"
artifacts:
name: webpack-report
expire_in: 31d
paths:
- webpack-report/
- public/assets/
script:
- node --version
- retry yarn install --frozen-lockfile --production --cache-folder .yarn-cache --prefer-offline
......@@ -42,43 +46,41 @@
- install_api_client_dependencies_with_apt
- play_job "review-build-cng" || true # this job might not exist so ignore the failure if it cannot be played
- play_job "schedule:review-build-cng" || true # this job might not exist so ignore the failure if it cannot be played
artifacts:
name: webpack-report
expire_in: 31d
paths:
- webpack-report/
- public/assets/
only:
- /.+/@gitlab-org/gitlab-ce
- /.+/@gitlab-org/gitlab-ee
- /.+/@gitlab/gitlabhq
- /.+/@gitlab/gitlab-ee
tags:
- docker
- gitlab-org
- docker
gitlab:assets:compile:
extends: .gitlab:assets:compile-metadata
only:
refs:
- master@gitlab-org/gitlab-ce
- master@gitlab-org/gitlab-ee
cache:
policy: pull-push
only:
- master@gitlab-org/gitlab-ce
- master@gitlab-org/gitlab-ee
gitlab:assets:compile pull-cache:
extends: .gitlab:assets:compile-metadata
cache:
policy: pull
except:
refs:
- master@gitlab-org/gitlab-ce
- master@gitlab-org/gitlab-ee
- /(^docs[\/-].+|.+-docs$)/
cache:
policy: pull
.compile-assets-metadata:
extends: .dedicated-runner
<<: *use-pg
<<: *assets-compile-cache
extends:
- .default-tags
- .default-retry
- .assets-compile-cache
- .default-before_script
- .use-pg
stage: prepare
script:
- node --version
......@@ -89,6 +91,8 @@ gitlab:assets:compile pull-cache:
variables:
# we override the max_old_space_size to prevent OOM errors
NODE_OPTIONS: --max_old_space_size=3584
cache:
key: "assets-compile:test:vendor_ruby:.yarn-cache:tmp_cache_assets_sprockets:v6"
artifacts:
expire_in: 7d
paths:
......@@ -96,30 +100,34 @@ gitlab:assets:compile pull-cache:
- public/assets
compile-assets:
extends: .compile-assets-metadata
extends:
- .compile-assets-metadata
only:
refs:
- master@gitlab-org/gitlab-ce
- master@gitlab-org/gitlab-ee
cache:
policy: pull-push
only:
- master@gitlab-org/gitlab-ce
- master@gitlab-org/gitlab-ee
compile-assets pull-cache:
extends: .compile-assets-metadata
cache:
policy: pull
except:
refs:
- master@gitlab-org/gitlab-ce
- master@gitlab-org/gitlab-ee
- /(^docs[\/-].+|.+-docs$)/
cache:
policy: pull
karma:
extends: .dedicated-no-docs-pull-cache-job
<<: *use-pg
dependencies:
- compile-assets
- compile-assets pull-cache
- setup-test-env
extends:
- .default-tags
- .default-retry
- .default-cache
- .default-before_script
- .use-pg
- .except-docs
dependencies: ["compile-assets", "compile-assets pull-cache", "setup-test-env"]
variables:
# we override the max_old_space_size to prevent OOM errors
NODE_OPTIONS: --max_old_space_size=3584
......@@ -142,12 +150,14 @@ karma:
junit: junit_karma.xml
jest:
extends: .dedicated-no-docs-and-no-qa-pull-cache-job
<<: *use-pg
dependencies:
- compile-assets
- compile-assets pull-cache
- setup-test-env
extends:
- .default-tags
- .default-retry
- .default-cache
- .default-before_script
- .use-pg
- .except-docs-qa
dependencies: ["compile-assets", "compile-assets pull-cache", "setup-test-env"]
script:
- scripts/gitaly-test-spawn
- date
......@@ -170,36 +180,41 @@ jest:
- tmp/jest/jest/
policy: pull-push
qa:internal:
extends: .dedicated-no-docs-no-db-pull-cache-job
services: []
script:
.qa:
extends:
- .default-tags
- .default-retry
- .default-cache
- .except-docs
dependencies: ["setup-test-env"]
variables:
SETUP_DB: "false"
before_script:
- cd qa/
- bundle install
qa:internal:
extends: .qa
script:
- bundle exec rspec
dependencies:
- setup-test-env
qa:selectors:
extends: .dedicated-no-docs-no-db-pull-cache-job
services: []
extends: .qa
script:
- cd qa/
- bundle install
- bundle exec bin/qa Test::Sanity::Selectors
dependencies:
- setup-test-env
.qa-frontend-node: &qa-frontend-node
extends: .dedicated-no-docs-no-db-pull-cache-job
stage: test
.qa-frontend-node:
extends:
- .default-tags
- .default-retry
- .default-cache
- .except-docs
dependencies: []
cache:
key: "$CI_JOB_NAME"
paths:
- .yarn-cache/
policy: pull-push
dependencies: []
before_script: []
script:
- date
- yarn install --frozen-lockfile --cache-folder .yarn-cache --prefer-offline
......@@ -207,23 +222,28 @@ qa:selectors:
- yarn run webpack-prod
qa-frontend-node:8:
<<: *qa-frontend-node
extends: .qa-frontend-node
image: node:carbon
qa-frontend-node:10:
<<: *qa-frontend-node
extends: .qa-frontend-node
image: node:dubnium
qa-frontend-node:latest:
<<: *qa-frontend-node
extends: .qa-frontend-node
image: node:latest
allow_failure: true
lint:javascript:report:
extends: .dedicated-no-docs-no-db-pull-cache-job
extends:
- .default-tags
- .default-retry
- .default-cache
- .except-docs
variables:
SETUP_DB: "false"
stage: post-test
dependencies: []
before_script: []
script:
- date
- yarn run eslint-report || true # ignore exit code
......@@ -234,12 +254,15 @@ lint:javascript:report:
- eslint-report.html
jsdoc:
extends: .dedicated-no-docs-no-db-pull-cache-job
extends:
- .default-tags
- .default-retry
- .default-cache
- .except-docs
variables:
SETUP_DB: "false"
stage: post-test
dependencies:
- compile-assets
- compile-assets pull-cache
before_script: []
dependencies: ["compile-assets", "compile-assets pull-cache"]
script:
- date
- yarn run jsdoc || true # ignore exit code
......
.dedicated-runner:
.default-tags:
tags:
- gitlab-org
.default-retry:
retry:
max: 2 # This is confusing but this means "3 runs at max".
when:
- unknown_failure
- api_failure
- runner_system_failure
tags:
- gitlab-org
.default-cache: &default-cache
key: "debian-stretch-ruby-2.6.3-node-12.x"
paths:
- vendor/ruby
- .yarn-cache/
- vendor/gitaly-ruby
.dedicated-runner-default-cache:
extends: .dedicated-runner
cache:
<<: *default-cache
.default-before_script:
before_script:
- date
- source scripts/utils.sh
- source scripts/prepare_build.sh
- date
# Jobs that only need to pull cache
.dedicated-pull-cache-job:
extends: .dedicated-runner
.default-cache:
cache:
<<: *default-cache
key: "debian-stretch-ruby-2.6.3-node-12.x"
paths:
- vendor/ruby
- .yarn-cache/
- vendor/gitaly-ruby
policy: pull
stage: test
.no-docs:
.except-docs:
except:
refs:
- /(^docs[\/-].+|.+-docs$)/
.no-docs-and-no-qa:
.except-qa:
except:
refs:
- /(^docs[\/-].+|.+-docs$)/
- /(^qa[\/-].*|.*-qa$)/
.dedicated-no-docs-pull-cache-job:
extends:
- .dedicated-pull-cache-job
- .no-docs
.dedicated-no-docs-and-no-qa-pull-cache-job:
extends:
- .dedicated-pull-cache-job
- .no-docs-and-no-qa
# Jobs that do not need a DB
.dedicated-no-docs-no-db-pull-cache-job:
extends: .dedicated-no-docs-pull-cache-job
variables:
SETUP_DB: "false"
# Jobs that need a dedicated runner, with no cache
.dedicated-no-docs:
extends:
- .dedicated-runner
- .no-docs
.except-docs-qa:
except:
refs:
- /(^docs[\/-].+|.+-docs$)/
- /(^qa[\/-].*|.*-qa$)/
.single-script-job-dedicated-runner:
extends: .dedicated-runner
image: ruby:2.6-alpine
stage: test
cache: {}
dependencies: []
variables:
GIT_STRATEGY: none
before_script:
# We don't clone the repo by using GIT_STRATEGY: none and only download the
# single script we need here so it's much faster than cloning.
- export SCRIPT_NAME="${SCRIPT_NAME:-$CI_JOB_NAME}"
- apk add --update openssl
- wget $CI_PROJECT_URL/raw/$CI_COMMIT_SHA/scripts/$SCRIPT_NAME
- chmod 755 $(basename $SCRIPT_NAME)
.except-docs-qa-geo:
except:
refs:
- /(^docs[\/-].+|.+-docs$)/
- /(^qa[\/-].*|.*-qa$)/
- /(^geo[\/-].*|.*-geo$)/
.review-only: &review-only
.review-only:
only:
refs:
- branches@gitlab-org/gitlab-ce
......@@ -88,3 +61,16 @@
- master
- /^\d+-\d+-auto-deploy-\d+$/
- /(^docs[\/-].+|.+-docs$)/
.use-pg:
services:
- name: postgres:9.6.14
command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
- name: redis:alpine
.use-pg-10:
image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.6.3-golang-1.11-git-2.22-chrome-73.0-node-12.x-yarn-1.16-postgresql-10-graphicsmagick-1.3.33"
services:
- name: postgres:10.9
command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
- name: redis:alpine
memory-static:
extends: .dedicated-no-docs-no-db-pull-cache-job
extends:
- .default-tags
- .default-retry
- .default-cache
- .default-before_script
- .except-docs
variables:
SETUP_DB: "false"
script:
# Uses two different reports from the 'derailed_benchmars' gem.
......@@ -23,7 +30,13 @@ memory-static:
# The application is booted in `production` environment.
# All tests are run without a webserver (directly using Rack::Mock by default).
memory-on-boot:
extends: .rspec-metadata-pg-10
extends:
- .default-tags
- .default-retry
- .default-cache
- .default-before_script
- .use-pg-10
- .except-docs-qa
variables:
NODE_ENV: "production"
RAILS_ENV: "production"
......
pages:
extends: .dedicated-no-docs-no-db-pull-cache-job
before_script: []
extends:
- .default-tags
- .default-retry
- .default-cache
- .except-docs
only:
refs:
- master@gitlab-org/gitlab-ce
- master@gitlab-org/gitlab-ee
stage: pages
dependencies:
- coverage
- karma
- gitlab:assets:compile
- lint:javascript:report
- jsdoc
dependencies: ["coverage", "karma", "gitlab:assets:compile", "lint:javascript:report", "jsdoc"]
script:
- mv public/ .public/
- mkdir public/
......@@ -21,6 +23,3 @@ pages:
artifacts: