diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7a7f9ec8f1c06de938921331c523706f6ffd4d4b_LmdpdGxhYi1jaS55bWw=..a687d96ee9f997dc4eb2eab41e84ed99c7a4358e_LmdpdGxhYi1jaS55bWw= 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,37 +21,14 @@ - trigger-qa - verify -workflow: - rules: - # Do not create a pipeline on branch push to QA mirror - - if: '$CI_PROJECT_NAME == "omnibus-gitlab-mirror" && $CI_PIPELINE_SOURCE == "push"' - when: never - # No pipeline on auto-deploy branches as a tag will definitely follow - - if: '$CI_COMMIT_BRANCH =~ /^[0-9]+-[0-9]+-auto-deploy-[0-9]+$/' - when: never - # Set CACHE_EDITION variable to GITLAB for pipelines triggered from gitlab-org/gitlab project so that they don't pollute regular branch cache - - if: '$TOP_UPSTREAM_SOURCE_PROJECT == "gitlab-org/gitlab"' - variables: - CACHE_EDITION: "GITLAB" - # Set CACHE_EDITION variable to STABLE on stable branch pipelines so that they don't pollute regular branch cache - - if: '$CI_COMMIT_BRANCH =~ /^.*-stable$/' - variables: - CACHE_EDITION: "STABLE" - # Set CACHE_EDITION variable to EE on EE pipelines - - if: '$ee == "true" || $GITLAB_VERSION =~ /^*-ee/ || $CI_COMMIT_TAG =~ /^\d+\.\d+\.\d+\+[^ ]{7,}\.[^ ]{7,}$/ || $CI_COMMIT_TAG =~ /^.*ee\.\d.*$/ || $CI_COMMIT_BRANCH =~ /^.*-ee$/' - variables: - CACHE_EDITION: "EE" - # For all other branches, create a pipeline. We are explicitly specifying - # this so that this rule gets matched earlier before MR pipelines gets - # triggered, thus causing two pipelines for a branch push - a regular one - # and a detached one. If we ever decide not to run pipelines on branch - # pushes that doesn't cause an MR, we can change the following to - # $CI_MERGE_REQUEST_IID - - if: '$CI_COMMIT_BRANCH' - when: always - # For tags, always create a pipeline. - - if: '$CI_COMMIT_TAG' - when: always +include: + - local: '/gitlab-ci-config/workflow-rules.yml' + - local: '/gitlab-ci-config/dev-gitlab-org.yml' + rules: + - if: '$CI_SERVER_HOST == "dev.gitlab.org"' + - local: '/gitlab-ci-config/gitlab-com.yml' + rules: + - if: '$CI_SERVER_HOST != "dev.gitlab.org"' default: tags: @@ -99,6 +76,7 @@ - gitlab-org-docker before_script: + - echo "PIPELINE_TYPE detected as ${PIPELINE_TYPE}" - export CE_ONLY=(Raspberry) - export EE_ONLY=(SLES RAT) - for job in "${CE_ONLY[@]}"; do @@ -166,22 +144,12 @@ reports: dotenv: build_facts/env_vars rules: - - if: '$DEPS_PIPELINE || $DEPENDENCY_SCANNING' - when: never - # The CACHE_UPDATE variable has two use cases: - # 1. Scheduled nightly builds in omnibus-gitlab-mirror that update the - # cache to improve speed for package-and-qa builds from - # omnibus-gitlab merge requests. - # 1. The gitlab-org/gitlab project schedules a pipeline with - # CACHE_UPDATE every two hours. The resulting package-and-qa - # pipeline refreshes the cache and reduces build times for - # other GitLab merge requests that trigger package-and-qa. - # The first one doesn't have a qa-test job, and hence doesn't need - # the generate-facts job. - - if: '$CACHE_UPDATE && $CI_PIPELINE_SOURCE == "schedule"' - when: never - - if: '$CI_BRANCH_NAME =~ /^[0-9]+-[0-9]+-auto-deploy-[0-9]+$/' - when: never - - when: always + # Temporary, until we use PIPELINE_TYPE in dev.gitlab.org pipelines also + - if: '$CI_PROJECT_PATH == "gitlab/omnibus-gitlab"' + - if: '$PIPELINE_TYPE == "TRIGGER_CACHE_UPDATE_PIPELINE"' + - if: '$PIPELINE_TYPE == "GITLAB_BRANCH_TEST_PIPELINE"' + - if: '$PIPELINE_TYPE == "GITLAB_MR_PIPELINE"' + - if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/' + - when: never needs: [] @@ -186,13 +154,5 @@ needs: [] -include: - - local: '/gitlab-ci-config/gitlab-com.yml' - rules: - - if: '$CI_SERVER_HOST == "gitlab.com"' - - local: '/gitlab-ci-config/dev-gitlab-org.yml' - rules: - - if: '$CI_SERVER_HOST == "dev.gitlab.org"' - .notify: before_script: - apk add --no-cache curl diff --git a/doc/development/pipelines.md b/doc/development/pipelines.md index 7a7f9ec8f1c06de938921331c523706f6ffd4d4b_ZG9jL2RldmVsb3BtZW50L3BpcGVsaW5lcy5tZA==..a687d96ee9f997dc4eb2eab41e84ed99c7a4358e_ZG9jL2RldmVsb3BtZW50L3BpcGVsaW5lcy5tZA== 100644 --- a/doc/development/pipelines.md +++ b/doc/development/pipelines.md @@ -86,6 +86,40 @@ ## Types of pipelines +Since the jobs in `omnibus-gitlab` are spread across multiple mirrors, the CI +config becomes relatively complex when using `rules` keyword for specifying +where/when to run jobs. Hence, a different approach is used for +`omnibus-gitlab`, which marks each necessary pipeline type with a specific +variable named `PIPELINE_TYPE`, and attach jobs to various pipeline types as +needed. The different pipeline types are documented in the below table: + +| Pipeline type | Mirror(s) where the pipeline runs | Remarks | +| -------------------------------- | --------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | +| `DEPENDENCY_SCANNING_PIPELINE` | Canonical | Check for security vulnerabilities in dependencies. Requires `DEPENDENCY_SCANNING` variable to be set to `true`. | +| `DEPS_IO_VERSION_BUMP_PIPELINE` | Canonical | On branch pushes done by `deps.io`. Requires branches to have `deps-` prefix. | +| `DEPS_IO_VERSION_CHECK_PIPELINE` | Canonical | Run `deps` to detect updates. Requires `DEPS_PIPELINE` variable to be set to `true`. | +| `LICENSE_PAGE_UPDATE_PIPELINE` | Canonical | Update license pages. Requires `PAGES_UPDATE` variable to be set to `true`. | +| `CACHE_UPDATE_PIPELINE` | Canonical, QA | Update the gem cache and package build cache. Requires `CACHE_UPDATE` variable to be set to `true`. | +| `DURATION_PLOTTER_PIPELINE` | QA | Package build to plot duration of builds. Requires `DURATION_PLOTTER` variable to be set to `true`. | +| `DOCS_PIPELINE` | Canonical, Security | Requires branch names to either have `docs-` as prefix or `-docs` as suffix. | +| `PROTECTED_TEST_PIPELINE` | Canonical, Security | On protected branches and tags. Does not include trigger jobs and other unwanted jobs like danger-review. | +| `GITLAB_BRANCH_TEST_PIPELINE` | Canonical, Security | On non-protected branch pushes. Does not include unwanted jobs like danger-review. Includes trigger jobs. | +| `GITLAB_MR_PIPELINE` | Canonical, Security | On MRs with branches in Canonical/Security (i.e. MRs from GitLab and team members). Has trigger jobs. | +| `AUTO_DEPLOY_BUILD_PIPELINE` | Release | Builds when auto-deploy tags are pushed. | +| `CE_BRANCH_BUILD_PIPELINE` | Release | Builds when regular branches are pushed. | +| `CE_NIGHTLY_BUILD_PIPELINE` | Release | Nightly CE builds. | +| `CE_RC_BUILD_PIPELINE` | Release | Builds when CE RC tags are pushed. | +| `CE_TAG_BUILD_PIPELINE` | Release | Builds when stable CE tags are pushed. | +| `EE_BRANCH_BUILD_PIPELINE` | Release | Builds when regular branches are pushed, but forced to be an EE pipeline. | +| `EE_NIGHTLY_BUILD_PIPELINE` | Release | Nightly EE builds. Requires pipeline to be forced to be an EE one. | +| `EE_RC_BUILD_PIPELINE` | Release | Builds when EE RC tags are pushed. | +| `EE_TAG_BUILD_PIPELINE` | Release | Builds when stable EE tags are pushed. | +| `TRIGGER_CACHE_UPDATE_PIPELINE` | QA | Updates build cache of triggered QA pipelines. Requires `CACHE_UPDATE` variable to be set to `true`. | +| `TRIGGERED_CE_PIPELINE` | QA | Triggered package-and-qa build with CE packages and images. | +| `TRIGGERED_EE_PIPELINE` | QA | Triggered package-and-qa build with EE packages and images. | +| `FORK_BRANCH_TEST_PIPELINE` | Forks | Test suite when run on forks of the project. Does not include trigger jobs and other unwanted jobs like danger-review. | +| `FORK_MR_PIPELINE` | Forks | On MRs from forks of the project. Does not include trigger jobs. | + ### Branch pipelines `omnibus-gitlab` doesn't make use of detached merge request pipelines yet. So, diff --git a/gitlab-ci-config/gitlab-com.yml b/gitlab-ci-config/gitlab-com.yml index 7a7f9ec8f1c06de938921331c523706f6ffd4d4b_Z2l0bGFiLWNpLWNvbmZpZy9naXRsYWItY29tLnltbA==..a687d96ee9f997dc4eb2eab41e84ed99c7a4358e_Z2l0bGFiLWNpLWNvbmZpZy9naXRsYWItY29tLnltbA== 100644 --- a/gitlab-ci-config/gitlab-com.yml +++ b/gitlab-ci-config/gitlab-com.yml @@ -4,73 +4,6 @@ ############# # Templates # ############# -.com-docs-except-triggers: &com-docs-except-triggers - except: - refs: - - /./@gitlab/omnibus-gitlab - - triggers - - /^deps/ - - /./@gitlab-org/build/omnibus-gitlab-mirror - variables: - - $DEPS_PIPELINE || $CACHE_UPDATE || $DEPENDENCY_SCANNING - - $CI_COMMIT_REF_NAME =~ /^[0-9]+-[0-9]+-auto-deploy-[0-9]+$/ - -.com-except-triggers-and-docs: &com-except-triggers-and-docs - except: - refs: - - /./@gitlab/omnibus-gitlab - - triggers - - /(^docs[\/-].*|.*-docs$)/ - - /./@gitlab-org/build/omnibus-gitlab-mirror - variables: - - $DEPS_PIPELINE || $CACHE_UPDATE || $DEPENDENCY_SCANNING - - $CI_COMMIT_REF_NAME =~ /^[0-9]+-[0-9]+-auto-deploy-[0-9]+$/ - -.com-except-triggers-docs-and-deps: &com-except-triggers-docs-and-deps - except: - refs: - - /./@gitlab/omnibus-gitlab - - triggers - - /^deps/ - - /(^docs[\/-].*|.*-docs$)/ - - /./@gitlab-org/build/omnibus-gitlab-mirror - variables: - - $DEPS_PIPELINE || $CACHE_UPDATE || $DEPENDENCY_SCANNING - - $CI_COMMIT_REF_NAME =~ /^[0-9]+-[0-9]+-auto-deploy-[0-9]+$/ - -.com-branches-except-triggers-docs-and-deps: &com-branches-except-triggers-docs-and-deps - only: - - branches@gitlab-org/omnibus-gitlab - - branches@gitlab-org/security/omnibus-gitlab - <<: *com-except-triggers-docs-and-deps - -.com-branches-except-triggers-and-docs: &com-branches-except-triggers-and-docs - only: - - branches@gitlab-org/omnibus-gitlab - - branches@gitlab-org/security/omnibus-gitlab - <<: *com-except-triggers-and-docs - -.com-trigger-only: &com-trigger-only - only: - - triggers@gitlab-org/build/omnibus-gitlab-mirror - - pipelines@gitlab-org/build/omnibus-gitlab-mirror - -.com-trigger-and-nightly-only: &com-trigger-and-nightly-only - rules: - - if: '$DEPS_PIPELINE || $DEPENDENCY_SCANNING' - when: never - - if: '$CI_PIPELINE_SOURCE == "trigger" && $CI_PROJECT_PATH == "gitlab-org/build/omnibus-gitlab-mirror"' - - if: '$CI_PIPELINE_SOURCE == "pipeline" && $CI_PROJECT_PATH == "gitlab-org/build/omnibus-gitlab-mirror"' - - if: '$CI_PIPELINE_SOURCE == "schedule" && $NIGHTLY == "true" && $CI_PROJECT_PATH == "gitlab-org/build/omnibus-gitlab-mirror"' - - when: never - -.depscan-rules: &depscan-rules - only: - refs: - - schedules@gitlab-org/omnibus-gitlab - variables: - - $DEPENDENCY_SCANNING - .knapsack-artifacts: &knapsack-artifacts expire_in: 31d paths: @@ -94,7 +27,9 @@ - export DISTRO_VERSION=${JOB_NAME[1]} - mkdir -p knapsack/ - '[[ -f knapsack/${DISTRO_NAME}_${DISTRO_VERSION}_main_rspec_report.json ]] || echo "{}" > knapsack/${DISTRO_NAME}_${DISTRO_VERSION}_main_rspec_report.json' - <<: *com-except-triggers-docs-and-deps + rules: + - if: '$PIPELINE_TYPE =~ /_TEST_PIPELINE$/' + - if: '$PIPELINE_TYPE =~ /_MR_PIPELINE$/' retry: 1 needs: - rubocop @@ -163,6 +98,9 @@ - job: generate-facts optional: true artifacts: true + rules: + - if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/' + - if: '$PIPELINE_TYPE == "DURATION_PLOTTER_PIPELINE"' .spec_template: &spec_template extends: .gems-cache @@ -172,10 +110,9 @@ # so tests run fine on forks - export ALTERNATIVE_SOURCES="true"; - *install-gems - <<: *com-except-triggers-docs-and-deps retry: 1 script: - bundle exec rspec spec/lib artifacts: reports: &spec_reports junit: junit_rspec.xml @@ -176,9 +113,12 @@ retry: 1 script: - bundle exec rspec spec/lib artifacts: reports: &spec_reports junit: junit_rspec.xml + rules: + - if: '$PIPELINE_TYPE =~ /_TEST_PIPELINE$/' + - if: '$PIPELINE_TYPE =~ /_MR_PIPELINE$/' .chef_spec_template: extends: .spec_template @@ -224,8 +164,7 @@ TOP_UPSTREAM_SOURCE_SHA: ${TOP_UPSTREAM_SOURCE_SHA} TOP_UPSTREAM_SOURCE_REF: ${TOP_UPSTREAM_SOURCE_REF} QA_BRANCH: ${QA_BRANCH} - when: manual trigger: project: 'gitlab-org/build/omnibus-gitlab-mirror' branch: $CI_COMMIT_REF_NAME strategy: depend @@ -228,10 +167,9 @@ trigger: project: 'gitlab-org/build/omnibus-gitlab-mirror' branch: $CI_COMMIT_REF_NAME strategy: depend - <<: *com-branches-except-triggers-docs-and-deps needs: - job: generate-facts artifacts: true - job: check-for-sha-in-mirror artifacts: false @@ -233,9 +171,21 @@ needs: - job: generate-facts artifacts: true - job: check-for-sha-in-mirror artifacts: false + allow_failure: true + rules: + # Because of inherit:variables being false, `PIPELINE_TYPE` isn't available here. :( + # https://gitlab.com/gitlab-org/gitlab/-/issues/368759 + - if: '$CI_PIPELINE_SOURCE=="schedule" || $CI_PIPELINE_SOURCE=="pipeline"' + when: never + - if: '$CI_COMMIT_REF_PROTECTED == "true"' + when: never + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_PROJECT_PATH == "gitlab-org/omnibus-gitlab" || $CI_PROJECT_PATH == "gitlab-org/security/omnibus-gitlab")' + when: manual + - if: '$CI_COMMIT_BRANCH && ($CI_PROJECT_PATH == "gitlab-org/omnibus-gitlab" || $CI_PROJECT_PATH == "gitlab-org/security/omnibus-gitlab")' + when: manual .trigger-pipeline: stage: trigger-qa image: "${RUBY_IMAGE}" @@ -238,6 +188,5 @@ .trigger-pipeline: stage: trigger-qa image: "${RUBY_IMAGE}" - when: manual allow_failure: true @@ -243,5 +192,7 @@ allow_failure: true - <<: *com-trigger-only + rules: + - if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/' + when: manual needs: - job: Trigger:package artifacts: false @@ -265,15 +216,14 @@ variables: GIT_STRATEGY: none DOCS_REVIEW_APPS_DOMAIN: 35.193.151.162.nip.io - when: manual - only: - refs: - - branches@gitlab-org/omnibus-gitlab - except: - variables: - - $CI_COMMIT_REF_NAME =~ /^[0-9]+-[0-9]+-auto-deploy-[0-9]+$/ - - $CI_COMMIT_REF_NAME =~ /^deps/ - - $DEPS_PIPELINE || $CACHE_UPDATE || $DEPENDENCY_SCANNING + allow_failure: true + rules: + - if: '$PIPELINE_TYPE == "GITLAB_BRANCH_TEST_PIPELINE"' + when: manual + - if: '$PIPELINE_TYPE == "GITLAB_MR_PIPELINE"' + when: manual + - if: '$PIPELINE_TYPE == "DOCS_PIPELINE"' + when: manual ##################### # Cache update jobs # @@ -288,11 +238,8 @@ - echo "Cache is up to date!" cache: policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. - only: - refs: - - schedules # Jobs that take advantage of this cache are in the canonical project and the mirror one - variables: - - $CACHE_UPDATE + rules: + - if: '$PIPELINE_TYPE == "CACHE_UPDATE_PIPELINE"' # We need to populate the cache for jobs with the `gitlab-org-docker` tag. Ideally, we wouldn't need this if # we'd use Kaniko to build the Docker images, allowing to use the `gitlab-org` tag instead of the `gitlab-org-docker` tag. @@ -312,12 +259,8 @@ policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. tags: - triggered-packages - only: - refs: - - schedules@gitlab-org/build/omnibus-gitlab-mirror # Jobs that take advantage of this cache are exclusively in the mirror project - - pipelines@gitlab-org/build/omnibus-gitlab-mirror - variables: - - $CACHE_UPDATE + rules: + - if: '$PIPELINE_TYPE == "TRIGGER_CACHE_UPDATE_PIPELINE"' needs: - job: fetch-assets optional: true @@ -332,7 +275,9 @@ image: "${RUBY_IMAGE}" script: - support/wait_for_sha - <<: *com-branches-except-triggers-and-docs + rules: + - if: '$PIPELINE_TYPE == "GITLAB_MR_PIPELINE"' + - if: '$PIPELINE_TYPE == "GITLAB_BRANCH_TEST_PIPELINE"' timeout: 3h needs: [] @@ -352,7 +297,9 @@ - *install-gems script: - bundle exec rubocop --parallel - <<: *com-except-triggers-docs-and-deps + rules: + - if: '$PIPELINE_TYPE =~ /_TEST_PIPELINE$/' + - if: '$PIPELINE_TYPE =~ /_MR_PIPELINE$/' needs: [] # Perform documentation linting on Markdown files @@ -367,7 +314,10 @@ - vale --minAlertLevel error doc # Lint Markdown - markdownlint --config .markdownlint.yml 'doc/**/*.md' - <<: *com-docs-except-triggers + rules: + - if: '$PIPELINE_TYPE =~ /_TEST_PIPELINE$/' + - if: '$PIPELINE_TYPE =~ /_MR_PIPELINE$/' + - if: '$PIPELINE_TYPE == "DOCS_PIPELINE"' # Perform link checks on published HTML files docs-lint links: @@ -386,7 +336,10 @@ - bundle exec nanoc check internal_links # Check the internal anchor links - bundle exec nanoc check internal_anchors - <<: *com-docs-except-triggers + rules: + - if: '$PIPELINE_TYPE =~ /_TEST_PIPELINE$/' + - if: '$PIPELINE_TYPE =~ /_MR_PIPELINE$/' + - if: '$PIPELINE_TYPE == "DOCS_PIPELINE"' yard: extends: .gems-cache @@ -400,9 +353,12 @@ - *install-gems script: - bundle exec yardoc - <<: *com-docs-except-triggers + rules: + - if: '$PIPELINE_TYPE =~ /_TEST_PIPELINE$/' + - if: '$PIPELINE_TYPE =~ /_MR_PIPELINE$/' + - if: '$PIPELINE_TYPE == "LICENSE_PAGE_UPDATE_PIPELINE"' artifacts: expire_in: 1 week paths: - yard/* @@ -404,9 +360,8 @@ artifacts: expire_in: 1 week paths: - yard/* - # Trigger a docs build in gitlab-docs # Useful to preview the docs changes live # https://docs.gitlab.com/ee/development/documentation/index.html#previewing-the-changes-live @@ -439,17 +394,8 @@ cache: {} needs: [] before_script: [] - only: - refs: - - branches@gitlab-org/omnibus-gitlab - except: - refs: - - schedules@gitlab-org/omnibus-gitlab - - /^[\d-]+-stable(-patch-[\d]+)?$/ - - master - variables: - - $DEPS_PIPELINE || $CACHE_UPDATE || $DEPENDENCY_SCANNING - - $CI_COMMIT_REF_NAME =~ /^[0-9]+-[0-9]+-auto-deploy-[0-9]+$/ + rules: + - if: '$PIPELINE_TYPE =~ /_MR_PIPELINE$/' script: - danger --fail-on-errors=true @@ -546,7 +492,9 @@ script: - support/merge-reports knapsack - rm -f knapsack/*node* - <<: *com-except-triggers-docs-and-deps + rules: + - if: '$PIPELINE_TYPE =~ /_TEST_PIPELINE$/' + - if: '$PIPELINE_TYPE =~ /_MR_PIPELINE$/' retry: 1 ############################ @@ -558,7 +506,6 @@ variables: BASE_OS: "ubuntu_20.04" PACKAGE_DIRECTORY: "pkg/ubuntu-focal" - <<: *com-trigger-and-nightly-only Trigger:package:fips: extends: @@ -568,8 +515,10 @@ USE_SYSTEM_SSL: "true" BASE_OS: "ubuntu_20.04_fips" PACKAGE_DIRECTORY: "pkg/ubuntu-focal_fips" - <<: *com-trigger-only - when: manual + allow_failure: true + rules: + - if: '$PIPELINE_TYPE == "TRIGGERED_EE_PIPELINE"' + when: manual package_size_check: extends: .trigger-package-cache @@ -578,7 +527,6 @@ script: - bundle exec rake build:package:generate_sizefile - bundle exec rake check:package_size - <<: *com-trigger-only needs: - job: Trigger:package artifacts: false @@ -582,6 +530,8 @@ needs: - job: Trigger:package artifacts: false + rules: + - if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/' Trigger:gitlab-docker: extends: @@ -591,7 +541,8 @@ script: - bundle exec rake docker:build:image - bundle exec rake docker:push:triggered - <<: *com-trigger-only + rules: + - if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/' needs: - job: Trigger:package artifacts: false @@ -607,10 +558,10 @@ script: - bundle exec rake qa:build - bundle exec rake qa:push:triggered - <<: *com-trigger-only - except: - variables: - - $SKIP_QA_DOCKER == "true" + rules: + - if: '$SKIP_QA_DOCKER == "true"' + when: never + - if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/' needs: - Trigger:package - job: generate-facts @@ -638,10 +589,12 @@ project: "gitlab-org/gitlab-qa-mirror" branch: $QA_BRANCH strategy: depend - <<: *com-trigger-only - except: - variables: - - $SKIP_QA_TEST == "true" + rules: + - if: '$SKIP_QA_TEST == "true"' + when: never + # Because of inherit:variables being false, `PIPELINE_TYPE` isn't available here. :( + # https://gitlab.com/gitlab-org/gitlab/-/issues/368759 + - if: '$CI_PROJECT_PATH == "gitlab-org/build/omnibus-gitlab-mirror" && $CI_PIPELINE_SOURCE=="pipeline"' needs: - job: generate-facts artifacts: true @@ -661,7 +614,8 @@ - curl -L "https://github.com/docker/compose/releases/download/1.29.1/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose - chmod +x /usr/local/bin/docker-compose - bundle exec rake qa:test_letsencrypt - <<: *com-trigger-only + rules: + - if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/' needs: - job: Trigger:gitlab-docker artifacts: false @@ -669,7 +623,6 @@ RAT: stage: trigger-qa image: "${RUBY_IMAGE}" - when: manual allow_failure: true script: - bundle exec rake qa:rat:trigger @@ -673,7 +626,9 @@ allow_failure: true script: - bundle exec rake qa:rat:trigger - <<: *com-trigger-only + rules: + - if: '$PIPELINE_TYPE == "TRIGGERED_EE_PIPELINE"' + when: manual needs: - job: Trigger:package artifacts: false @@ -695,6 +650,8 @@ GET:Geo: extends: .trigger-pipeline + rules: + - if: '$PIPELINE_TYPE == "TRIGGERED_EE_PIPELINE"' script: - bundle exec rake qa:get:geo:trigger @@ -706,7 +663,8 @@ TERM: xterm-256color script: - bundle exec omnibus manifest gitlab -l nothing 2> /dev/null > version-manifest.json - <<: *depscan-rules + rules: + - if: '$PIPELINE_TYPE == "DEPENDENCY_SCANNING_PIPELINE"' artifacts: expire_in: 7 days paths: @@ -721,7 +679,8 @@ before_script: [] script: - /gitlab-depscan.sh version-manifest.json - <<: *depscan-rules + rules: + - if: '$PIPELINE_TYPE == "DEPENDENCY_SCANNING_PIPELINE"' allow_failure: true needs: - create_omnibus_manifest @@ -740,12 +699,11 @@ script: - curl https://deps.app/install.sh | bash -s -- -b $HOME/bin - $HOME/bin/deps ci - only: - variables: - - $DEPS_PIPELINE + rules: + - if: '$PIPELINE_TYPE == "DEPS_IO_VERSION_CHECK_PIPELINE"' dependencies_io_check: extends: .trigger-job when: always variables: ee: "true" @@ -746,16 +704,11 @@ dependencies_io_check: extends: .trigger-job when: always variables: ee: "true" - only: - refs: - - /^deps.*/@gitlab-org/omnibus-gitlab - except: - refs: - - triggers - - branches@gitlab/omnibus-gitlab + rules: + - if: '$PIPELINE_TYPE == "DEPS_IO_VERSION_BUMP_PIPELINE"' validate_packer_changes: before_script: [] @@ -764,10 +717,13 @@ script: - cd "${CI_PROJECT_DIR}/support/packer" && packer validate ce.json - cd "${CI_PROJECT_DIR}/support/packer" && packer validate ee.json - only: - changes: - - support/packer/* - <<: *com-except-triggers-docs-and-deps + rules: + - if: '$PIPELINE_TYPE == "_TEST_PIPELINE"' + changes: + - support/packer/* + - if: '$PIPELINE_TYPE == "GITLAB_MR_PIPELINE"' + changes: + - support/packer/* ############################## # Scheduled pipeline # @@ -786,8 +742,5 @@ artifacts: paths: - public - only: - - schedules@gitlab-org/omnibus-gitlab - except: - variables: - - $DEPS_PIPELINE || $CACHE_UPDATE || $DEPENDENCY_SCANNING + rules: + - if: '$PIPELINE_TYPE == "LICENSE_PAGE_UPDATE_PIPELINE"' diff --git a/gitlab-ci-config/workflow-rules.yml b/gitlab-ci-config/workflow-rules.yml new file mode 100644 index 0000000000000000000000000000000000000000..a687d96ee9f997dc4eb2eab41e84ed99c7a4358e_Z2l0bGFiLWNpLWNvbmZpZy93b3JrZmxvdy1ydWxlcy55bWw= --- /dev/null +++ b/gitlab-ci-config/workflow-rules.yml @@ -0,0 +1,115 @@ +workflow: + rules: + # Avoid duplicate pipeline when an MR is open + - if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push"' + when: never + # Detect pipeline type + - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == "gitlab-org/omnibus-gitlab" && $PAGES_UPDATE == "true" && $CI_PIPELINE_SOURCE == "schedule"' + variables: + PIPELINE_TYPE: LICENSE_PAGE_UPDATE_PIPELINE + - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == "gitlab-org/omnibus-gitlab" && $DEPENDENCY_SCANNING == "true" && $CI_PIPELINE_SOURCE == "schedule"' + variables: + PIPELINE_TYPE: DEPENDENCY_SCANNING_PIPELINE + - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == "gitlab-org/omnibus-gitlab" && $DEPS_PIPELINE == "true" && $CI_PIPELINE_SOURCE == "schedule"' + variables: + PIPELINE_TYPE: DEPS_IO_VERSION_CHECK_PIPELINE + - if: '$PIPELINE_TYPE == null && $CACHE_UPDATE == "true" && $CI_PIPELINE_SOURCE == "schedule"' + variables: + PIPELINE_TYPE: CACHE_UPDATE_PIPELINE + - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == "gitlab-org/build/omnibus-gitlab-mirror" && $CACHE_UPDATE == "true"' + variables: + PIPELINE_TYPE: TRIGGER_CACHE_UPDATE_PIPELINE + - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == "gitlab-org/build/omnibus-gitlab-mirror" && $DURATION_PLOTTER == "true" && $CI_PIPELINE_SOURCE == "schedule"' + variables: + PIPELINE_TYPE: DURATION_PLOTTER_PIPELINE + - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == "gitlab-org/omnibus-gitlab" && $CI_PROJECT_BRANCH =~ /^deps/' + variables: + PIPELINE_TYPE: DEPS_IO_VERSION_BUMP_PIPELINE + - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == "gitlab-org/omnibus-gitlab" && ($CI_PROJECT_BRANCH =~ /^docs/ || $CI_PROJECT_BRANCH =~ /-docs$/)' + variables: + PIPELINE_TYPE: DOCS_PIPELINE + - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == "gitlab-org/omnibus-gitlab" && $CI_PIPELINE_SOURCE == "merge_request_event"' + variables: + PIPELINE_TYPE: GITLAB_MR_PIPELINE + - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == "gitlab-org/omnibus-gitlab" && $CI_COMMIT_REF_PROTECTED == "true"' + variables: + PIPELINE_TYPE: PROTECTED_TEST_PIPELINE + - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == "gitlab-org/omnibus-gitlab" && $CI_COMMIT_BRANCH' + variables: + PIPELINE_TYPE: GITLAB_BRANCH_TEST_PIPELINE + - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == "gitlab/omnibus-gitlab" && $NIGHTLY && $CI_COMMIT_BRANCH && ($ee == "true" || $GITLAB_VERSION =~ /-ee$/)' + variables: + PIPELINE_TYPE: EE_NIGHTLY_BUILD_PIPELINE + - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == "gitlab/omnibus-gitlab" && $CI_COMMIT_BRANCH && ($ee == "true" || $GITLAB_VERSION =~ /-ee$/)' + variables: + PIPELINE_TYPE: EE_BRANCH_BUILD_PIPELINE + - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == "gitlab/omnibus-gitlab" && $NIGHTLY && $CI_COMMIT_BRANCH' + variables: + PIPELINE_TYPE: CE_NIGHTLY_BUILD_PIPELINE + - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == "gitlab/omnibus-gitlab" && $CI_COMMIT_BRANCH' + variables: + PIPELINE_TYPE: CE_BRANCH_BUILD_PIPELINE + - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == "gitlab/omnibus-gitlab" && $CI_COMMIT_TAG =~ /^\d+\.\d+\.\d+\+[^ ]7,\.[^ ]7,$/' + variables: + PIPELINE_TYPE: AUTO_DEPLOY_BUILD_PIPELINE + - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == "gitlab/omnibus-gitlab" && $CI_COMMIT_TAG =~ /^.*\+rc.*\.ce\..*/' + variables: + PIPELINE_TYPE: CE_RC_BUILD_PIPELINE + - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == "gitlab/omnibus-gitlab" && $CI_COMMIT_TAG =~ /^.*\+rc.*\.ee\..*/' + variables: + PIPELINE_TYPE: EE_RC_BUILD_PIPELINE + - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == "gitlab/omnibus-gitlab" && $CI_COMMIT_TAG =~ /^.*ce\.\d.*$/' + variables: + PIPELINE_TYPE: CE_TAG_BUILD_PIPELINE + - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == "gitlab/omnibus-gitlab" && $CI_COMMIT_TAG =~ /^.*ee\.\d.*$/' + variables: + PIPELINE_TYPE: EE_TAG_BUILD_PIPELINE + - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == "gitlab-org/security/omnibus-gitlab" && $CI_PIPELINE_SOURCE == "merge_request_event"' + variables: + PIPELINE_TYPE: GITLAB_MR_PIPELINE + - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == "gitlab-org/security/omnibus-gitlab" && $CI_COMMIT_REF_PROTECTED == "true"' + variables: + PIPELINE_TYPE: PROTECTED_TEST_PIPELINE + - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == "gitlab-org/security/omnibus-gitlab" && $CI_COMMIT_BRANCH' + variables: + PIPELINE_TYPE: GITLAB_BRANCH_TEST_PIPELINE + - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == "gitlab-org/build/omnibus-gitlab-mirror" && ($CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "trigger") && $ee == "true"' + variables: + PIPELINE_TYPE: TRIGGERED_EE_PIPELINE + - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == "gitlab-org/build/omnibus-gitlab-mirror" && ($CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "trigger")' + variables: + PIPELINE_TYPE: TRIGGERED_CE_PIPELINE + - if: '$PIPELINE_TYPE == null && $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_PROJECT_PATH != "gitlab-org/omnibus-gitlab" && $CI_PROJECT_PATH != "gitlab-org/security/omnibus-gitlab" && $CI_PROJECT_PATH != "gitlab-org/build/omnibus-gitlab-mirror" && $CI_PROJECT_PATH != "gitlab/omnibus-gitlab"' + variables: + PIPELINE_TYPE: FORK_MR_PIPELINE + - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH != "gitlab-org/omnibus-gitlab" && $CI_PROJECT_PATH != "gitlab-org/security/omnibus-gitlab" && $CI_PROJECT_PATH != "gitlab-org/build/omnibus-gitlab-mirror" && $CI_PROJECT_PATH != "gitlab/omnibus-gitlab"' + variables: + PIPELINE_TYPE: FORK_BRANCH_TEST_PIPELINE + # If pipeline type could not be detected, do not create a pipeline + - if: '$PIPELINE_TYPE == null' + when: never + # Skip pipelines if configured to do so. For example, + # $SKIP_PIPELINE_SOURCE is set to "push" in QA mirror + - if: '$SKIP_PIPELINE_SOURCE == $CI_PIPELINE_SOURCE' + when: never + # No pipeline on auto-deploy branches as a tag will definitely follow + - if: '$CI_COMMIT_BRANCH =~ /^[0-9]+-[0-9]+-auto-deploy-[0-9]+$/' + when: never + # Set CACHE_EDITION variable to GITLAB for pipelines triggered from gitlab-org/gitlab project so that they don't pollute regular branch cache + - if: '$TOP_UPSTREAM_SOURCE_PROJECT == "gitlab-org/gitlab"' + variables: + CACHE_EDITION: "GITLAB" + # Set CACHE_EDITION variable to STABLE on stable branch pipelines so that they don't pollute regular branch cache + - if: '$CI_COMMIT_BRANCH =~ /^.*-stable$/' + variables: + CACHE_EDITION: "STABLE" + # Set CACHE_EDITION variable to EE on EE pipelines + - if: '$ee == "true" || $GITLAB_VERSION =~ /^*-ee/ || $CI_COMMIT_TAG =~ /^\d+\.\d+\.\d+\+[^ ]{7,}\.[^ ]{7,}$/ || $CI_COMMIT_TAG =~ /^.*ee\.\d.*$/ || $CI_COMMIT_BRANCH =~ /^.*-ee$/' + variables: + CACHE_EDITION: "EE" + # Support Merge Request pipelines + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + # Run regular pipelines on pushes to branches that does not have an MR open + - if: '$CI_COMMIT_BRANCH' + # For tags, always create a pipeline. + - if: '$CI_COMMIT_TAG'