Commit 03a6bce6 authored by Kamil Trzciński's avatar Kamil Trzciński
Browse files

Merge branch 'ci-config-on-policy' into 'master'

Flexible Rules for CI Build config

See merge request gitlab-org/gitlab-ce!29011
---
title: Introduced Build::Rules configuration for Ci::Build
merge_request: 29011
author:
type: added
......@@ -10,7 +10,7 @@ def initialize(expressions)
end
def satisfied_by?(pipeline, seed)
variables = seed.to_resource.scoped_variables_hash
variables = seed.scoped_variables_hash
statements = @expressions.map do |statement|
::Gitlab::Ci::Pipeline::Expression::Statement
......
# frozen_string_literal: true
module Gitlab
module Ci
module Build
class Rules
include ::Gitlab::Utils::StrongMemoize
Result = Struct.new(:when, :start_in)
def initialize(rule_hashes, default_when = 'on_success')
@rule_list = Rule.fabricate_list(rule_hashes)
@default_when = default_when
end
def evaluate(pipeline, build)
if @rule_list.nil?
Result.new(@default_when)
elsif matched_rule = match_rule(pipeline, build)
Result.new(
matched_rule.attributes[:when] || @default_when,
matched_rule.attributes[:start_in]
)
else
Result.new('never')
end
end
private
def match_rule(pipeline, build)
@rule_list.find { |rule| rule.matches?(pipeline, build) }
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module Ci
module Build
class Rules::Rule
attr_accessor :attributes
def self.fabricate_list(list)
list.map(&method(:new)) if list
end
def initialize(spec)
@clauses = []
@attributes = {}
spec.each do |type, value|
if clause = Clause.fabricate(type, value)
@clauses << clause
else
@attributes.merge!(type => value)
end
end
end
def matches?(pipeline, build)
@clauses.all? { |clause| clause.satisfied_by?(pipeline, build) }
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module Ci
module Build
class Rules::Rule::Clause
##
# Abstract class that defines an interface of a single
# job rule specification.
#
# Used for job's inclusion rules configuration.
#
UnknownClauseError = Class.new(StandardError)
def self.fabricate(type, value)
type = type.to_s.camelize
self.const_get(type).new(value) if self.const_defined?(type)
end
def initialize(spec)
@spec = spec
end
def satisfied_by?(pipeline, seed = nil)
raise NotImplementedError
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module Ci
module Build
class Rules::Rule::Clause::Changes < Rules::Rule::Clause
def initialize(globs)
@globs = Array(globs)
end
def satisfied_by?(pipeline, seed)
return true if pipeline.modified_paths.nil?
pipeline.modified_paths.any? do |path|
@globs.any? do |glob|
File.fnmatch?(glob, path, File::FNM_PATHNAME | File::FNM_DOTMATCH | File::FNM_EXTGLOB)
end
end
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module Ci
module Build
class Rules::Rule::Clause::If < Rules::Rule::Clause
def initialize(expression)
@expression = expression
end
def satisfied_by?(pipeline, seed)
variables = seed.scoped_variables_hash
::Gitlab::Ci::Pipeline::Expression::Statement.new(@expression, variables).truthful?
end
end
end
end
end
......@@ -11,7 +11,8 @@ class Job < ::Gitlab::Config::Entry::Node
include ::Gitlab::Config::Entry::Configurable
include ::Gitlab::Config::Entry::Attributable
ALLOWED_KEYS = %i[tags script only except type image services
ALLOWED_WHEN = %w[on_success on_failure always manual delayed].freeze
ALLOWED_KEYS = %i[tags script only except rules type image services
allow_failure type stage when start_in artifacts cache
dependencies needs before_script after_script variables
environment coverage retry parallel extends].freeze
......@@ -19,12 +20,19 @@ class Job < ::Gitlab::Config::Entry::Node
REQUIRED_BY_NEEDS = %i[stage].freeze
validations do
validates :config, type: Hash
validates :config, allowed_keys: ALLOWED_KEYS
validates :config, required_keys: REQUIRED_BY_NEEDS, if: :has_needs?
validates :config, presence: true
validates :script, presence: true
validates :name, presence: true
validates :name, type: Symbol
validates :config,
disallowed_keys: {
in: %i[only except when start_in],
message: 'key may not be used with `rules`'
},
if: :has_rules?
with_options allow_nil: true do
validates :tags, array_of_strings: true
......@@ -32,17 +40,19 @@ class Job < ::Gitlab::Config::Entry::Node
validates :parallel, numericality: { only_integer: true,
greater_than_or_equal_to: 2,
less_than_or_equal_to: 50 }
validates :when,
inclusion: { in: %w[on_success on_failure always manual delayed],
message: 'should be on_success, on_failure, ' \
'always, manual or delayed' }
validates :when, inclusion: {
in: ALLOWED_WHEN,
message: "should be one of: #{ALLOWED_WHEN.join(', ')}"
}
validates :dependencies, array_of_strings: true
validates :needs, array_of_strings: true
validates :extends, array_of_strings_or_string: true
validates :rules, array_of_hashes: true
end
validates :start_in, duration: { limit: '1 day' }, if: :delayed?
validates :start_in, absence: true, unless: :delayed?
validates :start_in, absence: true, if: -> { has_rules? || !delayed? }
validate do
next unless dependencies.present?
......@@ -91,6 +101,9 @@ class Job < ::Gitlab::Config::Entry::Node
entry :except, Entry::Policy,
description: 'Refs policy this job will be executed for.'
entry :rules, Entry::Rules,
description: 'List of evaluable Rules to determine job inclusion.'
entry :variables, Entry::Variables,
description: 'Environment variables available for this job.'
......@@ -112,7 +125,7 @@ class Job < ::Gitlab::Config::Entry::Node
:parallel, :needs
attributes :script, :tags, :allow_failure, :when, :dependencies,
:needs, :retry, :parallel, :extends, :start_in
:needs, :retry, :parallel, :extends, :start_in, :rules
def self.matching?(name, config)
!name.to_s.start_with?('.') &&
......@@ -151,6 +164,10 @@ def delayed?
self.when == 'delayed'
end
def has_rules?
@config.try(:key?, :rules)
end
def ignored?
allow_failure.nil? ? manual_action? : allow_failure
end
......
# frozen_string_literal: true
module Gitlab
module Ci
class Config
module Entry
class Rules < ::Gitlab::Config::Entry::Node
include ::Gitlab::Config::Entry::Validatable
validations do
validates :config, presence: true
validates :config, type: Array
end
def compose!(deps = nil)
super(deps) do
@config.each_with_index do |rule, index|
@entries[index] = ::Gitlab::Config::Entry::Factory.new(Entry::Rules::Rule)
.value(rule)
.with(key: "rule", parent: self, description: "rule definition.") # rubocop:disable CodeReuse/ActiveRecord
.create!
end
@entries.each_value do |entry|
entry.compose!(deps)
end
end
end
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module Ci
class Config
module Entry
class Rules::Rule < ::Gitlab::Config::Entry::Node
include ::Gitlab::Config::Entry::Validatable
include ::Gitlab::Config::Entry::Attributable
CLAUSES = %i[if changes].freeze
ALLOWED_KEYS = %i[if changes when start_in].freeze
ALLOWED_WHEN = %w[on_success on_failure always never manual delayed].freeze
attributes :if, :changes, :when, :start_in
validations do
validates :config, presence: true
validates :config, type: { with: Hash }
validates :config, allowed_keys: ALLOWED_KEYS
validates :config, disallowed_keys: %i[start_in], unless: :specifies_delay?
validates :start_in, presence: true, if: :specifies_delay?
validates :start_in, duration: { limit: '1 day' }, if: :specifies_delay?
with_options allow_nil: true do
validates :if, expression: true
validates :changes, array_of_strings: true
validates :when, allowed_values: { in: ALLOWED_WHEN }
end
end
def specifies_delay?
self.when == 'delayed'
end
def default
end
end
end
end
end
end
......@@ -7,7 +7,7 @@ module Seed
class Build < Seed::Base
include Gitlab::Utils::StrongMemoize
delegate :dig, to: :@attributes
delegate :dig, to: :@seed_attributes
# When the `ci_dag_limit_needs` is enabled it uses the lower limit
LOW_NEEDS_LIMIT = 5
......@@ -15,14 +15,20 @@ class Build < Seed::Base
def initialize(pipeline, attributes, previous_stages)
@pipeline = pipeline
@attributes = attributes
@seed_attributes = attributes
@previous_stages = previous_stages
@needs_attributes = dig(:needs_attributes)
@using_rules = attributes.key?(:rules)
@using_only = attributes.key?(:only)
@using_except = attributes.key?(:except)
@only = Gitlab::Ci::Build::Policy
.fabricate(attributes.delete(:only))
@except = Gitlab::Ci::Build::Policy
.fabricate(attributes.delete(:except))
@rules = Gitlab::Ci::Build::Rules
.new(attributes.delete(:rules))
end
def name
......@@ -31,8 +37,13 @@ def name
def included?
strong_memoize(:inclusion) do
all_of_only? &&
none_of_except?
if @using_rules
included_by_rules?
elsif @using_only || @using_except
all_of_only? && none_of_except?
else
true
end
end
end
......@@ -45,19 +56,13 @@ def errors
end
def attributes
@attributes.merge(
pipeline: @pipeline,
project: @pipeline.project,
user: @pipeline.user,
ref: @pipeline.ref,
tag: @pipeline.tag,
trigger_request: @pipeline.legacy_trigger,
protected: @pipeline.protected_ref?
)
@seed_attributes
.deep_merge(pipeline_attributes)
.deep_merge(rules_attributes)
end
def bridge?
attributes_hash = @attributes.to_h
attributes_hash = @seed_attributes.to_h
attributes_hash.dig(:options, :trigger).present? ||
(attributes_hash.dig(:options, :bridge_needs).instance_of?(Hash) &&
attributes_hash.dig(:options, :bridge_needs, :pipeline).present?)
......@@ -73,6 +78,18 @@ def to_resource
end
end
def scoped_variables_hash
strong_memoize(:scoped_variables_hash) do
# This is a temporary piece of technical debt to allow us access
# to the CI variables to evaluate rules before we persist a Build
# with the result. We should refactor away the extra Build.new,
# but be able to get CI Variables directly from the Seed::Build.
::Ci::Build.new(
@seed_attributes.merge(pipeline_attributes)
).scoped_variables_hash
end
end
private
def all_of_only?
......@@ -109,6 +126,28 @@ def max_needs_allowed
HARD_NEEDS_LIMIT
end
end
def pipeline_attributes
{
pipeline: @pipeline,
project: @pipeline.project,
user: @pipeline.user,
ref: @pipeline.ref,
tag: @pipeline.tag,
trigger_request: @pipeline.legacy_trigger,
protected: @pipeline.protected_ref?
}
end
def included_by_rules?
rules_attributes[:when] != 'never'
end
def rules_attributes
strong_memoize(:rules_attributes) do
@using_rules ? @rules.evaluate(@pipeline, self).to_h.compact : {}
end
end
end
end
end
......
......@@ -20,8 +20,10 @@ def validate_each(record, attribute, value)
present_keys = value.try(:keys).to_a & options[:in]
if present_keys.any?
record.errors.add(attribute, "contains disallowed keys: " +
present_keys.join(', '))
message = options[:message] || "contains disallowed keys"
message += ": #{present_keys.join(', ')}"
record.errors.add(attribute, message)
end
end
end
......@@ -65,6 +67,16 @@ def validate_each(record, attribute, value)
end
end
class ArrayOfHashesValidator < ActiveModel::EachValidator
include LegacyValidationHelpers
def validate_each(record, attribute, value)
unless value.is_a?(Array) && value.map { |hsh| hsh.is_a?(Hash) }.all?
record.errors.add(attribute, 'should be an array of hashes')
end
end
end
class ArrayOrStringValidator < ActiveModel::EachValidator
def validate_each(record, attribute, value)
unless value.is_a?(Array) || value.is_a?(String)
......@@ -231,6 +243,14 @@ def validate_each(record, attribute, value)
end
end
class ExpressionValidator < ActiveModel::EachValidator
def validate_each(record, attribute, value)
unless value.is_a?(String) && ::Gitlab::Ci::Pipeline::Expression::Statement.new(value).valid?
record.errors.add(attribute, 'Invalid expression syntax')
end
end
end
class PortNamePresentAndUniqueValidator < ActiveModel::EachValidator
def validate_each(record, attribute, value)
return unless value.is_a?(Array)
......
......@@ -13,7 +13,12 @@
build(:ci_build, pipeline: pipeline, project: project, ref: 'master')
end
let(:seed) { double('build seed', to_resource: ci_build) }
let(:seed) do
double('build seed',
to_resource: ci_build,
scoped_variables_hash: ci_build.scoped_variables_hash
)
end
before do
pipeline.variables.build(key: 'CI_PROJECT_NAME', value: '')
......@@ -83,7 +88,12 @@
build(:ci_bridge, pipeline: pipeline, project: project, ref: 'master')
end
let(:seed) { double('bridge seed', to_resource: bridge) }
let(:seed) do
double('bridge seed',
to_resource: bridge,
scoped_variables_hash: ci_build.scoped_variables_hash
)
end
it 'is satisfied by a matching expression for a bridge job' do
policy = described_class.new(['$MY_VARIABLE'])
......
require 'spec_helper'
describe Gitlab::Ci::Build::Rules::Rule do
let(:seed) do
double('build seed',
to_resource: ci_build,
scoped_variables_hash: ci_build.scoped_variables_hash
)
end
let(:pipeline) { create(:ci_pipeline) }
let(:ci_build) { build(:ci_build, pipeline: pipeline) }
let(:rule) { described_class.new(rule_hash) }
describe '#matches?' do
subject { rule.matches?(pipeline, seed) }
context 'with one matching clause' do
let(:rule_hash) do
{ if: '$VAR == null', when: 'always' }
end
it { is_expected.to eq(true) }
end
context 'with two matching clauses' do
let(:rule_hash) do
{ if: '$VAR == null', changes: '**/*', when: 'always' }
end
it { is_expected.to eq(true) }
end
context 'with a matching and non-matching clause' do
let(:rule_hash) do
{ if: '$VAR != null', changes: '$VAR == null', when: 'always' }
end
it { is_expected.to eq(false) }
end
context 'with two non-matching clauses' do
let(:rule_hash) do
{ if: '$VAR != null', changes: 'README', when: 'always' }
end
it { is_expected.to eq(false) }
end
end
end
require 'spec_helper'
describe Gitlab::Ci::Build::Rules do
let(:pipeline) { create(:ci_pipeline) }
let(:ci_build) { build(:ci_build, pipeline: pipeline) }
let(:seed) do
double('build seed',
to_resource: ci_build,
scoped_variables_hash: ci_build.scoped_variables_hash
)
end
let(:rules) { described_class.new(rule_list) }
describe '.new' do
let(:rules_ivar) { rules.instance_variable_get :@rule_list }
let(:default_when) { rules.instance_variable_get :@default_when }
context 'with no rules' do
let(:rule_list) { [] }
it 'sets @rule_list to an empty array' do
expect(rules_ivar).to eq([])
end
it 'sets @default_when to "on_success"' do
expect(default_when).to eq('on_success')
end
end
context 'with one rule' do
let(:rule_list) { [{ if: '$VAR == null', when: 'always' }] }
it 'sets @rule_list to an array of a single rule' do
expect(rules_ivar).to be_an(Array)
end
it 'sets @default_when to "on_success"' do
expect(default_when).to eq('on_success')