This instance will be upgraded to Heptapod 0.26.0rc1 on 2021-10-26 at 14:30 UTC+2 (a few minutes of down time)

Commit 2a94785f authored by Fabio Pitino's avatar Fabio Pitino
Browse files

CE port for pipelines for external pull requests

Detect if pipeline runs for a GitHub pull request

When using a mirror for CI/CD only we register a pull_request
webhook. When a pull_request webhook is received, if the
source branch SHA matches the actual head of the branch in the
repository we create immediately a new pipeline for the
external pull request. Otherwise we store the
pull request info for when the push webhook is received.

When using "only/except: external_pull_requests" we can detect
if the pipeline has a open pull request on GitHub and create or
not the job based on that.
parent ed9c127f842f
......@@ -23,6 +23,7 @@ class Pipeline < ApplicationRecord
belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline'
belongs_to :pipeline_schedule, class_name: 'Ci::PipelineSchedule'
belongs_to :merge_request, class_name: 'MergeRequest'
belongs_to :external_pull_request
has_internal_id :iid, scope: :project, presence: false, init: ->(s) do
s&.project&.all_pipelines&.maximum(:iid) || s&.project&.all_pipelines&.count
......@@ -64,6 +65,11 @@ class Pipeline < ApplicationRecord
validates :merge_request, presence: { if: :merge_request_event? }
validates :merge_request, absence: { unless: :merge_request_event? }
validates :tag, inclusion: { in: [false], if: :merge_request_event? }
validates :external_pull_request, presence: { if: :external_pull_request_event? }
validates :external_pull_request, absence: { unless: :external_pull_request_event? }
validates :tag, inclusion: { in: [false], if: :external_pull_request_event? }
validates :status, presence: { unless: :importing? }
validate :valid_commit_sha, unless: :importing?
validates :source, exclusion: { in: %w(unknown), unless: :importing? }, on: :create
......@@ -675,6 +681,10 @@ def predefined_variables
variables.append(key: 'CI_MERGE_REQUEST_TARGET_BRANCH_SHA', value: target_sha.to_s)
variables.concat(merge_request.predefined_variables)
end
if external_pull_request_event? && external_pull_request
variables.concat(external_pull_request.predefined_variables)
end
end
end
......
......@@ -23,7 +23,8 @@ def self.sources
api: 5,
external: 6,
chat: 8,
merge_request_event: 10
merge_request_event: 10,
external_pull_request_event: 11
}
end
......
# frozen_string_literal: true
# This model stores pull requests coming from external providers, such as
# GitHub, when GitLab project is set as CI/CD only and remote mirror.
#
# When setting up a remote mirror with GitHub we subscribe to push and
# pull_request webhook events. When a pull request is opened on GitHub,
# a webhook is sent out, we create or update the status of the pull
# request locally.
#
# When the mirror is updated and changes are pushed to branches we check
# if there are open pull requests for the source and target branch.
# If so, we create pipelines for external pull requests.
class ExternalPullRequest < ApplicationRecord
include Gitlab::Utils::StrongMemoize
include ShaAttribute
belongs_to :project
sha_attribute :source_sha
sha_attribute :target_sha
validates :source_branch, presence: true
validates :target_branch, presence: true
validates :source_sha, presence: true
validates :target_sha, presence: true
validates :source_repository, presence: true
validates :target_repository, presence: true
validates :status, presence: true
enum status: {
open: 1,
closed: 2
}
# We currently don't support pull requests from fork, so
# we are going to return an error to the webhook
validate :not_from_fork
scope :by_source_branch, ->(branch) { where(source_branch: branch) }
scope :by_source_repository, -> (repository) { where(source_repository: repository) }
def self.create_or_update_from_params(params)
find_params = params.slice(:project_id, :source_branch, :target_branch)
safe_find_or_initialize_and_update(find: find_params, update: params) do |pull_request|
yield(pull_request) if block_given?
end
end
def actual_branch_head?
actual_source_branch_sha == source_sha
end
def from_fork?
source_repository != target_repository
end
def source_ref
Gitlab::Git::BRANCH_REF_PREFIX + source_branch
end
def predefined_variables
Gitlab::Ci::Variables::Collection.new.tap do |variables|
variables.append(key: 'CI_EXTERNAL_PULL_REQUEST_IID', value: pull_request_iid.to_s)
variables.append(key: 'CI_EXTERNAL_PULL_REQUEST_SOURCE_BRANCH_SHA', value: source_sha)
variables.append(key: 'CI_EXTERNAL_PULL_REQUEST_TARGET_BRANCH_SHA', value: target_sha)
variables.append(key: 'CI_EXTERNAL_PULL_REQUEST_SOURCE_BRANCH_NAME', value: source_branch)
variables.append(key: 'CI_EXTERNAL_PULL_REQUEST_TARGET_BRANCH_NAME', value: target_branch)
end
end
private
def actual_source_branch_sha
project.commit(source_ref)&.sha
end
def not_from_fork
if from_fork?
errors.add(:base, 'Pull requests from fork are not supported')
end
end
def self.safe_find_or_initialize_and_update(find:, update:)
safe_ensure_unique(retries: 1) do
model = find_or_initialize_by(find)
if model.update(update)
yield(model) if block_given?
end
model
end
end
end
......@@ -291,6 +291,8 @@ class Project < ApplicationRecord
has_many :remote_mirrors, inverse_of: :project
has_many :cycle_analytics_stages, class_name: 'Analytics::CycleAnalytics::ProjectStage'
has_many :external_pull_requests, inverse_of: :project
accepts_nested_attributes_for :variables, allow_destroy: true
accepts_nested_attributes_for :project_feature, update_only: true
accepts_nested_attributes_for :import_data
......
......@@ -18,7 +18,8 @@ class CreatePipelineService < BaseService
Gitlab::Ci::Pipeline::Chain::Limit::Activity,
Gitlab::Ci::Pipeline::Chain::Limit::JobActivity].freeze
def execute(source, ignore_skip_ci: false, save_on_errors: true, trigger_request: nil, schedule: nil, merge_request: nil, **options, &block)
# rubocop: disable Metrics/ParameterLists
def execute(source, ignore_skip_ci: false, save_on_errors: true, trigger_request: nil, schedule: nil, merge_request: nil, external_pull_request: nil, **options, &block)
@pipeline = Ci::Pipeline.new
command = Gitlab::Ci::Pipeline::Chain::Command.new(
......@@ -32,6 +33,7 @@ def execute(source, ignore_skip_ci: false, save_on_errors: true, trigger_request
trigger_request: trigger_request,
schedule: schedule,
merge_request: merge_request,
external_pull_request: external_pull_request,
ignore_skip_ci: ignore_skip_ci,
save_incompleted: save_on_errors,
seeds_block: block,
......@@ -62,6 +64,7 @@ def execute(source, ignore_skip_ci: false, save_on_errors: true, trigger_request
pipeline
end
# rubocop: enable Metrics/ParameterLists
def execute!(*args, &block)
execute(*args, &block).tap do |pipeline|
......
# frozen_string_literal: true
# This service is responsible for creating a pipeline for a given
# ExternalPullRequest coming from other providers such as GitHub.
module ExternalPullRequests
class CreatePipelineService < BaseService
def execute(pull_request)
return unless pull_request.open? && pull_request.actual_branch_head?
create_pipeline_for(pull_request)
end
private
def create_pipeline_for(pull_request)
Ci::CreatePipelineService.new(project, current_user, create_params(pull_request))
.execute(:external_pull_request_event, external_pull_request: pull_request)
end
def create_params(pull_request)
{
ref: pull_request.source_ref,
source_sha: pull_request.source_sha,
target_sha: pull_request.target_sha
}
end
end
end
......@@ -160,6 +160,7 @@
- repository_import
- repository_remove_remote
- system_hook_push
- update_external_pull_requests
- update_merge_requests
- update_project_statistics
- upload_checksum
......
# frozen_string_literal: true
class UpdateExternalPullRequestsWorker
include ApplicationWorker
def perform(project_id, user_id, ref)
project = Project.find_by_id(project_id)
return unless project
user = User.find_by_id(user_id)
return unless user
branch = Gitlab::Git.branch_name(ref)
return unless branch
external_pull_requests = project.external_pull_requests
.by_source_repository(project.import_source)
.by_source_branch(branch)
external_pull_requests.find_each do |pull_request|
ExternalPullRequests::CreatePipelineService.new(project, user)
.execute(pull_request)
end
end
end
......@@ -115,3 +115,4 @@
- [export_csv, 1]
- [incident_management, 2]
- [jira_connect, 1]
- [update_external_pull_requests, 3]
# frozen_string_literal: true
class CreateExternalPullRequests < ActiveRecord::Migration[5.2]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
INDEX = 'index_external_pull_requests_on_project_and_branches'
def change
create_table :external_pull_requests do |t|
t.timestamps_with_timezone null: false
t.references :project, null: false, foreign_key: { on_delete: :cascade }, index: false
t.integer :pull_request_iid, null: false
t.integer :status, null: false, limit: 2
t.string :source_branch, null: false, limit: 255
t.string :target_branch, null: false, limit: 255
t.string :source_repository, null: false, limit: 255
t.string :target_repository, null: false, limit: 255
t.binary :source_sha, null: false
t.binary :target_sha, null: false
t.index [:project_id, :source_branch, :target_branch], unique: true, name: INDEX
end
end
end
# frozen_string_literal: true
class AddExternalPullRequestIdToCiPipelines < ActiveRecord::Migration[5.2]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def up
add_column :ci_pipelines, :external_pull_request_id, :bigint
end
def down
remove_column :ci_pipelines, :external_pull_request_id
end
end
# frozen_string_literal: true
class AddIndexToCiPipelinesExternalPullRequest < ActiveRecord::Migration[5.2]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_concurrent_index :ci_pipelines, :external_pull_request_id, where: 'external_pull_request_id IS NOT NULL'
end
def down
remove_concurrent_index :ci_pipelines, :external_pull_request_id
end
end
# frozen_string_literal: true
class AddForeignKeyToCiPipelinesExternalPullRequest < ActiveRecord::Migration[5.2]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_concurrent_foreign_key :ci_pipelines, :external_pull_requests, column: :external_pull_request_id, on_delete: :nullify
end
def down
remove_foreign_key :ci_pipelines, :external_pull_requests
end
end
......@@ -752,7 +752,9 @@
t.integer "merge_request_id"
t.binary "source_sha"
t.binary "target_sha"
t.bigint "external_pull_request_id"
t.index ["auto_canceled_by_id"], name: "index_ci_pipelines_on_auto_canceled_by_id"
t.index ["external_pull_request_id"], name: "index_ci_pipelines_on_external_pull_request_id", where: "(external_pull_request_id IS NOT NULL)"
t.index ["merge_request_id"], name: "index_ci_pipelines_on_merge_request_id", where: "(merge_request_id IS NOT NULL)"
t.index ["pipeline_schedule_id"], name: "index_ci_pipelines_on_pipeline_schedule_id"
t.index ["project_id", "iid"], name: "index_ci_pipelines_on_project_id_and_iid", unique: true, where: "(iid IS NOT NULL)"
......@@ -1321,6 +1323,21 @@
t.index ["target_type", "target_id"], name: "index_events_on_target_type_and_target_id"
end
create_table "external_pull_requests", force: :cascade do |t|
t.datetime_with_timezone "created_at", null: false
t.datetime_with_timezone "updated_at", null: false
t.bigint "project_id", null: false
t.integer "pull_request_iid", null: false
t.integer "status", limit: 2, null: false
t.string "source_branch", limit: 255, null: false
t.string "target_branch", limit: 255, null: false
t.string "source_repository", limit: 255, null: false
t.string "target_repository", limit: 255, null: false
t.binary "source_sha", null: false
t.binary "target_sha", null: false
t.index ["project_id", "source_branch", "target_branch"], name: "index_external_pull_requests_on_project_and_branches", unique: true
end
create_table "feature_gates", id: :serial, force: :cascade do |t|
t.string "feature_key", null: false
t.string "key", null: false
......@@ -3781,6 +3798,7 @@
add_foreign_key "ci_pipeline_variables", "ci_pipelines", column: "pipeline_id", name: "fk_f29c5f4380", on_delete: :cascade
add_foreign_key "ci_pipelines", "ci_pipeline_schedules", column: "pipeline_schedule_id", name: "fk_3d34ab2e06", on_delete: :nullify
add_foreign_key "ci_pipelines", "ci_pipelines", column: "auto_canceled_by_id", name: "fk_262d4c2d19", on_delete: :nullify
add_foreign_key "ci_pipelines", "external_pull_requests", name: "fk_190998ef09", on_delete: :nullify
add_foreign_key "ci_pipelines", "merge_requests", name: "fk_a23be95014", on_delete: :cascade
add_foreign_key "ci_pipelines", "projects", name: "fk_86635dbd80", on_delete: :cascade
add_foreign_key "ci_runner_namespaces", "ci_runners", column: "runner_id", on_delete: :cascade
......@@ -3845,6 +3863,7 @@
add_foreign_key "events", "namespaces", column: "group_id", on_delete: :cascade
add_foreign_key "events", "projects", on_delete: :cascade
add_foreign_key "events", "users", column: "author_id", name: "fk_edfd187b6f", on_delete: :cascade
add_foreign_key "external_pull_requests", "projects", on_delete: :cascade
add_foreign_key "fork_network_members", "fork_networks", on_delete: :cascade
add_foreign_key "fork_network_members", "projects", column: "forked_from_project_id", name: "fk_b01280dae4", on_delete: :nullify
add_foreign_key "fork_network_members", "projects", on_delete: :cascade
......
......@@ -19,6 +19,7 @@ def perform!
user: @command.current_user,
pipeline_schedule: @command.schedule,
merge_request: @command.merge_request,
external_pull_request: @command.external_pull_request,
variables_attributes: Array(@command.variables_attributes)
)
......
......@@ -7,7 +7,7 @@ module Chain
Command = Struct.new(
:source, :project, :current_user,
:origin_ref, :checkout_sha, :after_sha, :before_sha, :source_sha, :target_sha,
:trigger_request, :schedule, :merge_request,
:trigger_request, :schedule, :merge_request, :external_pull_request,
:ignore_skip_ci, :save_incompleted,
:seeds_block, :variables_attributes, :push_options,
:chat_data, :allow_mirror_update
......
......@@ -64,6 +64,8 @@ project_tree:
- :push_event_payload
- stages:
- :statuses
- :external_pull_request
- :external_pull_requests
- :auto_devops
- :triggers
- :pipeline_schedules
......
# frozen_string_literal: true
FactoryBot.define do
factory :external_pull_request do
sequence(:pull_request_iid)
project
source_branch 'feature'
source_repository 'the-repository'
source_sha '97de212e80737a608d939f648d959671fb0a0142'
target_branch 'master'
target_repository 'the-repository'
target_sha 'a09386439ca39abe575675ffd4b89ae824fec22f'
status :open
trait(:closed) { status 'closed' }
end
end
......@@ -84,6 +84,20 @@
.not_to be_satisfied_by(pipeline)
end
end
context 'when source is external_pull_request_event' do
let(:pipeline) { build_stubbed(:ci_pipeline, source: :external_pull_request_event) }
it 'is satisfied with only: external_pull_request' do
expect(described_class.new(%w[external_pull_requests]))
.to be_satisfied_by(pipeline)
end
it 'is not satisfied with only: external_pull_request_event' do
expect(described_class.new(%w[external_pull_request_events]))
.not_to be_satisfied_by(pipeline)
end
end
end
context 'when matching a ref by a regular expression' do
......
......@@ -128,4 +128,38 @@
expect(pipeline.target_sha).to eq(merge_request.target_branch_sha)
end
end
context 'when pipeline is running for an external pull request' do
let(:command) do
Gitlab::Ci::Pipeline::Chain::Command.new(
source: :external_pull_request_event,
origin_ref: 'feature',
checkout_sha: project.commit.id,
after_sha: nil,
before_sha: nil,
source_sha: external_pull_request.source_sha,
target_sha: external_pull_request.target_sha,
trigger_request: nil,
schedule: nil,
external_pull_request: external_pull_request,
project: project,
current_user: user)
end
let(:external_pull_request) { build(:external_pull_request, project: project) }
before do
step.perform!
end
it 'correctly indicated that this is an external pull request pipeline' do
expect(pipeline).to be_external_pull_request_event
expect(pipeline.external_pull_request).to eq(external_pull_request)
end
it 'correctly sets source sha and target sha to pipeline' do
expect(pipeline.source_sha).to eq(external_pull_request.source_sha)
expect(pipeline.target_sha).to eq(external_pull_request.target_sha)
end
end
end
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment