Commit 8d00a434 authored by Georges Racinet's avatar Georges Racinet 🦑
Browse files

Merge from heptapod-stable into support branch for release

--HG--
branch : heptapod-0-13
......@@ -165,6 +165,7 @@ class Project < ApplicationRecord
has_one :pushover_service
has_one :jira_service
has_one :redmine_service
has_one :readthedocs_service
has_one :youtrack_service
has_one :custom_issue_tracker_service
has_one :bugzilla_service
......@@ -1152,7 +1153,14 @@ def find_or_initialize_services(exceptions: [])
end
def disabled_services
[]
if mercurial?
[]
else
# Heptapod Read the Doc integration is for Mercurial only (assumes
# namings of branches). On Git repos, it's not needed: Read the Docs
# has a Gitlab integration, meant for the generic GitLab outgoing webhook.
["readthedocs"]
end
end
def find_or_initialize_service(name)
......
# coding: utf-8
# frozen_string_literal: true
# Send webhook to Read the docs according to https://docs.readthedocs.io/en/stable/webhooks.html#using-the-generic-api-integration
#
# we don't use the `ServiceHook` subsystem in this integration, because
# it is meant to pass the token in query params, and Read the Docs reference
# cited above instructs to send it as form data.
class ReadthedocsService < Service
boolean_accessor :skip_tls_verification
prop_accessor :project_url, :token
validates :project_url, presence: true, public_url: true, if: :activated?
validates :token, presence: true, if: :activated?
def title
'Read the Docs'
end
def description
'Automate Read the Docs builds upon changes'
end
def self.to_param
'readthedocs'
end
def self.supported_events
%w(push)
end
def help
'This service sends notifications about branch changes to Read the Docs
to trigger new builds.
To set up this service:
- Refer to the [Read the Docs instructions](https://docs.readthedocs.io/en/stable/webhooks.html#using-the-generic-api-integration)
- Paste the **Webhook URL** and the **token** into the fields below.
'
end
def webhook_placeholder
'https://readthedocs.org/api/v2/webhook/example-project/1/'
end
def execute(push_data)
gitlab_branch = Gitlab::Git::branch_name(push_data["ref"])
begin
branch, topic = Gitlab::Mercurial::parse_gitlab_branch_for_hg(
gitlab_branch)
rescue StandardError
# bookmark, TODO yes there could be a more semantic exception for that
return
end
# We don't want to notify Read the Docs for not landed changesets
# hence we filter out topics.
# TODO we could also restrict to a given set of named branches. After
# all only those that are configured on the Read the Docs side will
# be useful. This would require a new field/property.
return unless topic.nil?
Rails.logger.info("readthedocs service/integration "\
"notifying push_data branch=#{push_data['ref']} \
url=#{project_url}")
Gitlab::HTTP.post(project_url, body: { branches: branch, token: token },
verify: !skip_tls_verification?)
end
def fields
[
{ type: 'text', name: 'project_url',
title: 'Read the Docs project webhook URL',
placeholder: webhook_placeholder,
required: true},
{ type: 'password', name: 'token',
title: 'The Read the Docs token',
required: true},
{ type: 'checkbox', name: 'skip_tls_verification',
title: 'Skip TLS/SSL verification',
}
]
end
end
......@@ -274,6 +274,7 @@ def self.available_services_names
pivotaltracker
prometheus
pushover
readthedocs
redmine
youtrack
slack_slash_commands
......
# frozen_string_literal: true
module MergeRequests
# Service for manually merged Mercurial Merge Requests
#
# - update the MR to the latest relevant changeset (can be `nil` if caller
# cannot find it)
# - mark it as merged
# - execute all hooks and notifications
#
class HgManuallyMergedService < MergeRequests::BaseService
def execute(merge_request, latest_sha)
refresh(merge_request, latest_sha)
MergeRequests::PostMergeService.new(merge_request.target_project, @current_user).execute(merge_request)
end
# Refresh the MR to include a new commit, without reloading everything.
#
# Typically, with topics publication, the source branch has already vanished
# reloading everything would not understand that.
def refresh(merge_request, real_source_sha)
return if real_source_sha.nil?
recorded_source_sha = merge_request.diff_head_sha
return if recorded_source_sha == real_source_sha
# the new diff creation apparently makes several Gitaly calls,
# which can be an error depending on running options (probably not in
# production, but that may change in GitLab versions). It's not the
# responsibility of this service to avoid that.
Gitlab::GitalyClient.allow_n_plus_1_calls do
old_diff_refs = merge_request.diff_refs
# base should not have changed if target is a Mercurial named
# branch (TODO one day we'll have MRs between topics)
MergeRequestDiff.create(
merge_request_id: merge_request.id,
head_commit_sha: real_source_sha,
base_commit_sha: old_diff_refs.base_sha,
start_commit_sha: old_diff_refs.start_sha,
)
new_diff_refs = merge_request.diff_refs
merge_request.update_diff_discussion_positions(
old_diff_refs: old_diff_refs,
new_diff_refs: new_diff_refs,
current_user: @current_user)
end
end
end
end
......@@ -66,36 +66,12 @@ def hg_git?
@project.vcs_type == 'hg_git'
end
def hg_refresh_manually_merged(merge_request, real_source_sha)
recorded_source_sha = merge_request.diff_head_sha
return if recorded_source_sha == real_source_sha
old_diff_refs = merge_request.diff_refs
# base should not have changed if target is a Mercurial named
# branch (TODO one day we'll have MRs between topics)
new_diff = MergeRequestDiff.create(
merge_request_id: merge_request.id,
head_commit_sha: real_source_sha,
base_commit_sha: old_diff_refs.base_sha,
start_commit_sha: old_diff_refs.start_sha,
)
new_diff.save_git_content
new_diff_refs = merge_request.diff_refs
merge_request.update_diff_discussion_positions(
old_diff_refs: old_diff_refs,
new_diff_refs: new_diff_refs,
current_user: @current_user)
end
# Collect open merge requests that target same branch we push into
# and close if push to master include last commit from merge request
# We need this to close(as merged) merge requests that were merged into
# target branch manually
# rubocop: disable CodeReuse/ActiveRecord
def post_merge_manually_merged
mercurial = project.vcs_type == "hg_git" # TODO graft commit from 0.8?
commit_ids = @commits.map(&:id)
merge_requests = @project.merge_requests.opened
.preload(:latest_merge_request_diff)
......@@ -118,47 +94,13 @@ def post_merge_manually_merged
else
merge_request.head_source_sha
end
merged = commit_ids.include?(source_sha) &&
merge_request.merge_request_diff.state != 'empty'
hg_refresh_manually_merged(merge_request, source_sha) if hg_git? && merged
merged
end
is_topic = @push.branch_name.start_with?('topic/')
if hg_git? && is_topic
# we need to treat merge requests of which the branch is the source
# as well, because if the source branch is processed first,
# then it'll go to next step if not merged, i.e. reload_diff, which
# makes the diff empty on rebase/publish.
merge_requests.concat(
@project.merge_requests.opened
.preload(:latest_merge_request_diff)
.where(source_branch: @push.branch_name).to_a
.select(&:diff_head_commit)
.select do |merge_request|
real_target_sha = merge_request.target_branch_sha
merged = @project.repository.ancestor?(@push.newrev, real_target_sha)
hg_refresh_manually_merged(merge_request, @push.newrev) if merged
merged
end
)
commit_ids.include?(source_sha) &&
merge_request.merge_request_diff.state != 'empty'
end
for mr in merge_requests
Rails.logger.info("#{logprefix} marking MR id=#{mr.id} iid=#{mr.id} as merged")
end
if merge_requests.empty?
if is_topic
named_branch_name = 'branch/' + @push.branch_name.split('/')[1..-2].join('/')
named_branch_sha = @project.repository.find_branch(named_branch_name)&.target
if named_branch_sha.nil?
Rails.logger.warn("#{logprefix} could not find GitLab branch #{named_branch_name}")
else
if @project.repository.ancestor?(@push.newrev, named_branch_sha)
Rails.logger.info("#{logprefix} removing GitLab branch for published "\
"topic with no Merge Request")
DeleteBranchService.new(@project, current_user).execute(@push.branch_name, ignore_git_error: true)
end
end
end
Rails.logger.info("#{logprefix} no merge detected")
end
......@@ -170,7 +112,7 @@ def post_merge_manually_merged
# to work with Mercurial and publication of topics, and would be very
# clumsy anyway compared to what can be done with revsets.
# The feature is in itself interesting - we could provide it directly
commit_analyze_enabled = !mercurial && Feature.enabled?(:branch_push_merge_commit_analyze, @project, default_enabled: true)
commit_analyze_enabled = !project.mercurial? && Feature.enabled?(:branch_push_merge_commit_analyze, @project, default_enabled: true)
if commit_analyze_enabled
analyzer = Gitlab::BranchPushMergeCommitAnalyzer.new(
@commits.reverse,
......@@ -179,7 +121,7 @@ def post_merge_manually_merged
end
merge_requests.each do |merge_request|
if commit_analyze_enabled && !analyzer.nil?
if commit_analyze_enabled
merge_request.merge_commit_sha = analyzer.get_merge_commit(merge_request.diff_head_sha)
end
......
......@@ -5,7 +5,7 @@ class PostReceive
PIPELINE_PROCESS_LIMIT = 4
def perform(gl_repository, identifier, changes, push_options = {})
def perform(gl_repository, identifier, hg_prunes, changes, push_options = {})
project, repo_type = Gitlab::GlRepository.parse(gl_repository)
if project.nil?
......@@ -19,6 +19,10 @@ def perform(gl_repository, identifier, changes, push_options = {})
Sidekiq.logger.info "changes: #{changes.inspect}" if ENV['SIDEKIQ_LOG_ARGUMENTS']
post_received = Gitlab::GitPostReceive.new(project, identifier, changes, push_options)
if repo_type.project? && project.mercurial? && !hg_prunes.nil?
hg_process_project_prunes(post_received, hg_prunes)
end
if repo_type.wiki?
process_wiki_changes(post_received)
elsif repo_type.project?
......@@ -36,6 +40,131 @@ def identify_user(post_received)
end
end
def hg_process_project_prunes(post_received, prunes)
project = post_received.project
user = identify_user(post_received)
logprefix = "PostReceive.hg_process_project_prunes for Project id=#{project.id}"
prunes = JSON.parse(prunes)
published_topics = prunes["topic_published"]
unless published_topics.nil?
published_topics.each_pair do |gl_branch, latest_sha|
gl_branch = base64_to_utf8(gl_branch)
Rails.logger.info("#{logprefix} looking for MRs from published topic '#{gl_branch}'")
hg_branch, _topic = Gitlab::Mercurial::parse_gitlab_branch_for_hg(gl_branch)
# locate merge request with topic as source.
# The odds are that generally, a MR for a published topic will target the
# corresponding named branch. Therefore is most cases, we won't need to
# preload the MR diffs like GitLab's standard detection does: we'll reach the
# conclusion without them. Only if the MR is merged will we need the diff,
# for potential last update to `latest_sha`.
# TODO OPTIM make just one SQL request for all published topics
# and then, sort/match them (could be defeated by the way ActiveRecord
# fetches from the DB, though).
merged = project.merge_requests.opened
.where(source_branch: gl_branch, source_project: project).to_a
.select do |merge_request|
if merge_request.target_branch == 'branch/' + hg_branch
Rails.logger.info(
"#{logprefix} marking MR iid=#{merge_request.iid} merged: "\
"source is the published '#{gl_branch}' "\
"and target is the corresponding named branch")
true
else
real_target_sha = merge_request.target_branch_sha
# TODO what if previous position is merged and subsequent
# changes are not (or maybe in another branch?)
merged = project.repository.ancestor?(latest_sha, real_target_sha)
Rails.logger.info(
"#{logprefix} analysis of MR iid=#{merge_request.iid} "\
"gives merged=#{merged}. source is the published \
" "'#{gl_branch}', with latest SHA #{latest_sha} "\
"and target #{merge_request.target_branch} is another named "\
"branch with current head SHA #{real_target_sha}")
merged
# If merged is false, we don't need to close `merge_request`:
# MergeRequests::RefreshService will take care of that later
# with all necessary details.
end
end
merged.each do |merge_request|
MergeRequests::HgManuallyMergedService.new(project, user).execute(merge_request, latest_sha)
end
end
end
closed_branches = prunes['branch_closed']
unless closed_branches.nil?
merged_with_sha = []
closed_branches.each_pair do |gl_branch, info|
next if info.nil?
gl_branch = base64_to_utf8(gl_branch)
info.each do |closing_sha, parent_shas|
merged_with_sha.concat(merged_for_closing_changeset(
project, gl_branch,
closing_sha, parent_shas))
end
end
merged_with_sha.each do |merge_request, latest_sha|
MergeRequests::HgManuallyMergedService.new(project, user).execute(merge_request, latest_sha)
end
end
end
# Given a closing commit SHA and the SHAs of its parents, find the
# Merge Requests that have to be updated and merged.
def merged_for_closing_changeset(project, gl_branch, closing_sha, parent_shas)
logprefix = "PostReceive analyzing Merge Requests for changeset "\
"#{closing_sha}, that closes '#{gl_branch}'"
repo = project.repository
merged = []
project.merge_requests.opened
.where(source_branch: gl_branch, source_project: project)
.each do |merge_request|
real_target_sha = merge_request.target_branch_sha
if repo.ancestor?(closing_sha, real_target_sha)
merged << [merge_request, closing_sha]
Rails.logger.info("#{logprefix}: it is a direct ancestor "\
"of MR iid=#{merge_request.iid}, targeting "\
"'#{merge_request.target_branch}'. "\
"Marking as merged.")
else
if parent_shas.all? {|psha| repo.ancestor?(psha, real_target_sha)}
Rails.logger.info("#{logprefix}: all its parents #{parent_shas} "\
"are ancestors of MR iid=#{merge_request.iid}, "\
"targeting '#{merge_request.target_branch}'. "\
"Marking as merged.")
# if the closing commit has several parents, since we have to update
# the MR with a single head, let's put in the closing commit to see
# all of them and be faithful to what happened.
merged << [merge_request,
parent_shas.length == 1? parent_shas[0] : closing_sha]
end
end
end
merged
end
# Deserialize a binary string from Base64, then guess and convert to UTF-8
#
# This is the same as what happens for the `changes` param, here (base64)
# and in `Gitlab::GitPostReceive` (transcoding).
def base64_to_utf8(s)
decoded = Base64.decode64(s)
decoded.force_encoding('UTF-8')
return decoded if decoded.valid_encoding?
detection = CharlockHolmes::EncodingDetector.detect(changes)
return decoded unless detection && detection[:encoding]
CharlockHolmes::Converter.convert(decoded, detection[:encoding], 'UTF-8')
end
def hg_fixup_default_branch(post_received)
project = post_received.project
return if !project.mercurial?
......
# coding: utf-8
# frozen_string_literal: true
module API
......@@ -598,6 +599,26 @@ def self.services
desc: 'The sound of the notification'
}
],
'readthedocs' => [
{
required: true,
name: :project_url,
type: String,
desc: 'The Read the Docs project webhook URL'
},
{
required: true,
name: :token,
type: String,
desc: 'The Read the Docs token'
},
{
required: false,
name: :skip_tls_verification,
type: Boolean,
desc: 'Skip TLS/SSL verification for communication'
}
],
'redmine' => [
{
required: true,
......@@ -719,6 +740,7 @@ def self.service_classes
::PivotaltrackerService,
::PrometheusService,
::PushoverService,
::ReadthedocsService,
::RedmineService,
::YoutrackService,
::SlackService,
......
......@@ -296,6 +296,7 @@ def ee_post_receive_response_hook(response)
response.reference_counter_decreased = Gitlab::ReferenceCounter.new(params[:gl_repository]).decrease
PostReceive.perform_async(params[:gl_repository], params[:identifier],
params[:hg_prunes],
params[:changes], push_options.as_json)
mr_options = push_options.get(:merge_request)
......
......@@ -8,6 +8,16 @@ def hgrc_path
Gitlab.config.mercurial.hgrc.join(":")
end
def parse_gitlab_branch_for_hg(gitlab_branch)
if gitlab_branch.start_with?('branch/')
[gitlab_branch[7..], nil]
elsif gitlab_branch.start_with?('topic/')
branch, _slash, topic = gitlab_branch[6..].rpartition('/')
[branch, topic]
else
raise StandardError, "bookmarks not supported (got #{gitlab_branch})"
end
end
end
end
end
......@@ -179,6 +179,7 @@ def hg_changeset_branch(hgsha)
out.strip
end
# TODO duplication use the one from Mercurial module
def parse_gitlab_branch_for_hg(gitlab_branch)
if gitlab_branch.start_with?('branch/')
[gitlab_branch[7..], nil]
......
......@@ -4,4 +4,5 @@ gunicorn==19.10.0
sentry-sdk==0.9.5
heptapod==0.13.0
hg-git==0.9.0a1
dulwich==0.19.16
hg-loggingmod==0.2.0
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment