Skip to content
Snippets Groups Projects
Commit a9f1a24b9679 authored by Georges Racinet's avatar Georges Racinet
Browse files

GitLab hooks: post-receive directly through internal REST API

This implements the `post-receive` hook by a direct HTTP call to the
internal API endpoint. For this, two new config parameters in the
`heptapod` section are introduced:

- gitlab-internal-api-secret-file
- gitlab-internal-api-url


We're keeping the previous structure, with a `name`
that's now almost useless and leads us to the
complication of the `_hook_methods` dict.
This keeps compatibility for the caller (also
part of this package since Heptapod 0.12), allowing
us to refactor in a separate step.
parent cac20dfe000d
No related branches found
No related tags found
1 merge request!20Calling GitLab hooks through API
......@@ -4,4 +4,5 @@
# GNU General Public License version 2 or any later version.
#
# SPDX-License-Identifier: GPL-2.0-or-later
import logging
import os
......@@ -7,5 +8,8 @@
import os
import subprocess
import requests
import requests_unixsocket
logger = logging.getLogger(__name__)
def git_hook_format_changes(changes):
......@@ -20,6 +24,74 @@
for ref, (old_gitsha, new_gitsha) in changes.items())
def parse_broadcast_msg(msg, text_length):
"""Cut into several lines of at most text_length length """
if len(msg) <= text_length:
return [msg]
lines = []
current = []
current_len = 0
for word in msg.split():
if current_len + len(word) < text_length:
current.append(word)
current_len += len(word) + 1
else:
lines.append(' '.join(current))
current = [word]
current_len = 0
if current:
lines.append(' '.join(current))
return lines
def format_alert_message(message):
"""Directly translated from gitlab_post_receive.rb in gitlab-shell."""
# A standard terminal window is (at least) 80 characters wide.
total_width = 80
# Git prefixes remote messages with "remote: ", so this width is subtracted
# from the width available to us. TODO recheck for Mercurial
total_width -= len("remote: ")
# Our centered text shouldn't start or end right at the edge of the window,
# so we add some horizontal padding: 2 chars on either side.
text_width = total_width - 2 * 2
# Automatically wrap message at text_width (= 68) characters:
# Splits the message up into the longest possible chunks matching
# "<between 0 and text_width characters><space or end-of-line>".
lines = ["", "=" * total_width, ""]
for msg_line in parse_broadcast_msg(message, text_width):
padding = max((total_width - len(msg_line)) / 2, 0)
lines.append(" " * padding + msg_line)
lines.extend(("", "=" * total_width, ""))
return lines
def format_warnings(warnings):
return format_alert_message("\n".join(("WARNINGS", warnings)))
def format_mr_links(mrs):
if not mrs:
return ()
out_lines = [""]
for mr in mrs:
if mr.get("new_merge_request"):
fmt = "To create a merge request for {branch_name}, visit:"
else:
fmt = "View merge request for {branch_name}:"
fmt += "\n {url}"
out_lines.append(fmt.format(**mr))
print(repr(out_lines))
return out_lines
SKIP = object()
......@@ -34,7 +106,11 @@
at the right stage of Mercurial transaction end.
"""
_hook_methods = {
'post-receive': 'post_receive',
}
def __init__(self, name, repo):
self.name = name
self.repo = repo
self.git_fs_path = self.repo.root[:-3] + '.git'
......@@ -37,9 +113,14 @@
def __init__(self, name, repo):
self.name = name
self.repo = repo
self.git_fs_path = self.repo.root[:-3] + '.git'
url = str(repo.ui.config(b'heptapod', b'gitlab-internal-api-url',
'http://localhost:8080'))
self.gitlab_internal_api_endpoint = url + '/api/v4/internal'
self.gitlab_url = url
shell = repo.ui.config(b'heptapod', b'gitlab-shell')
if not shell:
raise RuntimeError("Path to GitLab Shell is unknown")
self.gitlab_shell = shell
......@@ -41,8 +122,17 @@
shell = repo.ui.config(b'heptapod', b'gitlab-shell')
if not shell:
raise RuntimeError("Path to GitLab Shell is unknown")
self.gitlab_shell = shell
secret = repo.ui.config(b'heptapod',
b'gitlab-internal-api-secret-file')
if not secret:
raise RuntimeError("heptapod.gitlab-internal-api-secret-file "
"config parameter is missing.")
with open(secret, 'r') as secretf:
self.gitlab_secret = secretf.read().strip()
def __str__(self):
return "GitLab %r hook wrapper" % self.name
......@@ -72,6 +162,52 @@
env['GL_REPOSITORY'] = 'project-' + project_id
return env
def gitlab_internal_api_request(
self, meth,
subpath, params, **kwargs): # pragma: no cover
params['secret_token'] = self.gitlab_secret
with requests_unixsocket.monkeypatch():
return requests.request(
meth,
'/'.join((self.gitlab_internal_api_endpoint,
subpath)),
data=params,
**kwargs)
def post_receive_params(self, changes, env=None):
if env is None:
env = self.environ()
return dict(gl_repository=env['GL_REPOSITORY'],
identifier=env['GL_ID'],
changes=git_hook_format_changes(changes))
def post_receive(self, changes, env=None):
"""Call GitLab internal API post-receive endpoint."""
params = self.post_receive_params(changes, env=env)
resp = self.gitlab_internal_api_request('POST',
subpath='post_receive',
params=params)
ok = resp.status_code == 200
as_json = resp.json()
logger.debug("post_receive for changes %r, response: %r",
changes, as_json)
return ok, self.post_receive_handle_response(as_json), ''
def post_receive_handle_response(self, as_json):
out_lines = []
broadcast_message = as_json.get('broadcast_message')
if broadcast_message is not None:
out_lines.extend(format_alert_message(broadcast_message))
out_lines.extend(format_mr_links(as_json.get('merge_request_urls')))
for msg_name in ('redirected_message', 'project_created_message', ):
msg = as_json.get(msg_name)
if msg is not None:
out_lines.append(msg)
warnings = as_json.get('warnings')
if warnings is not None:
out_lines.extend(format_warnings(warnings))
return '\n'.join(out_lines)
def __call__(self, changes):
"""Call GitLab Hook for the given changes.
......@@ -95,13 +231,9 @@
self.repo.ui.note('%s: bailing (explicitely told not to send)')
return 0, '', ''
self.repo.ui.note("%s: sending changes %r" % (self, changes))
pr = subprocess.Popen(
[os.path.join(self.gitlab_shell, 'hooks', self.name)],
env=env,
cwd=self.git_fs_path,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE)
out, err = pr.communicate(input=git_hook_format_changes(changes))
return pr.returncode, out, err
meth_name = self._hook_methods.get(self.name)
if meth_name is None:
raise NotImplementedError(self.name)
ok, out, err = getattr(self, meth_name)(changes, env=env)
return 0 if ok else 1, out, err
......@@ -4,4 +4,6 @@
# GNU General Public License version 2 or any later version.
#
# SPDX-License-Identifier: GPL-2.0-or-later
import io
import json
import pytest
......@@ -7,5 +9,7 @@
import pytest
import requests
from heptapod.testhelpers import (
LocalRepoWrapper,
)
from .. import (
......@@ -8,5 +12,8 @@
from heptapod.testhelpers import (
LocalRepoWrapper,
)
from .. import (
format_alert_message,
format_warnings,
format_mr_links,
git_hook_format_changes,
......@@ -12,4 +19,5 @@
git_hook_format_changes,
parse_broadcast_msg,
Hook,
)
......@@ -26,4 +34,82 @@
}
def test_parse_broadcast_msg():
msg = "Short message"
assert parse_broadcast_msg(msg, 80) == [msg]
msg = "A somewhat longer message, cut here"
assert parse_broadcast_msg(msg, 30) == ["A somewhat longer message,",
"cut here"]
def test_format_alert_message():
# here the point of the test is mostly to make sure that we don't
# get stupid errors, we don't care that much of the exact amount of
# padding or leading and trailing separators
msg = "Short message"
assert (format_alert_message(msg)[3]
==
' Short message'
)
msg = ' '.join('word%d' % i for i in range(20))
formatted = format_alert_message(msg)
assert (formatted[3]
==
(' word0 word1 word2 word3 word4 word5 word6 word7 word8 '
'word9 word10')
)
assert (formatted[4]
==
(' word11 word12 word13 word14 word15 '
'word16 word17 word18 word19')
)
def test_format_warnings():
"""Test format_warnings for non breakage, not for precise content wording.
"""
assert 'WARNINGS' in format_warnings('Short warning')[3]
def fmt_mr_links(mrs):
"""Call and convert return into a tuple.
The goal is that assertions don't depend onto the precise return type.
"""
return tuple(format_mr_links(mrs))
def test_format_mr_links():
"""Test format_mr_links for non breakage, not for precise content wording.
"""
assert fmt_mr_links(None) == ()
assert fmt_mr_links([]) == ()
assert fmt_mr_links(()) == ()
seen_create = False
seen_view = False
for line in fmt_mr_links(
[dict(new_merge_request=True,
branch_name='heptapod-next',
url="https://heptapod.example?query=something"),
dict(new_merge_request=False,
branch_name='heptapod-stable',
url="https://heptapod.example?query=other"),
]):
line = line.lower()
if 'view merge request' in line:
seen_view = True
assert "heptapod-stable" in line
# query string passed through
assert '?query=other' in line
elif 'create a merge request' in line:
seen_create = True
assert "heptapod-next" in line
assert '?query=something' in line
assert seen_view, "Did not get message to view existing MR"
assert seen_create, "Did not get message to create new MR"
def make_repo(base_dir, name):
......@@ -29,3 +115,5 @@
def make_repo(base_dir, name):
secret = base_dir.join('secret')
secret.write("TEHTOKEN")
wrapper = LocalRepoWrapper.init(
base_dir.join(name),
......@@ -30,7 +118,9 @@
wrapper = LocalRepoWrapper.init(
base_dir.join(name),
config=dict(heptapod={'gitlab-shell': str(base_dir)}),
)
config=dict(heptapod={'gitlab-shell': str(base_dir),
'gitlab-internal-api-secret-file': str(secret),
}),
)
# by default, ui.environ IS os.environ (shared) but that isn't true
# in WSGI contexts. In that case, it is a plain dict, the WSGI environment
# indeed. For these tests, we need them to be independent.
......@@ -38,6 +128,36 @@
return wrapper
def test_hook(tmpdir):
def request_recorder(records, responses):
def request(hook, meth, subpath, params, **kwargs):
records.append((meth, subpath, params, kwargs))
return responses.pop(0)
return request
def test_post_receive_params(tmpdir):
wrapper = make_repo(tmpdir, 'proj.hg')
repo = wrapper.repo
hook = Hook('post-receive', repo)
repo.ui.environ.update(
HEPTAPOD_USERINFO_ID='23',
HEPTAPOD_PROJECT_ID='1024',
)
changes = {
'branch/newbr': (ZERO_SHA, "beef1234dead" + "a" * 8),
'topic/default/zz': ("23cafe45" + "0" * 12, "1234" + "b" * 16),
}
params = hook.post_receive_params(changes)
assert params['identifier'] == 'user-23'
assert params['changes'].splitlines() == [
'23cafe45000000000000 1234bbbbbbbbbbbbbbbb topic/default/zz',
'00000000000000000000 beef1234deadaaaaaaaa branch/newbr']
assert params['gl_repository'] == 'project-1024'
def test_post_receive_handle_response(tmpdir):
wrapper = make_repo(tmpdir, 'proj.hg')
repo = wrapper.repo
......@@ -42,9 +162,82 @@
wrapper = make_repo(tmpdir, 'proj.hg')
repo = wrapper.repo
hook = Hook('post-something', repo)
hook = Hook('post-receive', repo)
out = hook.post_receive_handle_response(
# example taken from the functional tests
{u'broadcast_message': None,
u'reference_counter_decreased': True,
u'merge_request_urls': [
{u'url': u'http://localhost:3000/root/'
'test_project_1583330760_9216309/merge_requests/1',
u'branch_name': u'topic/default/antelope',
u'new_merge_request': False}
]})
out = [l.strip() for l in out.splitlines() if l.strip()]
assert out == [
'View merge request for topic/default/antelope:',
'http://localhost:3000/root/test_project_1583330760_9216309/'
'merge_requests/1']
broadcast = "Check out the new tutorial"
out = hook.post_receive_handle_response(
{u'broadcast_message': broadcast,
u'reference_counter_decreased': True,
})
out = (l.strip() for l in out.splitlines() if l.strip())
assert any(broadcast in l for l in out)
project_created = "Congrats!"
out = hook.post_receive_handle_response(
{u'project_created_message': project_created,
u'reference_counter_decreased': True,
})
out = (l.strip() for l in out.splitlines() if l.strip())
assert any(project_created in l for l in out)
# GitLab internal API gives warnings as a unique string
# see lib/api/helpers/internal_helpers.rb
warnings = "You probably shouldn't have"
out = hook.post_receive_handle_response(
{u'warnings': warnings,
u'reference_counter_decreased': True,
})
out = (l.strip() for l in out.splitlines() if l.strip())
assert any(warnings in l for l in out)
def make_response(status_code, body):
"""Body is bytes"""
resp = requests.models.Response()
resp.status_code = status_code
resp.raw = io.BytesIO(body)
return resp
def make_json_response(obj, status_code=200):
return make_response(status_code, json.dumps(obj).encode())
def test_integration_post_receive(tmpdir, monkeypatch):
records = []
resp = make_json_response(
{u'broadcast_message': None,
u'reference_counter_decreased': True,
u'merge_request_urls': [
{u'url': u'http://localhost:3000/root/'
'test_project_1583330760_9216309/merge_requests/1',
u'branch_name': u'topic/default/antelope',
u'new_merge_request': False}
]})
responses = [resp]
monkeypatch.setattr(Hook, 'gitlab_internal_api_request',
request_recorder(records, responses))
repo_wrapper = make_repo(tmpdir, 'repo.hg')
repo = repo_wrapper.repo
repo.ui.environ.update(
HEPTAPOD_USERINFO_ID='23',
HEPTAPOD_PROJECT_ID='1024',
)
......@@ -45,13 +238,12 @@
repo.ui.environ.update(
HEPTAPOD_USERINFO_ID='23',
HEPTAPOD_PROJECT_ID='1024',
)
assert str(hook) == "GitLab 'post-something' hook wrapper"
env = hook.environ()
assert env.get('HG_GIT_SYNC') == 'yes'
assert env.get('GL_ID') == 'user-23'
assert env.get('GL_REPOSITORY') == 'project-1024'
hook = Hook('post-receive', repo_wrapper.repo)
changes = {
'branch/newbr': (ZERO_SHA, "beef1234dead" + "a" * 8),
'topic/default/zz': ("23cafe45" + "0" * 12, "1234" + "b" * 16),
}
......@@ -57,4 +249,9 @@
# we didn't pollute original environment
assert 'GL_ID' not in repo.ui.environ
return_code, out, err = hook(changes)
assert return_code == 0
out = [l.strip() for l in out.splitlines() if l.strip()]
assert out == [
'View merge request for topic/default/antelope:',
'http://localhost:3000/root/test_project_1583330760_9216309/'
'merge_requests/1']
......@@ -60,8 +257,2 @@
# calling with empty changes
assert hook(()) == (0, '', '')
# now with an executable
# we need a pseudo Git repo
tmpdir.join('proj.git').ensure(dir=True)
......@@ -67,15 +258,13 @@
hook_bin = tmpdir.ensure('hooks', dir=True).join('post-something')
hook_bin.write('\n'.join((
"#!/usr/bin/env python",
"import sys",
"sys.stderr.write('stderr of the post something hook')",
"for line in sys.stdin.readlines():",
" sys.stdout.write(line)",
"sys.exit(3)",
)))
hook_bin.chmod(0o700)
changes = {'branch/newbr': (ZERO_SHA, "beef1234dead" + "a" * 8),
'topic/default/zz': ("23cafe45" + "0" * 12, "1234" + "b" * 16),
}
def test_empty_changes(tmpdir):
hook = Hook('post-receive', make_repo(tmpdir, 'repo.hg').repo)
code, out, err = hook(())
assert code == 0
assert not out
assert not err
def test_unknown_hook(tmpdir):
# that's a typo, the most likely way for it to happen
name = 'post-recieve'
......@@ -81,8 +270,8 @@
code, out, err = hook(changes)
assert code, err == (3, 'stderr of the post something hoko')
assert set(out.splitlines()) == {
'00000000000000000000 beef1234deadaaaaaaaa branch/newbr',
'23cafe45000000000000 1234bbbbbbbbbbbbbbbb topic/default/zz'
repo = make_repo(tmpdir, 'repo.hg').repo
hook = Hook(name, repo)
changes = {
'branch/newbr': (ZERO_SHA, "beef1234dead" + "a" * 8),
'topic/default/zz': ("23cafe45" + "0" * 12, "1234" + "b" * 16),
}
......@@ -87,5 +276,13 @@
}
repo.ui.environ.update(
HEPTAPOD_USERINFO_ID='23',
HEPTAPOD_PROJECT_ID='1024',
)
with pytest.raises(NotImplementedError) as exc_info:
hook(changes)
assert exc_info.value.args[0] == name
def test_missing_gitlab_shell(tmpdir):
repo = LocalRepoWrapper.init(tmpdir).repo
......@@ -89,4 +286,6 @@
def test_missing_gitlab_shell(tmpdir):
repo = LocalRepoWrapper.init(tmpdir).repo
repo.ui.setconfig(b'heptapod', b'gitlab-internal-api-secret-file',
b'/some/path')
with pytest.raises(RuntimeError) as exc_info:
......@@ -92,5 +291,5 @@
with pytest.raises(RuntimeError) as exc_info:
Hook('post-something', repo)
Hook('post-receive', repo)
assert "Path to GitLab Shell" in exc_info.value.args[0]
......@@ -94,6 +293,14 @@
assert "Path to GitLab Shell" in exc_info.value.args[0]
def test_missing_secret(tmpdir):
repo = LocalRepoWrapper.init(tmpdir).repo
repo.ui.setconfig(b'heptapod', b'gitlab-shell', b'/some/path')
with pytest.raises(RuntimeError) as exc_info:
Hook('post-receive', repo)
assert "secret-file" in exc_info.value.args[0]
def test_hook_missing_user(tmpdir):
wrapper = make_repo(tmpdir, 'proj2.hg')
repo = wrapper.repo
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment