Read about our upcoming Code of Conduct on this issue

Commit 3868b21e authored by Georges Racinet's avatar Georges Racinet 🦑
Browse files

heptapod PAAS runner: variable expansion for dependency proxy

Not doing it means that image specifications such as `$BASE_IMAGE/project`
are all considered to come from Docker Hub, whereas this is typically
a pattern used not to harcode the local registry name (encouraged in
GitLab CI/CD doc beside being common sense).

The expansion is recursive (we don't expect it to reach Python recursion
limits), which is different from the unsupported nested syntax. There
are know projects in the wild using a cascade of variables for image
definition, HGitaly being our prime example of that.

We're actually performing the expansion only to decide whethere
prefixing should occur or not, the end result is still based on
the incoming pattern. This has the advantage of being less sensitive
to potential divergences with GitLab Runner's interpolation.

--HG--
branch : heptapod
parent 393afebe2c55
......@@ -9,7 +9,10 @@ import logging
import pathlib
import subprocess
from .job import get_job_variable
from .job import (
all_job_variables,
expand_variables,
)
logger = logging.getLogger(__file__)
......@@ -24,8 +27,9 @@ DOCKERFILE = (
)
def is_from_docker_hub(image_tag):
host = image_tag.rsplit(':', 1)[0].split('/', 1)[0]
def is_from_docker_hub(image_tag, job_vars):
expanded = expand_variables(image_tag, job_vars)
host = expanded.rsplit(':', 1)[0].split('/', 1)[0]
# TODO check that '.' is indeed forbidden in an image name or plainly
# check the official Docker pull implementation.
return '.' not in host
......@@ -84,8 +88,10 @@ class DockerBuildHelper:
Reference: https://docs.gitlab.com/ee/user/packages/dependency_proxy/#authenticate-within-cicd
""" # noqa long URL
job_id = job_data['id']
proxy_url = get_job_variable(job_data,
'CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX')
# TODO replace job_data by a Job class, and perform lazy
# once-and-for-all parsing of variables.
job_vars = all_job_variables(job_data)
proxy_url = job_vars.get('CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX')
if not proxy_url:
logger.debug("Job %d: no dependency proxy available",
job_id)
......@@ -94,7 +100,7 @@ class DockerBuildHelper:
main_image = job_data.get('image')
if main_image is not None:
main_image_name = main_image['name']
if is_from_docker_hub(main_image['name']):
if is_from_docker_hub(main_image_name, job_vars):
main_image['name'] = '/'.join((proxy_url, main_image_name))
services = job_data.get('services')
......@@ -113,7 +119,7 @@ class DockerBuildHelper:
logger.warning("Job %d: did not understand image/tag "
"in service definition %r",
job_id, service)
elif is_from_docker_hub(srv_image_name):
elif is_from_docker_hub(srv_image_name, job_vars):
service['name'] = '/'.join((proxy_url, srv_image_name))
new_services.append(service)
......
......@@ -118,9 +118,16 @@ def expand_variables(s, job_vars):
'somethin-}-hop'
>>> expand_variables('somethin-${FOO-${BAR}}-hop', {'FOO-${BAR': "foo"})
'somethin-foo}-hop'
Logically nested (but not syntactically)::
>>> expand_variables('somethin-${FOO}-hop',
... {'FOO': "foo${BAR}", 'BAR': "bar"})
'somethin-foobar-hop'
"""
def repl(match):
# perhaps a more clever regexp would prevent the need to strip braces
return job_vars.get(match.group(1).strip('{}'), '')
return expand_variables(job_vars.get(match.group(1).strip('{}'), ''),
job_vars)
return VAR_SUBST_RX.sub(repl, s)
......@@ -18,12 +18,21 @@ from ..testing import (
def test_is_from_docker_hub():
assert not any(is_from_docker_hub(image) for image in (
assert not any(is_from_docker_hub(image, job_vars={}) for image in (
'registry.heptapod.net/heptapod/omnibus-heptapod:0-24-testing',
'registry.heptapod.net:443/heptapod/omnibus-heptapod:0-24-testing',
'registry.heptapod.net:443/heptapod/omnibus-heptapod',
)
)
assert not any(is_from_docker_hub('$MY_VAR/group/proj$TAG',
job_vars=dict(MY_VAR=myvar,
BASE='registry.example',
TAG=tag))
for myvar in ('registry.heptapod.net',
'$BASE:1443',
'registry.heptapod.net:443')
for tag in ('', ':testing')
)
def test_docker_git(tmpdir):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment