Commit d6ed1361 authored by Georges Racinet's avatar Georges Racinet 🌳
Browse files

PAAS Runner: Clever Runner

Like the official CLI "Clever Tools" does, we need to
request the current version of the `docker` instance type
to create a new one. This should be later on optimized.

Like the `LocalDockerRunner`, the `launch()` method creates
a Docker build context, but deployment now means to push
it via Git.

In practice, this means that we have a foreign (non Python)
dependency: the Git executable.

We should at least later on make its path customizable and
make a startup error if missing.

--HG--
branch : heptapod
parent cce95f6f51a6
# imports to trigger registration of Runner classes
from . import ( # noqa F401
clever,
local_docker,
)
# Copyright 2021 Georges Racinet <georges.racinet@octobus.net>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
#
# SPDX-License-Identifier: GPL-3.0-or-later
from overrides import overrides
import requests
import tempfile
from .docker import DockerBuildHelper
from .exceptions import (
PaasProvisioningError,
PaasResourceError,
)
from .runner import PaasRunner
class CleverCloudDockerRunner(PaasRunner):
executor = 'clever-docker'
def __init__(self, config):
super(CleverCloudDockerRunner, self).__init__(config)
self.init_cc_global()
self.init_cc_orga()
self.init_cc_headers()
self.init_cc_git()
self.init_common_app_params()
def init_cc_global(self):
self.cc_base_url = self.config.get('cc_api_url',
'https://api.clever-cloud.com/v2')
self.cc_zone = self.config.get('cc_zone', 'par')
def init_cc_orga(self):
self.cc_orga_id = self.config['cc_orga_id']
self.cc_token = self.config['cc_token']
self.cc_app_create_url = '/'.join((self.cc_base_url,
'organisations',
self.cc_orga_id,
'applications'
))
def cc_app_url(self, app_id):
return '/'.join((self.cc_base_url,
'organisations',
self.cc_orga_id,
'applications',
app_id,
))
def init_cc_headers(self):
self.cc_headers = {
'Content-Type': 'application/json',
'Authorization': 'token ' + self.cc_token,
}
def init_cc_git(self):
self.cc_git_user = self.config.get('cc_auth_user', 'Jenkins')
def init_common_app_params(self):
self.common_app_params = {
# TODO could depend on job rather than runner
'zone': self.cc_zone,
'instanceLifetime': 'TASK',
'instanceType': 'docker',
# TODO clever-tools does request the correct value for
# instanceVersion ffrom GET /products/instances
# we need to do the same, cache it, and update if outdated
'instanceVersion': '20210629',
'deploy': 'git',
'maxInstances': 1, # we'll do our own auto scaling
'minInstances': 1,
}
def inner_executor(self):
return 'docker'
def is_config_item_for_inner(self, key):
return not key.startswith('cc_')
def convert_http_error(self, http_response):
"""Return error code, message."""
try:
resp_data = http_response.json()
assert resp_data['type'] == 'error'
# checked with Clever, `id` is definitely an error code
return resp_data['id'], resp_data['message']
except Exception:
return None, http_response.text
def cc_docker_instance_type(self):
resp = requests.get(self.cc_base_url + '/products/instances')
if resp.status_code >= 400:
error_code, message = self.convert_http_error(resp)
raise PaasProvisioningError(action='get-instance-type',
transport_code=resp.status_code,
code=error_code,
error_details=message,
executor=self.executor)
assert resp.status_code == 200
docker_types = [inst for inst in resp.json()
if inst['name'] == 'Docker']
# TODO what if several match? Are, e.g., several versions
# possible at a given time?
return docker_types[0]
@overrides
def provision(self, job):
# TODO cache instance_type, and reload only if provision ends
# with errors possibly due to outdated information (including change
# of flavor availability)
# (error code for unknown application should be 4004)
instance_type = self.cc_docker_instance_type()
req_data = self.common_app_params.copy()
req_data['instanceVersion'] = instance_type['version']
# TODO flavor should be flexible (perhaps job-dependent)
# TODO flavor should be compared with those listed in instance_type
req_data['minFlavor'] = 'XS'
req_data['maxFlavor'] = 'XS'
req_data['name'] = 'hpd-job-%s-%d' % (self.unique_name, job['id'])
resp = requests.post(self.cc_app_create_url,
headers=self.cc_headers,
json=req_data)
if resp.status_code >= 400:
# TODO catch error due to outdated version to recompute it
# and raise something appropriate for other errors.
# Actual error got due to lack of version is currently in use in
# test_clever.py for easy reference.
error_code, message = self.convert_http_error(resp)
raise PaasProvisioningError(action='create-app',
action_details=req_data,
code=error_code,
transport_code=resp.status_code,
error_details=message,
executor=self.executor,
)
resp_data = resp.json()
app_id = resp_data['id']
self.cc_put_env(app_id, {'CC_MOUNT_DOCKER_SOCKET': 'true',
'CC_TROUBLESHOOT': 'true'})
return CleverCloudApplication(app_id,
resp_data['deployment']['httpUrl'],
self.cc_git_user,
self.cc_token)
def cc_put_env(self, app_id, env):
resp = requests.put(self.cc_app_url(app_id) + '/env',
json=env,
headers=self.cc_headers)
# success code should be 328
if resp.status_code >= 400:
error_code, message = self.convert_http_error(resp)
raise PaasResourceError(self.executor, app_id,
action='put-env',
action_details=env,
code=error_code,
transport_code=resp.status_code,
error_details=message)
return resp
@overrides
def launch(self, paas_resource, job_json):
with tempfile.TemporaryDirectory() as tmp_path:
build_helper = DockerBuildHelper(tmp_path)
build_helper.write_build_context(self, job_json)
build_helper.git_push(paas_resource.deploy_url)
CleverCloudDockerRunner.register()
class CleverCloudApplication:
def __init__(self, app_id, deploy_url, user, password):
self.app_id = app_id
# TODO use proper URL parsing, of course
self.deploy_url = deploy_url.replace(
'https://',
'https://%s:%s@' % (user, password))
# Copyright 2021 Georges Racinet <georges.racinet@octobus.net>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
#
# SPDX-License-Identifier: GPL-3.0-or-later
import json
import pytest
import requests
import toml
from ..clever import (
CleverCloudDockerRunner,
CleverCloudApplication,
)
from ..docker import DockerBuildHelper
from ..exceptions import (
PaasProvisioningError,
PaasResourceError,
)
from ..testing import (
COORDINATOR_URL,
request_recorder,
make_json_response,
make_response,
)
HEPTAPOD_TOKEN = 'Cl3vr1234'
CC_ORGA_ID = 'orga_some_uuid'
CC_TOKEN = 'clever-orga-deploying-token'
parametrize = pytest.mark.parametrize
API_ERROR_KINDS = ['parseable', 'unparseable', 'standard-fields']
@pytest.fixture
def runner():
yield CleverCloudDockerRunner(dict(executor='clever-docker',
token=HEPTAPOD_TOKEN,
cc_orga_id=CC_ORGA_ID,
cc_token=CC_TOKEN,
url=COORDINATOR_URL))
def test_config_methods(tmpdir, runner):
dump_path = tmpdir / 'runner.toml'
runner.dump_inner_config(dump_path)
dumped = toml.loads(dump_path.read())
# Clever Cloud creds must not be propagated
assert dumped == dict(runners=[dict(executor='docker',
url=COORDINATOR_URL,
token=HEPTAPOD_TOKEN,
)
])
def test_provision_launch(runner, monkeypatch):
# testing first runner.provision()
records = []
responses = [
# instance type query
make_json_response([dict(name='UnrelatedApplication',
version='20220202'),
dict(name='Docker',
version='20210101'),
]),
# app creation
make_json_response(
dict(id='some-uuid',
deployment=dict(httpUrl='https://clever.test/app-git-repo'),
)),
# environment variable setting TODO QUESTION Clever:
# don't know actual expected status code, might be 204
make_json_response(dict(), status_code=200),
]
monkeypatch.setattr(requests.api, 'request',
request_recorder(records, responses))
job = dict(id=777)
resource = runner.provision(job)
assert isinstance(resource, CleverCloudApplication)
assert resource.app_id == 'some-uuid'
assert len(records) == 3
provision_req = records[1]
meth, url, prov_kwargs = provision_req
assert meth.lower() == 'post'
assert url == ('https://api.clever-cloud.com/v2'
'/organisations/orga_some_uuid/applications')
prov_data = prov_kwargs['json']
assert prov_data['name'] == 'hpd-job-Cl3vr123-777'
assert prov_data['instanceVersion'] == '20210101'
assert prov_data['zone'] == 'par'
# now testing runner.launch()
git_push_records = [] # list mostly for easy mutability
def fake_git_push(helper, url):
git_push_records.append(url)
monkeypatch.setattr(DockerBuildHelper, 'git_push', fake_git_push)
runner.launch(resource, json.dumps(job))
assert git_push_records == [
'https://Jenkins:clever-orga-deploying-token@clever.test/app-git-repo',
]
@parametrize('response_kind', API_ERROR_KINDS)
def test_provision_error(runner, monkeypatch, response_kind):
if response_kind == 'standard-fields':
# this is an actual error got due to missing version in early
# experiments, so that's maybe one we might later not to trigger
# a raise. Meanwhile it is kept here for further reference.
error_response = make_json_response(
dict(
id=9017,
message='The specified instance does not exist',
type='error',
),
status_code=400)
elif response_kind == 'parseable':
error_response = make_json_response(dict(msg="not your day"),
status_code=404)
else:
error_response = make_response(500, b'not even JSON')
records = []
responses = [
# instance type query
make_json_response([dict(name='Docker',
version='20210101'),
]),
# app creation
error_response,
]
monkeypatch.setattr(requests.api, 'request',
request_recorder(records, responses))
with pytest.raises(PaasProvisioningError) as exc_info:
runner.provision(dict(id=123))
exc = exc_info.value
assert exc.executor == 'clever-docker'
assert exc.action == 'create-app'
if response_kind == 'standard-fields':
assert exc.transport_code == 400
# can change once we get more info on Clever API
assert exc.code == 9017
assert exc.error_details == 'The specified instance does not exist'
# just asserting it's the req_data dict:
assert exc.action_details['instanceType'] == 'docker'
elif response_kind == 'parseable':
assert exc.transport_code == 404
assert exc.error_details == json.dumps(dict(msg='not your day'))
else:
assert exc.transport_code == 500
assert exc.error_details == "not even JSON"
@parametrize('response_kind', API_ERROR_KINDS)
def test_cc_docker_instance_type_error(runner, monkeypatch, response_kind):
records = []
if response_kind == 'standard-fields':
error_response = make_json_response(
dict(id=3857,
type='error',
message='getting instances failed'),
status_code=400)
elif response_kind == 'parseable':
error_response = make_json_response(
dict(msg='Unexpected error payload'),
status_code=400)
else:
error_response = make_response(500, b'not even JSON')
monkeypatch.setattr(requests.api, 'request',
request_recorder(records, [error_response]))
with pytest.raises(PaasProvisioningError) as exc_info:
runner.cc_docker_instance_type()
exc = exc_info.value
if response_kind == 'standard-fields':
assert exc.transport_code == 400
assert exc.code == 3857
assert exc.error_details == 'getting instances failed'
elif response_kind == 'parseable':
assert exc.transport_code == 400
assert exc.error_details == json.dumps(
dict(msg='Unexpected error payload'))
else:
assert exc.transport_code == 500
assert exc.error_details == "not even JSON"
@parametrize('response_kind', API_ERROR_KINDS)
def test_cc_put_env_error(runner, monkeypatch, response_kind):
records = []
if response_kind == 'standard-fields':
error_response = make_json_response(
dict(id=273,
type='error',
message='putting env failed'),
status_code=400)
elif response_kind == 'parseable':
error_response = make_json_response(
dict(msg='I said no'),
status_code=400)
else:
error_response = make_response(500, b'not even JSON')
monkeypatch.setattr(requests.api, 'request',
request_recorder(records, [error_response]))
with pytest.raises(PaasResourceError) as exc_info:
runner.cc_put_env('clever-app-uuid', {'CC_MY_ENVIRON': 'something'})
exc = exc_info.value
if response_kind == 'standard-fields':
assert exc.code == 273
assert exc.transport_code == 400
assert exc.error_details == 'putting env failed'
elif response_kind == 'parseable':
assert exc.code is None
assert exc.transport_code == 400
assert exc.error_details == json.dumps(dict(msg='I said no'))
else:
assert exc.code is None
assert exc.transport_code == 500
assert exc.error_details == "not even JSON"
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment