Commit 0ce0070b authored by Georges Racinet's avatar Georges Racinet 🦑
Browse files

Merged OmnibusHeptapod and full CI from stable branch

Pipeline #11095 failed with stage
in 38 minutes and 30 seconds
......@@ -3,7 +3,7 @@ variables:
stages:
- lint
- test
- test-heptapod
flake8:
stage: lint
......@@ -11,8 +11,8 @@ flake8:
script:
- flake8 tests conftest.py
"functional-tests":
stage: test
docker-remote:
stage: test-heptapod
services:
- name: octobus/heptapod:$TARGET_HEPTAPOD_IMAGE
alias: heptapod
......@@ -23,7 +23,27 @@ flake8:
- apt-get update
- apt-get -y install tox openssh-client
- pip install --user hg-evolve>=9.3.1
- ./ci-run-tests
- ci/run-tests
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- when: manual # default case
docker-inside:
stage: test-heptapod
image: octobus/heptapod:$TARGET_HEPTAPOD_IMAGE
script:
- apt-get update
- apt-get -y install sudo
- visudo --file=ci/omnibus.sudoers -c
- install -m 0440 -o root -g root ci/omnibus.sudoers /etc/sudoers.d/omnibus
- apt-get -y install tox chromium-driver
- ci/run-tests-inside || ci/fail-copy-logs
artifacts:
when: on_failure
paths:
- log/gitlab-rails/production_json.log
- log/hgserve/mercurial.log
- log/hgserve/gunicorn.log
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- when: manual # default case
......
#!/bin/sh
# copy logs inside project so that they can be exposed
# as artifacts
mkdir -p log
cp /var/log/gitlab/*.log log/
# forward the failure
exit 1
\ No newline at end of file
git ALL=(ALL) NOPASSWD:/opt/gitlab/bin/gitlab-ctl
#!/bin/sh
set -e
# creating data and starting the application
# (we'll provide logs for failed jobs in another way, the general tailing
# that the wrapper script does is really too much clutter for the job log).
/assets/wrapper >/dev/null 2>&1 &
COMMON_FLAGS="-v \
--heptapod-url http://localhost:80 \
--heptapod-ssh-port 22 \
--heptapod-ssh-user git \
--heptapod-reverse-call-host=localhost \
--heptapod-omnibus"
chown -R git:git .
GIT_HOME=`ls -d ~git` # for explicit guaranteed forwarding
chown git:git $GIT_HOME
TOX="sudo -u git HOME=$GIT_HOME tox -e omnibus"
INITIAL_WAIT=180
for MARKER in fs_access services; do
echo "Running functional tests marked with $MARKER"
if $TOX -- -m $MARKER $COMMON_FLAGS \
--heptapod-wait-after-first-response $INITIAL_WAIT; then
echo "First pass for $MARKER succeeded"
else
echo "Rerunning failed tests of first pass for $MARKER"
$TOX -- -m $MARKER --lf $COMMON_FLAGS
fi
INITIAL_WAIT=0
done
......@@ -12,6 +12,7 @@ from tests.utils.group import Group
from tests.utils.user import User
from tests.utils.heptapod import (
Heptapod,
OmnibusHeptapod,
DockerHeptapod,
SourceHeptapod,
GdkHeptapod,
......@@ -37,6 +38,9 @@ def pytest_addoption(parser):
"and filesystem manipulations."
"This mode needs rbenv or equivalent to be active."
)
parser.addoption('--heptapod-omnibus', action="store_true",
help="Test a Heptapod Omnibus by running the tests "
"as root on the same system ")
parser.addoption('--heptapod-remote', action='store_true',
help="Test a remote server. This means that no "
"direct command nor file system access is possible "
......@@ -106,7 +110,8 @@ def pytest_collection_modifyitems(config, items):
remote = config.getoption('heptapod_remote')
source_install = config.getoption('heptapod_source_install')
gdk = config.getoption('heptapod_gdk')
docker = not (gdk or source_install or remote)
omnibus = config.getoption('heptapod_omnibus')
docker = not (gdk or source_install or remote or omnibus)
for item in items:
if remote and "fs_access" in item.keywords:
......@@ -162,6 +167,8 @@ def heptapod(pytestconfig):
**common)
elif pytestconfig.getoption('heptapod_remote'):
heptapod = Heptapod(**common)
elif pytestconfig.getoption('heptapod_omnibus'):
heptapod = OmnibusHeptapod(**common)
else:
ct = pytestconfig.getoption('heptapod_docker_container')
heptapod = DockerHeptapod(docker_container=ct, **common)
......
......@@ -14,6 +14,7 @@ from .utils.project import (
ProjectAccess,
)
from .utils.runner import Runner
from .utils.selenium import could_click_element
parametrize = pytest.mark.parametrize
......@@ -136,11 +137,15 @@ def rebase(repo, topic):
def rebase_publish_push(repo, topic):
"""Rebase the topic created with `prepare_topic()`, publish and push.
Return full Node ID of the topic head after rebase and before publication
"""
rebase(repo, topic)
sha = repo.node(topic)
repo.hg('push', '--publish')
print("Graph after push/publish:")
print(repo.graphlog())
return sha
def assert_merge_detected(project, mr_id, named_branch=None, timeout=10):
......@@ -215,6 +220,25 @@ def api_accept_merge_request(proj, mr_id,
return mr
def assert_commit_link(project, mr_id, expected_text, expected_sha):
"""Return commit link element whose text contained expected, or None.
"""
for li in project.webdriver_get_merge_request_commits(mr_id):
if expected_text in li.text:
# finer assertions about commit hash widget
copy_label = li.find_element_by_xpath(
# div with class 'label', in a div having a button child
'//div[button]/div[contains(@class, "label")]')
copy_button = li.find_element_by_xpath('.//button')
assert copy_button.get_attribute(
'data-clipboard-text') == expected_sha
assert expected_sha.startswith(copy_label.text)
return
else:
raise AssertionError(
"Could not find a commit row with text %r" % expected_text)
def test_mergerequest_rebase_publish(test_project, tmpdir):
"""
When a topic is rebased and pushed with the --publish flag,
......@@ -226,15 +250,11 @@ def test_mergerequest_rebase_publish(test_project, tmpdir):
default_url=test_project.owner_basic_auth_url)
topic = prepare_topic(repo)
mr_id = create_mr(test_project, topic=topic)
rebase_publish_push(repo, topic)
topic_sha = rebase_publish_push(repo, topic)
# and merge has been detected
# merge has been detected and displays the right commit
assert_merge_detected(test_project, mr_id)
expected = "Même une antilope !"
commit_links = test_project.webdriver_get_merge_request_commits(mr_id)
assert any(expected in l.text for l in commit_links), (
"Commit summaries don't contain %r" % expected)
assert_commit_link(test_project, mr_id, "Même une antilope !", topic_sha)
def test_mergerequest_topic_different_branch_publish(test_project, tmpdir):
......@@ -322,7 +342,7 @@ def test_mergerequest_api_stacked_publish_first(test_project, tmpdir):
msg="Expected update of second MR didn't happen after 10s")
commit_links = test_project.webdriver_get_merge_request_commits(mr2_id)
assert len(commit_links) == 1
assert commit_links[0].text == 'food: horse eats grass'
assert 'food: horse eats grass' in commit_links[0].text
def test_mergerequest_api_stacked_publish_second(test_project, tmpdir):
......@@ -1091,3 +1111,65 @@ def test_mergerequest_api_obsolete_multiple_heads(public_project, tmpdir):
expected = "Même une antilope !"
assert any(expected in l.text for l in commit_links), (
"Commit summaries don't contain %r" % expected)
@parametrize('change_meth', ('api_change', 'webui_change'))
@parametrize('merge_meth', ('api_merge', 'rebase_push'))
def test_mergerequest_change_source(
test_project, tmpdir, change_meth, merge_meth):
"""Test for changing source GitLab branch of a MR (heptapod#138)
"""
repo = hg.LocalRepo.init(tmpdir.join('repo'),
default_url=test_project.owner_basic_auth_url)
topic = prepare_topic(repo)
mr_id = create_mr(test_project, topic=topic)
new_topic = 'cloven'
repo.hg('topic', new_topic)
# changing title to be easily sure the MR refresh worked well
repo.hg('amend', '-m', "all cloven-hoofed")
repo.hg('push', '-r', '.')
# we don't want to reopen in concurrency with the async job that closes it
test_project.wait_assert_merge_request(
mr_id, lambda mr: mr.get('state') == 'closed')
new_source_branch = 'topic/default/' + new_topic
edit_payload = dict(state_event='reopen')
if change_meth == 'api_change':
edit_payload['source_branch'] = new_source_branch
mr = test_project.api_update_merge_request(mr_id, **edit_payload)
if change_meth == 'webui_change':
webdriver = test_project.owner_webdriver
webdriver.get('%s/-/merge_requests/%d/edit' % (test_project.url,
mr_id))
wait = WebDriverWait(webdriver, 10)
wait.until(could_click_element(
lambda d: d.find_element_by_css_selector(
'.js-source-branch-select')
))
wait.until(could_click_element(
lambda d: d.find_element_by_xpath(
'//div[contains(text(), "%s")'
' and @role="option"]' % new_source_branch
)))
wait.until(could_click_element(
lambda d: d.find_element_by_xpath(
'//input[@value="Save changes"]'
)))
mr = test_project.api_get_merge_request(mr_id)
assert mr.get('source_branch') == new_source_branch
expected = "cloven-hoofed"
commit_links = test_project.webdriver_get_merge_request_commits(mr_id)
assert any(expected in l.text for l in commit_links), (
"Commit summaries don't contain %r" % expected)
if merge_meth == 'rebase_push':
rebase_publish_push(repo, new_topic)
else:
api_accept_merge_request(test_project, mr_id, check_merged=False)
# and merge has been detected
assert_merge_detected(test_project, mr_id)
......@@ -61,7 +61,8 @@ def test_owner_push_pull(test_project, tmpdir):
git_ssh_cmd, git_ssh_address = test_project.git_ssh_params(
test_project.owner)
git_clone = tmpdir.join('git_ssh_clone')
git = subprocess.Popen(['git', 'clone', git_ssh_address, str(git_clone)],
git = subprocess.Popen([test_project.heptapod.git_executable,
'clone', git_ssh_address, str(git_clone)],
env={'GIT_SSH_COMMAND': git_ssh_cmd},
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
......
......@@ -10,6 +10,7 @@ import shutil
import subprocess
import sys
import tarfile
import tempfile
import time
from urllib.parse import urlparse
......@@ -62,6 +63,10 @@ class Heptapod:
"""Time to wait after we've had a first non error HTTP response.
"""
git_executable = 'git'
chrome_driver_args = ()
def __init__(self, url, ssh_user, ssh_port,
reverse_call_host=None,
wait_after_first_response=0,
......@@ -106,6 +111,17 @@ class Heptapod:
def basic_user_token_headers(self):
return {'Private-Token': self.users['test_basic']['token']}
def run_shell(self, command, **kw):
exit_code, output = self.execute(command, **kw)
if exit_code != 0:
raise RuntimeError(
('Heptapod command {command} returned a non-zero '
'exit code {exit_code}').format(
command=command,
exit_code=exit_code,
))
return output
def get_user(self, name):
"""Return a :class:`User` instance, or `None`."""
info = self.users.get(name)
......@@ -127,6 +143,8 @@ class Heptapod:
def new_webdriver(self):
options = selenium.webdriver.ChromeOptions()
for arg in self.chrome_driver_args:
options.add_argument(arg)
options.add_argument('--headless')
if self.webdriver_remote_url:
return selenium.webdriver.Remote(
......@@ -357,8 +375,8 @@ class Heptapod:
return
self.close_webdrivers()
def run_shell(self, command, **kw):
raise NotImplementedError('run_shell')
def execute(self, command, **kw):
raise NotImplementedError('execute')
def force_remove_route(self, route_path, source_type='Project'):
logger.error("Attempt to force-remove route %r, not implemented "
......@@ -448,7 +466,7 @@ class Heptapod:
assert self.settings['hashed_storage_enabled'] is hashed_storage
class DockerHeptapod(Heptapod):
class OmnibusHeptapod(Heptapod):
fs_access = True
......@@ -456,8 +474,84 @@ class DockerHeptapod(Heptapod):
repositories_root = '/var/opt/gitlab/git-data/repositories'
backups_dir = '/var/opt/gitlab/backups'
RAILS_SERVICES = ('puma', 'sidekiq')
gitlab_ctl_command = ('sudo', 'gitlab-ctl')
reverse_call_host = 'localhost'
ssh_url = 'ssh://git@localhost'
git_executable = '/opt/gitlab/embedded/bin/git'
chrome_driver_args = ('--no-sandbox', )
def execute(self, command, user='git'):
if user != 'git':
raise NotImplementedError(
"On Omnibus Heptapod, only 'git' user is allowed")
logger.debug("OmnibusHeptapod: executing command %r", command)
process = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = [o.decode() for o in process.communicate()]
retcode = process.poll()
if out:
sys.stdout.write(out)
if err:
sys.stderr.write(err)
return retcode, out
def put_archive(self, dest, path, owner='git'):
# this assumes owner has write access to parent directory,
# but that should be enough, and avoids some processing as root
self.run_shell(['tar', '-xf', path, '-C', dest], user=owner)
def put_archive_bin(self, dest, fobj):
with tempfile.NamedTemporaryFile() as tempf:
tempf.write(fobj.read())
self.put_archive(dest, tempf.name)
def put_file_lines(self, path, lines, uid=998, gid=998):
with open(path, 'w') as fobj:
fobj.writelines(lines)
os.chown(path, uid, gid)
def get_file_lines(self, path):
with open(path) as fobj:
return fobj.readlines()
def remove_all_backups(self):
# using find in order not to rely on shell expansion for *.tar
self.run_shell(('find', '/var/opt/gitlab/backups',
'-name', '*.tar',
'-delete'))
def rake(self, *args):
cmd = ['gitlab-rake']
cmd.extend(args)
code, out = self.execute(cmd, user='git')
return out.encode() # Consistency with HDK (returns bytes)
def gitlab_ctl(self, command, services=None):
base_cmd = self.gitlab_ctl_command + (command, )
if services is None:
self.run_shell(base_cmd)
else:
for service in services:
self.run_shell(base_cmd + (service, ))
class DockerHeptapod(OmnibusHeptapod):
gitlab_ctl_command = ('gitlab-ctl', )
chrome_driver_args = ()
def __init__(self, docker_container, **kw):
super(DockerHeptapod, self).__init__(**kw)
self.docker_container = docker_container
......@@ -519,26 +613,6 @@ class DockerHeptapod(Heptapod):
" AND path='%s'" % (source_type, route_path)
])
def rake(self, *args):
cmd = ['gitlab-rake']
cmd.extend(args)
code, out = self.execute(cmd, user='git')
return out.encode() # Consistency with HDK (returns bytes)
def remove_all_backups(self):
# using find in order not to rely on shell expansion for *.tar
self.run_shell(('find', '/var/opt/gitlab/backups',
'-name', '*.tar',
'-delete'))
def gitlab_ctl(self, command, services=None):
base_cmd = ('gitlab-ctl', command)
if services is None:
self.run_shell(base_cmd)
else:
for service in services:
self.run_shell(base_cmd + (service, ))
class SourceHeptapod(Heptapod):
"""An Heptapod server installed from source on the same system.
......@@ -582,17 +656,6 @@ class SourceHeptapod(Heptapod):
sys.stderr.write(err)
return retcode, out
def run_shell(self, command, **kw):
exit_code, output = self.execute(command, **kw)
if exit_code != 0:
raise RuntimeError(
('Heptapod command {command} returned a non-zero '
'exit code {exit_code}').format(
command=command,
exit_code=exit_code,
))
return output
def put_archive(self, dest, path, owner='git'):
if owner != 'git':
raise NotImplementedError(
......
......@@ -10,10 +10,6 @@ import time
from urllib.parse import urlencode
from selenium.webdriver.common.by import By
from selenium.common.exceptions import (
ElementNotVisibleException,
ElementNotInteractableException,
)
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
......@@ -21,6 +17,7 @@ from . import wait_assert
from .constants import DATA_DIR
from .hg import LocalRepo
from .group import UserNameSpace
from .selenium import could_click_element
logger = logging.getLogger(__name__)
......@@ -34,22 +31,6 @@ class ProjectAccess:
OWNER = 50
def could_click_element(selector):
def try_click_element(driver):
try:
elem = selector(driver)
elem.click()
return True
except (ElementNotVisibleException,
ElementNotInteractableException):
return False
except Exception:
import traceback
traceback.print_exc()
return False
return try_click_element
def extract_gitlab_branch_titles(branches):
return {name: info['commit']['title']
for name, info in branches.items()}
......@@ -539,6 +520,12 @@ class Project(object):
with_rebase=True)
assert mr_info['merge_error'] is None
def api_update_merge_request(self, mr_iid, **data):
resp = self.owner_api_put(subpath='merge_requests/%d' % mr_iid,
data=data)
assert resp.status_code == 200 # this is synchronous
return resp.json()
def api_get_merge_request(self, mr_id, **params):
resp = self.owner_api_get(subpath='merge_requests/%s' % mr_id,
params=params)
......@@ -565,10 +552,12 @@ class Project(object):
webdriver = self.owner_webdriver
# this is the 'commits' pane, loading is dynamic, hence the wait
webdriver.get(self.url + '/merge_requests/%d/commits' % mr_id)
return WebDriverWait(webdriver, 10).until(
WebDriverWait(webdriver, 10).until(
EC.presence_of_all_elements_located(
(By.CSS_SELECTOR, 'a.commit-row-message')))
return webdriver.find_elements_by_css_selector('li.commit')
def api_create_deploy_token(self, name, scopes=('read_repository',)):
"""Will be available with GitLab 12.9."""
resp = self.owner_api_post(subpath='deploy_tokens',
......
"""Shared Selenium utils."""
import logging
import time
from selenium.common.exceptions import (
ElementNotVisibleException,
ElementNotInteractableException,
)
logger = logging.getLogger(__name__)
......@@ -29,3 +33,19 @@ def webdriver_wait_get(heptapod, driver,
url, driver.title, retry_delay)
time.sleep(retry_delay)
assert not heptapod.dead, dead_msg
def could_click_element(selector):
def try_click_element(driver):
try:
elem = selector(driver)
elem.click()
return True
except (ElementNotVisibleException,
ElementNotInteractableException):
return False
except Exception:
import traceback
traceback.print_exc()
return False
return try_click_element
......@@ -10,3 +10,6 @@ deps =
pytest-parallel
commands =
py.test {posargs}
[testenv:omnibus]
basepython = /opt/gitlab/embedded/bin/python3
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment