Commit 4b031e16 authored by Georges Racinet's avatar Georges Racinet 🦑
Browse files

Tests and CI: merge from heptapod-stable

The difference between the CI base image for default and stable
are so minimal (both Heptapod specific and upstream GitLab) that
I hope this can pass right away.

--HG--
branch : heptapod
Pipeline #14356 skipped with stages
......@@ -68,9 +68,7 @@
policy: pull
.use-pg11:
# TODO minor base image change between v13.1 and v13.3, so we should
# build a new image and make Docker tags '0.15', '0.16' etc.
image: "registry.heptapod.net/heptapod/ci-images/heptapod-base"
image: "registry.heptapod.net/heptapod/ci-images/heptapod-base:default"
services:
- name: postgres:11.6
command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
......
......@@ -1352,7 +1352,7 @@ test:
default:
path: tmp/tests/repositories/
gitaly_address: unix:tmp/tests/gitaly/gitaly.socket
hgitaly_address: unix:tmp/tests/gitaly/hgitaly.socket
hgitaly_address: unix:tmp/tests/hgitaly/hgitaly.socket
gitaly:
client_path: tmp/tests/gitaly
......
......@@ -6,6 +6,7 @@
# to upstream, but we are just at the beginning of the CI process for Heptapod.
stages:
- prepare
- test
- upload
......@@ -44,20 +45,54 @@ include:
# Merge Requests (see `.shared:rules:update-cache`)
policy: pull-push
# inspired from rails.gitlab-ci.yml (setup-test-env notably)
# this plays the role of the setup-test-env and compile-test-assets jobs
# of upstream GitLab. The main difference is that it is "colder" a single
# tarball per GitLab version. Even with frontend assets, the differences
# we can introduce on a given GitLab version don't matter for RSpec tests.
# It will be a different story when we launch functional tests ("qa").
gitlab-prebuilds:
rules:
- if: '$UPDATE_GITLAB_PREBUILDS'
extends:
- .use-pg11 # base image and services
- .default-before_script # shell functions and some environment vars
- .heptapod-rails-cache
stage: prepare
variables:
GITLAB_TEST_EAGER_LOAD: "0"
script:
- rm -rf tmp/tests/gitaly # stale files from previous builds can prevent proper installation
- run_timed_command "bundle exec ruby -I. -e 'require \"config/environment\"; TestEnv.init'"
- run_timed_command "scripts/gitaly-test-build" # Do not use 'bundle exec' here
- rm tmp/tests/gitaly/.ruby-bundle # This file prevents gems from being installed even if vendor/gitaly-ruby is missing
# frontend assets production, copied from compile-assets-base.
- node --version
- run_timed_command "retry yarn install --frozen-lockfile"
- free -m
- run_timed_command "bin/rake gitlab:assets:compile > assets-compile.log 2>&1"
- run_timed_command "scripts/clean-old-cached-assets"
# end of frontent assets production
- run_timed_command "scripts/hgitaly-test-build"
- scripts/heptapod-make-push-gitlab-prebuilds.sh
artifacts:
paths:
- assets-compile.log
when: always
rspec:
rules:
- changes:
- lib
- app/models
- heptapod-ci.yml
- .gitlab/ci
allow_failure: false
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
allow_failure: false
- if: '$CI_PIPELINE_SOURCE != "merge_request_event"'
when: manual
allow_failure: true
# sharding (automatically sets the CI_NODE_{INDEX,TOTAL} expected by knapsack)
parallel: 6
variables:
# reconverted for rspec-retries (so this is a per-example retry, not
# retry of the full shard), including the silly missing +1
RETRIES: 6
extends:
- .use-pg11 # base image and services
- .default-before_script # shell functions and some environment vars
......@@ -68,27 +103,19 @@ rspec:
- log/hgitaly-test.log
- config/gitlab.yml
script:
- rm -rf tmp/tests/gitaly # stale files from previous builds can prevent proper installation
- run_timed_command "bundle exec ruby -I. -e 'require \"config/environment\"; TestEnv.init'"
# copied from compile-assets-base. The day we want several jobs, this
# will be one prime candidate to do like upstream and just produce an
# artifact instead
- node --version
- run_timed_command "retry yarn install --frozen-lockfile"
- free -m
- run_timed_command "bin/rake gitlab:assets:compile > assets-compile.log 2>&1"
- run_timed_command "scripts/clean-old-cached-assets"
# end of assets production
- run_timed_command "python3 scripts/prepare_build_py_heptapod.py"
# TODO put it in cellar and benefit from site-local caches
- wget --progress=dot:giga -O /tmp/gitlab-prebuilds.tar.gz https://download.heptapod.net/gitlab-prebuilds/ci-$(cat VERSION).tar.gz
- tar xzf /tmp/gitlab-prebuilds.tar.gz
- ls -al tmp/tests/*
- run_timed_command "scripts/gitaly-test-build" # Do not use 'bundle exec' here
- rm tmp/tests/gitaly/.ruby-bundle # This file prevents gems from being installed even if vendor/gitaly-ruby is missing
- run_timed_command "scripts/gitaly-test-spawn"
# python components already installed, but this will refresh them
# and update subsequent conf (gitlab.yml etc)
- run_timed_command "scripts/hgitaly-test-build"
- run_timed_command "scripts/hgitaly-test-spawn"
- grep -A3 'hgrc:' config/gitlab.yml
- cat $PWD/config/tests.hgrc
- cat $PWD/vendor/py-heptapod/heptapod/required.hgrc
- HGRCPATH=vendor/py-heptapod/heptapod/required.hgrc /usr/local/bin/hg hpd-versions; echo
- scripts/heptapod_rspec.sh --format d
- scripts/heptapod_knapsack.sh
tarball:
......@@ -106,6 +133,6 @@ tarball:
# this CI image may be overkill, but it gives us in particular a working
# recent Mercurial, and it should already be present in all runners
image: "registry.heptapod.net/heptapod/ci-images/heptapod-base"
image: "registry.heptapod.net/heptapod/ci-images/heptapod-base:default"
script:
- scripts/heptapod-make-push-tarball.sh
This directory holds knapsack reports.
knapsack is a system for balanced tests sharding.
It relies on pre-generated "report" files that list the
time needed for each test file, in order to achieve balancing.
These files are mandatory, even if one does not care so much
about balancing.
{
"spec/helpers/application_settings_helper_spec.rb": 3.420536,
"spec/helpers/vcs_type_helper_spec.rb": 0.015230,
"spec/lib/gitlab/build_access_spec.rb": 0.735082,
"spec/lib/gitlab/ci_access_spec.rb": 0.003541,
"spec/lib/gitlab/cross_project_access_spec.rb": 0.011619,
"spec/lib/gitlab/git/attributes_at_ref_parser_spec.rb": 0.761250,
"spec/lib/gitlab/git/attributes_parser_spec.rb": 0.038397,
"spec/lib/gitlab/git/base_error_spec.rb": 0.010658,
"spec/lib/gitlab/git/blame_spec.rb": 0.048718,
"spec/lib/gitlab/git/blob_spec.rb": 9.665832,
"spec/lib/gitlab/git/branch_spec.rb": 1.810208,
"spec/lib/gitlab/git/bundle_file_spec.rb": 0.002991,
"spec/lib/gitlab/git/changes_spec.rb": 0.016697,
"spec/lib/gitlab/git/commit_spec.rb": 1.059822,
"spec/lib/gitlab/git/compare_spec.rb": 0.322407,
"spec/lib/gitlab/git/cross_repo_comparer_spec.rb": 3.232121,
"spec/lib/gitlab/git/diff_collection_spec.rb": 0.576913,
"spec/lib/gitlab/git/diff_spec.rb": 0.573924,
"spec/lib/gitlab/git/diff_stats_collection_spec.rb": 0.008139,
"spec/lib/gitlab/git/gitmodules_parser_spec.rb": 0.001580,
"spec/lib/gitlab/git/hook_env_spec.rb": 0.123032,
"spec/lib/gitlab/git/keep_around_spec.rb": 1.182760,
"spec/lib/gitlab/git/lfs_changes_spec.rb": 0.057396,
"spec/lib/gitlab/git/lfs_pointer_file_spec.rb": 0.008269,
"spec/lib/gitlab/git/merge_base_spec.rb": 0.148415,
"spec/lib/gitlab/git/object_pool_spec.rb": 5.694974,
"spec/lib/gitlab/git/pre_receive_error_spec.rb": 0.035186,
"spec/lib/gitlab/git/push_spec.rb": 0.241563,
"spec/lib/gitlab/git/raw_diff_change_spec.rb": 0.008636,
"spec/lib/gitlab/git/remote_mirror_spec.rb": 0.481099,
"spec/lib/gitlab/git/remote_repository_spec.rb": 0.188688,
"spec/lib/gitlab/git/repository_cleaner_spec.rb": 0.373568,
"spec/lib/gitlab/git/repository_spec.rb": 22.324063,
"spec/lib/gitlab/git/tag_spec.rb": 2.704617,
"spec/lib/gitlab/git/tree_spec.rb": 0.494723,
"spec/lib/gitlab/git/user_spec.rb": 0.028075,
"spec/lib/gitlab/git/util_spec.rb": 0.006868,
"spec/lib/gitlab/git/wiki_spec.rb": 7.294624,
"spec/lib/gitlab/git/wraps_gitaly_errors_spec.rb": 0.006764,
"spec/lib/gitlab/git_access_design_spec.rb": 0.014600,
"spec/lib/gitlab/git_access_project_spec.rb": 1.972361,
"spec/lib/gitlab/git_access_snippet_spec.rb": 16.528636,
"spec/lib/gitlab/git_access_spec.rb": 99.720437,
"spec/lib/gitlab/git_access_wiki_spec.rb": 0.390284,
"spec/lib/gitlab/hg_access_main_spec.rb": 153.706134,
"spec/lib/gitlab/hg_access_protected_branches_spec.rb": 291.131276,
"spec/lib/gitlab/mercurial/hg_git_repository_spec.rb": 85.332174,
"spec/lib/gitlab/mercurial/hgitaly_client_spec.rb": 3.881960,
"spec/lib/gitlab/mercurial/hgitaly_repository_spec.rb": 222.594542,
"spec/lib/gitlab/user_access_snippet_spec.rb": 1.558270,
"spec/lib/gitlab/user_access_spec.rb": 19.428001,
"spec/lib/gitlab_spec.rb": 0.109080,
"spec/mailers/emails/pipelines_spec.rb": 3.756551,
"spec/models/ci/job_artifact_spec.rb": 51.806456,
"spec/models/ci/job_variable_spec.rb": 2.216094,
"spec/models/ci/pipeline_artifact_spec.rb": 2.861438,
"spec/models/ci/pipeline_config_spec.rb": 0.014650,
"spec/models/ci/pipeline_message_spec.rb": 0.980447,
"spec/models/ci/pipeline_schedule_spec.rb": 7.407743,
"spec/models/ci/pipeline_schedule_variable_spec.rb": 1.165911,
"spec/models/ci/pipeline_spec.rb": 103.112268,
"spec/models/ci/pipeline_variable_spec.rb": 1.725846,
"spec/models/commit_collection_spec.rb": 5.450583,
"spec/models/commit_range_spec.rb": 4.765426,
"spec/models/commit_status_spec.rb": 9.822903,
"spec/models/commit_with_pipeline_spec.rb": 2.371279,
"spec/models/compare_spec.rb": 3.393206,
"spec/models/merge_request_assignee_spec.rb": 2.425250,
"spec/models/merge_request_context_commit_diff_file_spec.rb": 0.020940,
"spec/models/merge_request_context_commit_spec.rb": 1.020070,
"spec/models/merge_request_diff_commit_spec.rb": 15.875874,
"spec/models/merge_request_diff_file_spec.rb": 14.699201,
"spec/models/merge_request_diff_spec.rb": 92.419155,
"spec/models/merge_request_reviewer_spec.rb": 1.099232,
"spec/models/project_wiki_spec.rb": 16.675641,
"spec/models/user_spec.rb": 113.056185,
"spec/models/wiki_directory_spec.rb": 0.708761,
"spec/models/wiki_page_spec.rb": 44.254186,
"spec/models/wiki_spec.rb": 0.010347,
"spec/services/projects/create_service_spec.rb": 105.052845,
"spec/services/projects/update_remote_mirror_service_spec.rb": 7.628911
}
#!/bin/sh
set -e
# it's nicer for console output to add the extension right away
TARBALL=ci-$(cat VERSION).tar.gz
TARBALL_PATH=/tmp/${TARBALL}
echo "Generating ${TARBALL}"
# same paths as artifacts in upstream GitLab `setup-test-env` job and
# `compile-test-assets`, plus tmp/tests/hgitaly, minus build logs kept
# as artifacts
tar zcf ${TARBALL_PATH} \
config/secrets.yml \
public/assets \
node_modules/@gitlab/svgs/dist/icons.json \
tmp/tests/gitaly \
tmp/tests/hgitaly \
tmp/tests/gitlab-shell \
tmp/tests/gitlab-test-fork \
tmp/tests/gitlab-test-fork_bare \
tmp/tests/gitlab-test \
tmp/tests/gitlab-workhorse \
tmp/tests/repositories \
tmp/tests/second_storage
if [ -n "$HEPTAPOD_TARBALL_UPLOAD_KEY" ]; then
`dirname $0`/heptapod-sftp-push \
download.heptapod.net \
"$HEPTAPOD_TARBALL_UPLOAD_KEY" \
"${TARBALL_PATH}" \
"public/gitlab-prebuilds/"
fi
......@@ -25,36 +25,9 @@ hg archive -t tgz ${TARBALL_PATH}
sha256sum ${TARBALL_PATH} > ${TARBALL_PATH}.sha256sum
if [ -n "$HEPTAPOD_TARBALL_UPLOAD_KEY" ]; then
HEPTAPOD_DOWNLOAD_SITE=download.heptapod.net
echo "Pushing ${TARBALL} to ${HEPTAPOD_DOWNLOAD_SITE} via SFTP"
# TODO better to ship a known_hosts file
#
# StricHostKeyChecking=accept-new not available in base Docker image
#
# ssh_config(5) on Debian 9 (OpenSSH 7.4):
# If this flag is set to no, ssh will automatically add new host keys
# to the user known hosts files.
#
# ssh_config(5) on Debian 10 (OpenSSH 7.9):
# If this flag is set to “accept-new” then ssh will automatically add
# new host keys to the user known hosts files, but will not permit
# connections to hosts with changed host keys. If this flag is set
# to “no” or “off”, ssh will automatically add new host keys
# to the user known hosts files and allow connections to hosts
# with changed hostkeys to proceed, subject to some restrictions.
SSH_FLAGS="-o IdentitiesOnly=yes -o StrictHostKeyChecking=no"
if [ -n "$HEPTAPOD_UPLOAD_SSH_PORT" ]; then
SSH_FLAGS="$SSH_FLAGS -P $HEPTAPOD_UPLOAD_SSH_PORT"
fi
# TODO it would be nice of the runner to set restrictive permissions
# this right at file creation time (perhaps based on a declarative option),
# especially since there's no job filtering except based on environments
chmod 600 ${HEPTAPOD_TARBALL_UPLOAD_KEY}
echo "put ${TARBALL_PATH}*" | sftp -b- \
${SSH_FLAGS} \
-i ${HEPTAPOD_TARBALL_UPLOAD_KEY} \
${HEPTAPOD_UPLOAD_SSH_USER}@${HEPTAPOD_DOWNLOAD_SITE}:${HEPTAPOD_UPLOAD_SSH_USER}/${SUBDIR}/src
`dirname $0`/heptapod-sftp-push \
download.heptapod.net \
"$HEPTAPOD_TARBALL_UPLOAD_KEY" \
"$TARBALL_PATH" \
"${SUBDIR}/src"
fi
#!/bin/sh
# Simple script to push any artifact to some Heptapod SFTP server,
# using the heptapod_known_hosts.ssh file
set -eu
SFTP_HOST=$1
# This "key" is just a file name, content is set by file mode of CI variables
HEPTAPOD_UPLOAD_KEY=$2
ARTIFACT=$3
TARGET_DIR=$4
echo "Pushing ${ARTIFACT} to ${SFTP_HOST} via SFTP"
KNOWN_HOSTS=$(realpath $(dirname $0)/heptapod_known_hosts.ssh)
SSH_FLAGS="-o IdentitiesOnly=yes -o UserKnownHostsFile=${KNOWN_HOSTS}"
if [ -n "$HEPTAPOD_UPLOAD_SSH_PORT" ]; then
SSH_FLAGS="$SSH_FLAGS -P $HEPTAPOD_UPLOAD_SSH_PORT"
fi
# TODO it would be nice of the runner to set restrictive permissions
# this right at file creation time (perhaps based on a declarative option),
# especially since there's no job filtering except based on environments
chmod 600 ${HEPTAPOD_UPLOAD_KEY}
# wildcard to include any checksum or signatures
echo "put ${ARTIFACT}*" | sftp -b- \
${SSH_FLAGS} \
-i ${HEPTAPOD_UPLOAD_KEY} \
${HEPTAPOD_UPLOAD_SSH_USER}@${SFTP_HOST}:${HEPTAPOD_UPLOAD_SSH_USER}/${TARGET_DIR}
#!/bin/sh
# this script has to be run from the root of the Rails application
# set these environment variables to something relevant for your run
#export CI_NODE_TOTAL=2
#export CI_NODE_INDEX=0
export KNAPSACK_TEST_FILE_PATTERN="spec/{lib/gitlab,lib/gitlab/mercurial/*,lib/gitlab/*_access*,lib/gitlab/git/*,mailers/emails/pipelines,models/user,models/merge_request_*,models/compare,models/commit_*,models/ci/job_*,models/ci/pipeline*,models/*wiki*,services/projects/update_remote_mirror_service,services/projects/create_service,helpers/application_settings_helper,helpers/vcs_type_helper}_spec.rb"
export KNAPSACK_REPORT_PATH="knapsack/rspec.json"
knapsack rspec "-Ispec --color --format d"
[download.heptapod.net]:7022 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC5HYl4+LHEY9n4+3JhXPLHLwj+55H3lik/cp91AlXpQmmIzVRcFlQEqzMjxFI1kfNOd9Y/k7lZjyF86AVmZ6+e726J4i8rCwyMtATkdM3acRNdtONFVcC9a5CvT71H6U0OFCOS8hVCv/pBs/FoYwC+foxmMltZp2kpmhhqRQyuCTLoFQUdYYaFysXRtoo3Yy5nO8Road1M1fIib0e+PB2ALzJsgKjTiINk7sWXF2AfC7Phw8kOkYIL285eBftoh6LQ/oKshYXs1AULe/gaFgbhfQ9H2M5xiR+ddqkVNAlV1nJLYtRvCdGavYzX3F/hKXQjRS9TDmiinspK9CnaGje9
[download.heptapod.net]:7022 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFhzij3QcQHkTZ/d5pia53FPBKAw6uo1MugXtppURwqYd1Wlk/yEoFP4VfamWpZvLW5anB2SDh0PrkPv8NYDBnM=
[download.heptapod.net]:7022 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGQYhX/90l97jZgG33PXfobTUoetWwuBdAMDS0rSaU3v
#!/bin/sh
# This script has to be run from the root of the Rails application
# Generally speaking, one should prefer heptapod_knapsack.sh (for
# sharded parallel execution)
# use this to generate the knapsack report (times of individual test files)
#export KNAPSACK_GENERATE_REPORT=true
export KNAPSACK_REPORT_PATH="knapsack/rspec.json"
bin/rspec \
spec/lib/gitlab_spec.rb \
spec/lib/gitlab/mercurial_spec.rb \
......
#!/usr/bin/env python3
"""Clone and prepare py_heptapod for CI build of the Rails applications.
Expected to be run from a clone of the Rails application, and assumes
that `config/gitlab.yml` has already been initialized.
In the prebuild phase, the relevant that it does is to prepare the
clones. The installation in the interpreter will be lost and redone in
the actual testing phase. (TODO add a flag to skip it, although it's
probably negligible, given than most deps are in the base image)
"""
import os
import re
import subprocess
import sys
INTERPRETER = 'python3.8'
HG_BIN = '/usr/local/bin/hg'
RELEASE_BRANCH_RX = re.compile(r'heptapod-\d+-\d+')
CWD = os.getcwd()
BASE_DIR = os.path.join(CWD, 'tmp', 'tests', 'hgitaly')
# lazy, but I'm tired of reimplenting `mkdir -p` over and over
subprocess.check_call(['mkdir', '-p', BASE_DIR])
PY_HEPTAPOD_CLONE_PATH = os.path.join(BASE_DIR, 'py-heptapod')
HGITALY_CLONE_PATH = os.path.join(BASE_DIR, 'hgitaly')
def write_config(req_hgrc_path):
local_hgrc_path = os.path.join(CWD, 'config', 'tests.hgrc')
with open(local_hgrc_path, 'w') as local_hgrcf:
local_hgrcf.write("\n".join((
"[heptapod]",
"# we won't really send the pre- and post-receive hooks, ",
"# but `heptapod.hooks.Hook` is currently still instantiated and ",
"# that requires the configuration.",
"gitlab-internal-api-secret-file = " + os.path.join(
CWD, '.gitlab_shell_secret'),
"")))
yml_format = 'hgrc:\n - %s\n - %s\n bin_path: %s'
with open('config/gitlab.yml', 'r+') as conf:
out_lines = [line.replace('# hgrc:',
yml_format % (local_hgrc_path,
req_hgrc_path,
HG_BIN))
for line in conf.readlines()]
conf.seek(0)
conf.writelines(out_lines)
CI_TAG = os.environ.get('CI_COMMIT_TAG')
# probably not set for tag pipelines:
CI_BRANCH = os.environ.get('CI_COMMIT_HG_BRANCH', '')
if CI_TAG or RELEASE_BRANCH_RX.match(CI_BRANCH):
print("*** ATTENTION Pipeline running for a tag or a release branch ***")
print("*** ==> installing the fixed `python/requirements.txt` ***")
# just install the fixed deps, that's what is meant
subprocess.check_call((INTEPRETER, '-m', 'pip', 'install', '--upgrade',
'-r', 'python/requirements.txt'))
hepta_mod = os.fsdecode(
subprocess.check_output(
(INTERPRETER, '-c', 'import heptapod; print(heptapod.__file__)'),
cwd='/')
)
write_config(os.path.join(os.path.dirname(hepta_mod), 'required.hgrc'))
sys.exit(0)
elif CI_BRANCH == 'heptapod-stable':
PYDEPS_BRANCH = 'stable'
else:
PYDEPS_BRANCH = 'default'
HG_CMD = (HG_BIN,
'--config', 'extensions.evolve=',
'--config', 'extensions.topic=',
)
PYTHON_COMPONENTS = {
'py-heptapod': PY_HEPTAPOD_CLONE_PATH,
'hgitaly': HGITALY_CLONE_PATH,
}
# yes, Mercurial is readily available in GitLab base CI image!
# It is installed in the Python 3.8 interpreter
for slug, target in PYTHON_COMPONENTS.items():
if os.path.exists(target):
print("Updating %s clone at %s" % (slug, target))
subprocess.check_call(HG_CMD + ('-R', target, 'pull', '-q'))
subprocess.check_call(HG_CMD + ('-R', target, 'update', '-q',
'-C', PYDEPS_BRANCH))
else:
print("Cloning %s into %s" % (slug, target))
subprocess.check_call(HG_CMD + (
'clone', '-q',
'https://foss.heptapod.net/heptapod/' + slug,
target,
))
print("Current revision for %s:" % target)
subprocess.check_call(HG_CMD + ('-R', target, 'log', '-r', '.'))
pip_cmd = [INTERPRETER, '-m', 'pip', 'install', '--upgrade']
for target in PYTHON_COMPONENTS.values():
pip_cmd.extend(('-e', target))
print("Installing Python Heptapod components: %r" % pip_cmd)
subprocess.check_call(pip_cmd, stderr=sys.stdout)
write_config(os.path.join(PY_HEPTAPOD_CLONE_PATH,
'heptapod', 'required.hgrc'))
#!/bin/sh
# This script is used only in CI
set -eu
# CI base image has Mercurial, installed system-wide by pip3.8
# We need the specific exact version, because even python3 is 3.5.
# as this can change, let's derive it from the hashbang.
HG=/usr/local/bin/hg
INTERPRETER=$(head -1 $HG | cut -b 3-)
HEPTAPOD_PKG_PATH=$($INTERPRETER -c 'import heptapod,os; print(os.path.dirname(heptapod.__file__))')
export HGRCPATH=config/tests.hgrc:${HEPTAPOD_PKG_PATH}/required.hgrc
echo "Using ${HEPTAPOD_PKG_PATH}/required.hgrc"
echo "Content:"
cat ${HEPTAPOD_PKG_PATH}/required.hgrc
echo "Calling hpd-versions"
$HG hpd-versions
echo
echo "Starting HGitaly server"
set -x
$HG --config extensions.hgitaly= \
hgitaly-serve \
--repositories-root $PWD/tmp/tests/repositories \
--listen unix:tmp/tests/hgitaly/hgitaly.socket &
#!/usr/bin/env python3
"""Clone and prepare py_heptapod for CI build of the Rails applications.
Expected to be run from a clone of the Rails application, and assumes
that `config/gitlab.yml` has already been initialized.
"""
import os
import subprocess
INTERPRETER = 'python3.8'
CWD = os.getcwd()
PY_HEPTAPOD_CLONE_PATH = os.path.join(CWD, 'vendor', 'py-heptapod')
HGITALY_CLONE_PATH = os.path.join(CWD, 'vendor', 'hgitaly')
HG_BIN = '/usr/local/bin/hg'
PIP_BIN = 'pip3.8'
print("Upgrading pip and setuptools")
subprocess.check_call((PIP_BIN, 'install', '--upgrade',
'pip', 'setuptools', 'wheel'))
print("Upgrading hg-evolve")
subprocess.check_call((PIP_BIN, 'install', '--upgrade', 'hg-evolve'))
HG_CMD = (HG_BIN,
'--config', 'extensions.evolve=',
'--config', 'extensions.topic=',
)
# yes, Mercurial is readily available in GitLab base CI image!
# It is installed in the Python 3.8 interpreter
for slug, target in (('py-heptapod', PY_HEPTAPOD_CLONE_PATH),
('hgitaly', HGITALY_CLONE_PATH)):
if os.path.exists(target):
print("Updating %s clone at %s" % (slug, target))
subprocess.check_call(HG_CMD + ('-R', target, 'pull'))
subprocess.check_call(HG_CMD + ('-R', target, 'update', '-C', 'default'))
else:
print("Cloning %s into %s" % (slug, target))
subprocess.check_call(HG_CMD + (
'clone', '-q',
'https://foss.octobus.net/heptapod/' + slug,
target,
))
print("Current revision for %s:" % target)
subprocess.check_call(HG_CMD + ('-R', target, 'log', '-r', '.'))
pip_cmd = (PIP_BIN, 'install', '-e', target)
print("Installing %s: %r" % (slug, pip_cmd))
subprocess.check_call(pip_cmd)
pip_logging_cmd = (PIP_BIN, 'install', 'hg-loggingmod')
print("Installing hg-loggingmod: " + repr(pip_logging_cmd))
subprocess.check_call(pip_logging_cmd)
local_hgrc_path = os.path.join(CWD, 'config', 'tests.hgrc')
with open(local_hgrc_path, 'w') as local_hgrcf:
local_hgrcf.write("\n".join((
"[heptapod]",
"# we won't really send the pre- and post-receive hooks, ",
"# but `heptapod.hooks.Hook` is currently still instantiated and ",
"# that requires the configuration.",
"gitlab-internal-api-secret-file = " + os.path.join(CWD, '.gitlab_shell_secret'),
"")))
req_hgrc_path = os.path.join(PY_HEPTAPOD_CLONE_PATH,
'heptapod', 'required.hgrc')
yml_format = 'hgrc:\n - %s\n - %s\n bin_path: %s'
with open('config/gitlab.yml', 'r+') as conf:
out_lines = [l.replace('# hgrc:',
yml_format % (local_hgrc_path,
req_hgrc_path,
HG_BIN))
for l in conf.readlines()]
conf.seek(0)
conf.writelines(out_lines)
......@@ -9,7 +9,7 @@
let(:user) { create(:user) }
let(:actor) { user }
let(:project) { create(:hg_git_project, :hg_repository) }
let(:project) { create(:hg_git_project, :empty_repo) }