Skip to content
Snippets Groups Projects
Commit 26a2a246 authored by Raphaël Gomès's avatar Raphaël Gomès
Browse files

suites: add a setup script for the SWH git loader in a new `swh` suite

parent 64751f1b
No related branches found
No related tags found
1 merge request!44Suite agnostic tooling, improvements to simple-command, SWH suite
git-loader.poulpe-setup.py
\ No newline at end of file
#!/bin/env python
# WARNING: Do not run this script directly, use `poulpe bin-env-util setup-one`
#
# The script works using a hardcoded temporary directory
# in `/tmp/poulpe-clone-cache/swh/git-loader`
import hashlib
import json
import os
from pathlib import Path
import subprocess
import sys
from typing import Dict, List
import click
import poulpe
CLONE_PATH = "/tmp/poulpe-clone-cache/swh/swh-loader-git"
def run_command(command: List[str]):
environ = os.environ.copy()
environ["GIT_CONFIG_GLOBAL"] = "/dev/null"
environ["GIT_CONFIG_SYSTEM"] = "/dev/null"
completed = subprocess.run(command, capture_output=True, env=environ)
if completed.returncode != 0:
click.secho(completed.stderr, fg="red", file=sys.stderr)
completed.check_returncode()
return completed.stdout
def compute_unique_id(git_sha1, python_version, postgres_version):
hasher = hashlib.sha256()
# Version number to bust caches
hasher.update(b"0")
hasher.update(git_sha1.encode())
hasher.update(b"\x00")
hasher.update(python_version.encode())
hasher.update(b"\x00")
if postgres_version is not None:
hasher.update(postgres_version.encode())
return hasher.digest().hex()
def describe_parameters(ctx: click.Context, param, value):
if not value:
return
# TODO infer this from click context
print(
json.dumps(
{
"parameters": {
"loader-version": {
"doc": "an identifier of the swh-loader-git version we want to install",
"type": "<git-revision>",
"arg-mode": "CLI",
},
"target-repo": {
"default": "https://gitlab.softwareheritage.org/swh/devel/swh-loader-git",
"doc": "a path to the repo of swh-loader-git",
"type": "<url>",
"arg-mode": "CLI",
},
"python-version": {
"default": "python3",
"doc": "a path to the Python to use for the bin env",
"type": "<file-path>",
"arg-mode": "CLI",
},
}
}
)
)
ctx.exit(0)
def echo(message: str):
"""Wrapper around echo that is disabled if we're computing the unique id"""
click.echo(message)
def noop(message: str):
"""disabled echo when computing the unique id"""
# Explicitly don't set `no_args_is_help=True` because we want the script
# to return an error code if nothing is passed
@click.command()
@click.option(
"--describe-parameters",
default=False,
is_flag=True,
is_eager=True,
callback=describe_parameters,
)
@click.option(
"--get-unique-id",
default=False,
is_flag=True,
help="return the unique identifier derived from the all the parameters to this bin-env",
)
@click.option(
"--loader-version",
required=True,
help="an identifier of the swh-loader-git version we want to install",
)
@click.option(
"--target-repo",
help="a path to the repo of swh-loader-git",
default="https://gitlab.softwareheritage.org/swh/devel/swh-loader-git",
)
@click.option(
"--python-version",
default="python3",
help="a path to the Python to use for the bin env",
)
def git_loader_setup(
describe_parameters: bool,
get_unique_id: bool,
loader_version: str,
target_repo: str,
python_version: str,
):
if get_unique_id:
# Don't print anything if we're only getting the ID
global echo
echo = noop
echo(f"POULPE: using python version {python_version}")
echo(f"POULPE: using swh-git-loader version {loader_version}")
echo(f"POULPE: using loader url {target_repo}")
# Does it look like a clone? Don't bother to be more resistant for now
if Path(CLONE_PATH, ".git").is_dir():
# Do we already have the revision?
try:
run_command(["git", "-C", CLONE_PATH, "rev-parse", loader_version])
except subprocess.CalledProcessError:
# If not, fetch
echo(f"POULPE: failed to retrieve version, fetching")
run_command(["git", "-C", CLONE_PATH, "fetch", target_repo])
echo(f"POULPE: fetch done")
else:
echo(f"POULPE: expected version already in cache")
else:
echo(f"POULPE: cloning swh-git-loader from {target_repo}")
run_command(["git", "clone", target_repo, CLONE_PATH])
echo(f"POULPE: clone done")
git_rank = int(
(
run_command(
[
"git",
"-C",
CLONE_PATH,
"rev-list",
"--count",
loader_version,
]
)
.strip()
.decode()
)
)
git_sha1 = (
run_command(["git", "-C", CLONE_PATH, "rev-parse", loader_version])
.strip()
.decode()
)
unique_id = compute_unique_id(
git_sha1, python_version, postgres_version=None
)
if not get_unique_id:
# This means we've already got the unique ID, so make sure we match
name = Path().resolve().name
if name != unique_id:
raise poulpe.errors.BinEnvSetupFailure(
f"Bin-env '{name}' doesn't match expected ID '{unique_id}', "
f"environment changed during setup"
)
else:
click.echo(f"UNIQUE_ID: {unique_id}")
exit(0)
echo("POULPE: creating a new virtual environment")
run_command([python_version, "-m", "venv", "."])
echo("POULPE: installing swh-git-loader in the new venv")
run_command(
[
"bin/pip",
"install",
# install with no working copy for speed and making concurrent
# installs possible
f"git+file://{CLONE_PATH}@{git_sha1}",
"--require-virtualenv",
]
)
echo("POULPE: install OK")
output = run_command(
[
"bin/pip",
"list",
"--pre",
"--include-editable",
"--require-virtualenv",
"--format",
"json",
]
)
virtualenv_versions: Dict[str, str] = {
p["name"]: p["version"] for p in json.loads(output)
}
bin_env_vars = {
"python": {
"version": run_command(["bin/python", "--version"])
.decode()
.strip(),
"implementation": run_command(
[
"bin/python",
"-c",
"import sys; print(sys.implementation.name)",
]
)
.decode()
.strip(),
"dependencies": {},
},
"git": {
# kept as a useful shortcut, but redundant with `python.dependencies`
"version": virtualenv_versions["swh.loader.git"],
"rank": git_rank,
"sha1": git_sha1,
},
}
for name, version in virtualenv_versions.items():
bin_env_vars["python"]["dependencies"][name] = version
# XXX should we use version or sha1 here and in the sort key?
bin_env_vars["results-comparison"] = {
"default-compare-key": "bin-env-vars.git.version",
"sort-keys": {
"bin-env-vars.git.version": [
"bin-env-vars.git.rank",
"bin-env-vars.git.sha1",
]
},
"header-keys": [
"data-env-vars.name",
"benchmark.name",
"bin-env-vars.python.version",
"bin-env-vars.python.implementation",
],
}
poulpe.write_data("bin-env.poulpe", {"bin-env-vars": bin_env_vars})
with open("bin-env.shell", "w") as f:
f.write("""#!/bin/bash
set -Eeuo pipefail
this_script=$(realpath "$0")
POULPE_BIN_ENV_ROOT="$(dirname "$this_script")"
export POULPE_BIN_ENV_ROOT
. "${POULPE_BIN_ENV_ROOT}"/bin/activate
"$@"
""")
os.chmod("bin-env.shell", 0o775)
if __name__ == "__main__":
git_loader_setup()
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment