Skip to content
Snippets Groups Projects
Commit 86d5e2d2 authored by Pierre-Yves David's avatar Pierre-Yves David :octopus:
Browse files

setup changes

parent bef99438
No related tags found
No related merge requests found
Pipeline #98038 failed
......@@ -5,6 +5,7 @@
# in `/tmp/poulpe-clone-cache/swh/git-loader`
import hashlib
import shutil
import json
import os
from pathlib import Path
......@@ -14,7 +15,7 @@
import click
import poulpe
CLONE_PATH = "/tmp/poulpe-clone-cache/swh/swh-loader-git"
CLONE_PATH_TMPL = "/tmp/poulpe-clone-cache/swh/{name}"
def run_command(command: List[str]):
......@@ -24,8 +25,9 @@
completed = subprocess.run(command, capture_output=True, env=environ)
if completed.returncode != 0:
click.secho(completed.stdout, fg="yellow", file=sys.stderr)
click.secho(completed.stderr, fg="red", file=sys.stderr)
completed.check_returncode()
return completed.stdout
......@@ -27,9 +29,9 @@
click.secho(completed.stderr, fg="red", file=sys.stderr)
completed.check_returncode()
return completed.stdout
def compute_unique_id(git_sha1, python_version, postgres_version):
def compute_unique_id(all_git_sha1, python_version, postgres_version):
hasher = hashlib.sha256()
# Version number to bust caches
hasher.update(b"0")
......@@ -33,7 +35,11 @@
hasher = hashlib.sha256()
# Version number to bust caches
hasher.update(b"0")
hasher.update(git_sha1.encode())
for n, h in sorted(all_git_sha1):
hasher.update(b"\x00")
hasher.update(n.encode())
hasher.update(b"\x00")
hasher.update(h.encode())
hasher.update(b"\x00")
hasher.update(python_version.encode())
hasher.update(b"\x00")
......@@ -50,8 +56,8 @@
json.dumps(
{
"parameters": {
"loader-version": {
"swh-loader-git-version": {
"doc": "an identifier of the swh-loader-git version we want to install",
"type": "<git-revision>",
"arg-mode": "CLI",
},
......@@ -54,10 +60,10 @@
"doc": "an identifier of the swh-loader-git version we want to install",
"type": "<git-revision>",
"arg-mode": "CLI",
},
"target-repo": {
"swh-loader-git-target-repo": {
"default": "https://gitlab.softwareheritage.org/swh/devel/swh-loader-git",
"doc": "a path to the repo of swh-loader-git",
"type": "<url>",
"arg-mode": "CLI",
},
......@@ -59,8 +65,30 @@
"default": "https://gitlab.softwareheritage.org/swh/devel/swh-loader-git",
"doc": "a path to the repo of swh-loader-git",
"type": "<url>",
"arg-mode": "CLI",
},
"swh-storage-version": {
"doc": "an identifier of the swh-storage version we want to install",
"type": "<git-revision>",
"arg-mode": "CLI",
},
"swh-storage-target-repo": {
"default": "https://gitlab.softwareheritage.org/swh/devel/swh-storage",
"doc": "a path to the repo of swh-storage",
"type": "<url>",
"arg-mode": "CLI",
},
"swh-core-version": {
"doc": "an identifier of the swh-core version we want to install",
"type": "<git-revision>",
"arg-mode": "CLI",
},
"swh-core-target-repo": {
"default": "https://gitlab.softwareheritage.org/swh/devel/swh-core",
"doc": "a path to the repo of swh-core",
"type": "<url>",
"arg-mode": "CLI",
},
"python-version": {
"default": "python3",
"doc": "a path to the Python to use for the bin env",
......@@ -83,6 +111,66 @@
"""disabled echo when computing the unique id"""
def install_one(name: str, repo_url: str, version: str, get_unique_id: bool = False) -> tuple[int, str]:
clone_path = Path(CLONE_PATH_TMPL.format(name=name))
if (clone_path).is_dir():
shutil.rmtree(clone_path)
# Does it look like a clone? Don't bother to be more resistant for now
if (clone_path / ".git").is_dir():
# Do we already have the revision?
try:
run_command(["git", "-C", clone_path, "rev-parse", version])
except subprocess.CalledProcessError:
# If not, fetch
echo(f"POULPE: failed to retrieve version, fetching")
run_command(["git", "-C", clone_path, "pull", repo_url])
echo(f"POULPE: fetch done")
else:
echo(f"POULPE: expected version already in cache")
else:
echo(f"POULPE: cloning swh-git-loader from {repo_url}")
run_command(["git", "clone", "--mirror", repo_url, clone_path])
echo(f"POULPE: clone done")
git_rank = int(
(
run_command(
[
"git",
"-C",
str(clone_path),
"rev-list",
"--count",
version,
]
)
.strip()
.decode()
)
)
git_sha1 = (
run_command(["git", "-C", clone_path, "rev-parse", version])
.strip()
.decode()
)
if not get_unique_id:
echo(f"POULPE: installing {name} in the new venv")
run_command(
[
"bin/pip",
"install",
# install with no working copy for speed and making concurrent
# installs possible
f"git+file://{clone_path}@{git_sha1}",
"--require-virtualenv",
]
)
return git_rank, git_sha1
# Explicitly don't set `no_args_is_help=True` because we want the script
# to return an error code if nothing is passed
@click.command()
......@@ -100,8 +188,8 @@
help="return the unique identifier derived from the all the parameters to this bin-env",
)
@click.option(
"--loader-version",
"--swh-loader-git-version",
required=True,
help="an identifier of the swh-loader-git version we want to install",
)
@click.option(
......@@ -104,9 +192,9 @@
required=True,
help="an identifier of the swh-loader-git version we want to install",
)
@click.option(
"--target-repo",
"--swh-loader-git-target-repo",
help="a path to the repo of swh-loader-git",
default="https://gitlab.softwareheritage.org/swh/devel/swh-loader-git",
)
@click.option(
......@@ -109,7 +197,27 @@
help="a path to the repo of swh-loader-git",
default="https://gitlab.softwareheritage.org/swh/devel/swh-loader-git",
)
@click.option(
"--swh-storage-version",
required=True,
help="an identifier of the swh-storage version we want to install",
)
@click.option(
"--swh-storage-target-repo",
help="a path to the repo of swh-storage",
default="https://gitlab.softwareheritage.org/swh/devel/swh-storage",
)
@click.option(
"--swh-core-version",
required=True,
help="an identifier of the swh-core version we want to install",
)
@click.option(
"--swh-core-target-repo",
help="a path to the repo of swh-core",
default="https://gitlab.softwareheritage.org/swh/devel/swh-core",
)
@click.option(
"--python-version",
default="python3",
help="a path to the Python to use for the bin env",
......@@ -117,8 +225,12 @@
def git_loader_setup(
describe_parameters: bool,
get_unique_id: bool,
loader_version: str,
target_repo: str,
swh_loader_git_version: str,
swh_loader_git_target_repo: str,
swh_storage_version: str,
swh_storage_target_repo: str,
swh_core_version: str,
swh_core_target_repo: str,
python_version: str,
):
if get_unique_id:
......@@ -127,6 +239,17 @@
echo = noop
echo(f"POULPE: using python version {python_version}")
echo(f"POULPE: using swh-git-loader version {loader_version}")
echo(f"POULPE: using loader url {target_repo}")
echo(f"POULPE: using swh-loader-git version {swh_loader_git_version}")
echo(f"POULPE: using swh-loader-git url {swh_loader_git_target_repo}")
echo(f"POULPE: using swh-storage version {swh_storage_version}")
echo(f"POULPE: using swh-storage url {swh_storage_target_repo}")
echo(f"POULPE: using swh-core version {swh_core_version}")
echo(f"POULPE: using swh-core url {swh_core_target_repo}")
to_install = [
("swh-loader-git", swh_loader_git_target_repo, swh_loader_git_version),
("swh-storage", swh_storage_target_repo, swh_storage_version),
("swh-core", swh_core_target_repo, swh_core_version),
]
......@@ -132,18 +255,6 @@
# Does it look like a clone? Don't bother to be more resistant for now
if Path(CLONE_PATH, ".git").is_dir():
# Do we already have the revision?
try:
run_command(["git", "-C", CLONE_PATH, "rev-parse", loader_version])
except subprocess.CalledProcessError:
# If not, fetch
echo(f"POULPE: failed to retrieve version, fetching")
run_command(["git", "-C", CLONE_PATH, "fetch", target_repo])
echo(f"POULPE: fetch done")
else:
echo(f"POULPE: expected version already in cache")
else:
echo(f"POULPE: cloning swh-git-loader from {target_repo}")
run_command(["git", "clone", target_repo, CLONE_PATH])
echo(f"POULPE: clone done")
echo("POULPE: creating a new virtual environment")
run_command([python_version, "-m", "venv", "."])
......@@ -149,18 +260,12 @@
git_rank = int(
(
run_command(
[
"git",
"-C",
CLONE_PATH,
"rev-list",
"--count",
loader_version,
]
)
.strip()
.decode()
)
all_git_sha1 = []
all_git_rank = []
for t in to_install:
rank, gs1 = install_one(*t, get_unique_id=get_unique_id)
all_git_sha1.append((t[0], gs1))
all_git_rank.append((t[0], rank))
unique_id = compute_unique_id(
all_git_sha1, python_version, postgres_version=None
)
......@@ -165,14 +270,7 @@
)
git_sha1 = (
run_command(["git", "-C", CLONE_PATH, "rev-parse", loader_version])
.strip()
.decode()
)
unique_id = compute_unique_id(
git_sha1, python_version, postgres_version=None
)
# XXX We now check the name after starting install, so we destroyed the
# existing bin env in the process
if not get_unique_id:
# This means we've already got the unique ID, so make sure we match
name = Path().resolve().name
......@@ -185,20 +283,6 @@
click.echo(f"UNIQUE_ID: {unique_id}")
exit(0)
echo("POULPE: creating a new virtual environment")
run_command([python_version, "-m", "venv", "."])
echo("POULPE: installing swh-git-loader in the new venv")
run_command(
[
"bin/pip",
"install",
# install with no working copy for speed and making concurrent
# installs possible
f"git+file://{CLONE_PATH}@{git_sha1}",
"--require-virtualenv",
]
)
echo("POULPE: install OK")
output = run_command(
......@@ -234,9 +318,12 @@
},
"git": {
# kept as a useful shortcut, but redundant with `python.dependencies`
"version": virtualenv_versions["swh.loader.git"],
"rank": git_rank,
"sha1": git_sha1,
"version": '|'.join(
virtualenv_versions[n]
for n in ["swh.core", "swh.storage", "swh.loader.git"]
),
# XXX wrongy wrong
"rank": sum(r for n, r in all_git_rank),
},
}
for name, version in virtualenv_versions.items():
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment