Skip to content
Snippets Groups Projects
Commit d1847b58 authored by Pierre-Yves David's avatar Pierre-Yves David :octopus:
Browse files

swh: convert the benchmark to pkl

Now that the template is globally available, we can convert the toml to pkl.
parent 5feaab86
No related branches found
No related tags found
1 merge request!52benchmakr: add a global makefile to call the hg's benchmark one
Pipeline #102569 passed
benchmarks:
make -C suites/hg/benchmarks/
make -C suites/swh/benchmarks/
amends "lib/simple-command.pkl"
name = "swh.git-loader.from-disk.in-memory"
// we do not modify the loaded project
copy_data_env = false
environment {
["TARGET_REPO"] = "DATA-VARS:swh.swh-loader-git.target-repo-path"
}
command = """
swh loader run git $TARGET_REPO
"""
[meta]
format = "0"
name = "swh.git-loader.from-disk.postgres"
method = "simple-command"
......@@ -3,5 +4,4 @@
method = "simple-command"
name = "swh.git-loader.from-disk.postgres"
[run]
copy-data-env = false
......@@ -10,6 +10,4 @@
TARGET_REPO = "DATA-VARS:swh.swh-loader-git.target-repo-path"
[simple-command]
command = """swh -l ERROR loader -C "$POULPE_TMP_DIR/loader-conf.yml" run git "$TARGET_REPO" > /dev/null"""
gather-info = """
......@@ -15,9 +13,6 @@
gather-info = """
# TODO Use a separate pgcluster for this to make sure we don't do anything stupid
# TODO This requires CREATE DATABASE privileges anyway and no authentication,
# TODO so no huge risk right now
dropdb poulpe-swh-git-loader || true
createdb poulpe-swh-git-loader
postgres_server_version=$(psql poulpe-swh-git-loader -t -c 'SHOW server_version;')
postgres_client_version=$(psql --version)
dropdb poulpe-swh-git-loader
......@@ -19,7 +14,7 @@
dropdb poulpe-swh-git-loader || true
createdb poulpe-swh-git-loader
postgres_server_version=$(psql poulpe-swh-git-loader -t -c 'SHOW server_version;')
postgres_client_version=$(psql --version)
dropdb poulpe-swh-git-loader
cat > "$POULPE_TMP_DIR/gathered_info.json" << EOF
cat > \"$POULPE_TMP_DIR/gathered_info.json\" << EOF
{
......@@ -25,6 +20,6 @@
{
"postgres": {
"server_version": "$postgres_server_version",
"client_version": "$postgres_client_version"
\"postgres\": {
\"server_version\": \"$postgres_server_version\",
\"client_version\": \"$postgres_client_version\"
}
}
......@@ -29,11 +24,10 @@
}
}
EOF
"""
EOF"""
command = "swh -l ERROR loader -C \"$POULPE_TMP_DIR/loader-conf.yml\" run git \"$TARGET_REPO\" > /dev/null"
prepare-run = """
dropdb poulpe-swh-git-loader 2>/dev/null || true
createdb poulpe-swh-git-loader
swh -l ERROR db init-admin -d postgresql://localhost/poulpe-swh-git-loader storage >/dev/null 2>&1
swh -l ERROR db init -d postgresql://localhost/poulpe-swh-git-loader storage >/dev/null 2>&1
......@@ -34,10 +28,10 @@
prepare-run = """
dropdb poulpe-swh-git-loader 2>/dev/null || true
createdb poulpe-swh-git-loader
swh -l ERROR db init-admin -d postgresql://localhost/poulpe-swh-git-loader storage >/dev/null 2>&1
swh -l ERROR db init -d postgresql://localhost/poulpe-swh-git-loader storage >/dev/null 2>&1
cat > "$POULPE_TMP_DIR/loader-conf.yml" << EOF
cat > \"$POULPE_TMP_DIR/loader-conf.yml\" << EOF
storage:
cls: pipeline
steps:
......@@ -59,5 +53,5 @@
url: http://127.0.0.1:5002
EOF
mkdir "$POULPE_TMP_DIR/objstorage"
mkdir \"$POULPE_TMP_DIR/objstorage\"
......@@ -63,8 +57,10 @@
cat > "$POULPE_TMP_DIR/storage-conf.yml" << EOF
# XXX run that one with python3.11 not pypy
cat > \"$POULPE_TMP_DIR/storage-conf.yml\" << EOF
storage:
cls: postgresql
db: postgresql://localhost/poulpe-swh-git-loader
objstorage:
cls: pathslicing
root: $POULPE_TMP_DIR/objstorage
......@@ -65,11 +61,11 @@
storage:
cls: postgresql
db: postgresql://localhost/poulpe-swh-git-loader
objstorage:
cls: pathslicing
root: $POULPE_TMP_DIR/objstorage
slicing: "0:2/2:4/4:6"
slicing: \"0:2/2:4/4:6\"
EOF
# TODO process groups?
......@@ -72,8 +68,8 @@
EOF
# TODO process groups?
swh -l ERROR storage -C "$POULPE_TMP_DIR/storage-conf.yml" rpc-serve >/dev/null 2>&1 &
echo "$!" > "$POULPE_TMP_DIR/rpc-serve-pid"
swh -l ERROR storage -C \"$POULPE_TMP_DIR/storage-conf.yml\" rpc-serve >/dev/null 2>&1 &
echo \"$!\" > \"$POULPE_TMP_DIR/rpc-serve-pid\"
# in the preload case, load the data so that the benchmark has nothing left to do
......@@ -78,5 +74,5 @@
# in the preload case, load the data so that the benchmark has nothing left to do
if [ PRELOAD == "all" ]; then
swh -l ERROR loader -C "$POULPE_TMP_DIR/loader-conf.yml" run git "$TARGET_REPO" > /dev/null
if [ PRELOAD == \"all\" ]; then
swh -l ERROR loader -C \"$POULPE_TMP_DIR/loader-conf.yml\" run git \"$TARGET_REPO\" > /dev/null
fi
......@@ -82,5 +78,3 @@
fi
sync
"""
sync"""
cleanup-run = """
......@@ -86,3 +80,3 @@
cleanup-run = """
cat "$POULPE_TMP_DIR/rpc-serve-pid" | xargs kill
cat \"$POULPE_TMP_DIR/rpc-serve-pid\" | xargs kill
dropdb poulpe-swh-git-loader 2>/dev/null || true
......@@ -88,7 +82,6 @@
dropdb poulpe-swh-git-loader 2>/dev/null || true
rm -rf "$POULPE_TMP_DIR"
mkdir --parents "$POULPE_TMP_DIR"
"""
rm -rf \"$POULPE_TMP_DIR\"
mkdir --parents \"$POULPE_TMP_DIR\""""
[simple-command.variants.dimensions.preload.none]
default = true
......@@ -92,5 +85,4 @@
[simple-command.variants.dimensions.preload.none]
default = true
environment.PRELOAD = "none"
......@@ -96,3 +88,6 @@
[simple-command.variants.dimensions.preload.all]
environment.PRELOAD = "all"
[simple-command.variants.dimensions.preload.none.environment]
PRELOAD = "none"
[simple-command.variants.dimensions.preload.all.environment]
PRELOAD = "all"
amends "lib/simple-command.pkl"
name = "swh.git-loader.from-disk.postgres"
// we do not modify the loaded project
copy_data_env = false
local db_name = "poulpe-swh-git-loader"
local batch_item_size = 100
local batch_bytes_size = 52428800
local loader_conf_path = "loader-conf.yml"
local storage_conf_path = "storage-conf.yml"
local pids_file = "rpc-serve-pid"
environment {
["TARGET_REPO"] = "DATA-VARS:swh.swh-loader-git.target-repo-path"
}
command = """
swh -l ERROR loader -C "$POULPE_TMP_DIR/\(loader_conf_path)" run git "$TARGET_REPO" > /dev/null
"""
// TODO Use a separate pgcluster for this to make sure we don't do anything stupid
// TODO This requires CREATE DATABASE privileges anyway and no authentication,
// TODO so no huge risk right now
gather_info = """
dropdb \(db_name) || true
createdb \(db_name)
postgres_server_version=$(psql \(db_name) -t -c 'SHOW server_version;')
postgres_client_version=$(psql --version)
dropdb \(db_name)
cat > "$POULPE_TMP_DIR/gathered_info.json" << EOF
{
"postgres": {
"server_version": "$postgres_server_version",
"client_version": "$postgres_client_version"
}
}
EOF
"""
prepare_run = """
dropdb \(db_name) 2>/dev/null || true
createdb \(db_name)
swh -l ERROR db init-admin -d postgresql://localhost/\(db_name) storage >/dev/null 2>&1
swh -l ERROR db init -d postgresql://localhost/\(db_name) storage >/dev/null 2>&1
cat > "$POULPE_TMP_DIR/\(loader_conf_path)" << EOF
storage:
cls: pipeline
steps:
- cls: buffer
min_batch_size:
content: \(batch_item_size)
content_bytes: \(batch_bytes_size)
directory: \(batch_item_size)
directory_entries: \(batch_item_size * 5)
extid: \(batch_item_size)
release: \(batch_item_size)
release_bytes: \(batch_bytes_size)
revision: \(batch_item_size)
revision_bytes: \(batch_bytes_size)
revision_parents: \(batch_item_size * 2)
- cls: filter
- cls: retry
- cls: remote
url: http://127.0.0.1:5002
EOF
mkdir "$POULPE_TMP_DIR/objstorage"
# XXX run that one with python3.11 not pypy
cat > "$POULPE_TMP_DIR/\(storage_conf_path)" << EOF
storage:
cls: postgresql
db: postgresql://localhost/\(db_name)
objstorage:
cls: pathslicing
root: $POULPE_TMP_DIR/objstorage
slicing: "0:2/2:4/4:6"
EOF
# TODO process groups?
swh -l ERROR storage -C "$POULPE_TMP_DIR/\(storage_conf_path)" rpc-serve >/dev/null 2>&1 &
echo "$!" > "$POULPE_TMP_DIR/\(pids_file)"
# in the preload case, load the data so that the benchmark has nothing left to do
if [ PRELOAD == "all" ]; then
swh -l ERROR loader -C "$POULPE_TMP_DIR/\(loader_conf_path)" run git "$TARGET_REPO" > /dev/null
fi
sync
"""
cleanup_run = """
cat "$POULPE_TMP_DIR/\(pids_file)" | xargs kill
dropdb \(db_name) 2>/dev/null || true
rm -rf "$POULPE_TMP_DIR"
mkdir --parents "$POULPE_TMP_DIR"
"""
variants {
["preload"] {
default_key = "none"
cases {
for (preload in List("none", "all")) {
[preload] {
environment {
["PRELOAD"] = preload
}
}
}
}
}
}
amends "../../../_common/benchmarks/simple-command.pkl"
// just a local re-export for Mercurial, might contains mercurial specific content later.
// just a local re-export for SWH, might contains SWH specific content later.
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment