Skip to content
Snippets Groups Projects
Commit b10ae44f authored by Pierre-Yves David's avatar Pierre-Yves David :octopus:
Browse files

swh: convert the benchmark to pkl

Now that the template is globally available, we can convert the toml to pkl.
parent 1db916d2
No related branches found
No related tags found
No related merge requests found
Pipeline #102567 passed
benchmarks:
make -C suites/hg/benchmarks/
make -C suites/swh/benchmarks/
amends "lib/simple-command.pkl"
name = "swh.git-loader.from-disk.in-memory"
// we do not modify the loaded project
copy_data_env = false
environment {
["TARGET_REPO"] = "DATA-VARS:swh.swh-loader-git.target-repo-path"
}
command = """
swh loader run git $TARGET_REPO
"""
[meta]
format = "0"
name = "swh.git-loader.from-disk.postgres"
method = "simple-command"
......@@ -3,5 +4,4 @@
method = "simple-command"
name = "swh.git-loader.from-disk.postgres"
[run]
copy-data-env = false
......@@ -10,6 +10,4 @@
TARGET_REPO = "DATA-VARS:swh.swh-loader-git.target-repo-path"
[simple-command]
command = """swh -l ERROR loader -C "$POULPE_TMP_DIR/loader-conf.yml" run git "$TARGET_REPO" > /dev/null"""
gather-info = """
......@@ -15,9 +13,6 @@
gather-info = """
# TODO Use a separate pgcluster for this to make sure we don't do anything stupid
# TODO This requires CREATE DATABASE privileges anyway and no authentication,
# TODO so no huge risk right now
dropdb poulpe-swh-git-loader || true
createdb poulpe-swh-git-loader
postgres_server_version=$(psql poulpe-swh-git-loader -t -c 'SHOW server_version;')
postgres_client_version=$(psql --version)
dropdb poulpe-swh-git-loader
......@@ -19,7 +14,7 @@
dropdb poulpe-swh-git-loader || true
createdb poulpe-swh-git-loader
postgres_server_version=$(psql poulpe-swh-git-loader -t -c 'SHOW server_version;')
postgres_client_version=$(psql --version)
dropdb poulpe-swh-git-loader
cat > "$POULPE_TMP_DIR/gathered_info.json" << EOF
cat > \"$POULPE_TMP_DIR/gathered_info.json\" << EOF
{
......@@ -25,6 +20,6 @@
{
"postgres": {
"server_version": "$postgres_server_version",
"client_version": "$postgres_client_version"
\"postgres\": {
\"server_version\": \"$postgres_server_version\",
\"client_version\": \"$postgres_client_version\"
}
}
......@@ -29,11 +24,10 @@
}
}
EOF
"""
EOF"""
command = "swh -l ERROR loader -C \"$POULPE_TMP_DIR/loader-conf.yml\" run git \"$TARGET_REPO\" > /dev/null"
prepare-run = """
dropdb poulpe-swh-git-loader 2>/dev/null || true
createdb poulpe-swh-git-loader
swh -l ERROR db init-admin -d postgresql://localhost/poulpe-swh-git-loader storage >/dev/null 2>&1
swh -l ERROR db init -d postgresql://localhost/poulpe-swh-git-loader storage >/dev/null 2>&1
......@@ -34,10 +28,10 @@
prepare-run = """
dropdb poulpe-swh-git-loader 2>/dev/null || true
createdb poulpe-swh-git-loader
swh -l ERROR db init-admin -d postgresql://localhost/poulpe-swh-git-loader storage >/dev/null 2>&1
swh -l ERROR db init -d postgresql://localhost/poulpe-swh-git-loader storage >/dev/null 2>&1
cat > "$POULPE_TMP_DIR/loader-conf.yml" << EOF
cat > \"$POULPE_TMP_DIR/loader-conf.yml\" << EOF
storage:
cls: pipeline
steps:
......@@ -59,5 +53,5 @@
url: http://127.0.0.1:5002
EOF
mkdir "$POULPE_TMP_DIR/objstorage"
mkdir \"$POULPE_TMP_DIR/objstorage\"
......@@ -63,8 +57,10 @@
cat > "$POULPE_TMP_DIR/storage-conf.yml" << EOF
# XXX run that one with python3.11 not pypy
cat > \"$POULPE_TMP_DIR/storage-conf.yml\" << EOF
storage:
cls: postgresql
db: postgresql://localhost/poulpe-swh-git-loader
objstorage:
cls: pathslicing
root: $POULPE_TMP_DIR/objstorage
......@@ -65,11 +61,11 @@
storage:
cls: postgresql
db: postgresql://localhost/poulpe-swh-git-loader
objstorage:
cls: pathslicing
root: $POULPE_TMP_DIR/objstorage
slicing: "0:2/2:4/4:6"
slicing: \"0:2/2:4/4:6\"
EOF
# TODO process groups?
......@@ -72,8 +68,8 @@
EOF
# TODO process groups?
swh -l ERROR storage -C "$POULPE_TMP_DIR/storage-conf.yml" rpc-serve >/dev/null 2>&1 &
echo "$!" > "$POULPE_TMP_DIR/rpc-serve-pid"
swh -l ERROR storage -C \"$POULPE_TMP_DIR/storage-conf.yml\" rpc-serve >/dev/null 2>&1 &
echo \"$!\" > \"$POULPE_TMP_DIR/rpc-serve-pid\"
# in the preload case, load the data so that the benchmark has nothing left to do
......@@ -78,5 +74,5 @@
# in the preload case, load the data so that the benchmark has nothing left to do
if [ PRELOAD == "all" ]; then
swh -l ERROR loader -C "$POULPE_TMP_DIR/loader-conf.yml" run git "$TARGET_REPO" > /dev/null
if [ PRELOAD == \"all\" ]; then
swh -l ERROR loader -C \"$POULPE_TMP_DIR/loader-conf.yml\" run git \"$TARGET_REPO\" > /dev/null
fi
......@@ -82,5 +78,3 @@
fi
sync
"""
sync"""
cleanup-run = """
......@@ -86,3 +80,3 @@
cleanup-run = """
cat "$POULPE_TMP_DIR/rpc-serve-pid" | xargs kill
cat \"$POULPE_TMP_DIR/rpc-serve-pid\" | xargs kill
dropdb poulpe-swh-git-loader 2>/dev/null || true
......@@ -88,7 +82,6 @@
dropdb poulpe-swh-git-loader 2>/dev/null || true
rm -rf "$POULPE_TMP_DIR"
mkdir --parents "$POULPE_TMP_DIR"
"""
rm -rf \"$POULPE_TMP_DIR\"
mkdir --parents \"$POULPE_TMP_DIR\""""
[simple-command.variants.dimensions.preload.none]
default = true
......@@ -92,5 +85,4 @@
[simple-command.variants.dimensions.preload.none]
default = true
environment.PRELOAD = "none"
......@@ -96,3 +88,6 @@
[simple-command.variants.dimensions.preload.all]
environment.PRELOAD = "all"
[simple-command.variants.dimensions.preload.none.environment]
PRELOAD = "none"
[simple-command.variants.dimensions.preload.all.environment]
PRELOAD = "all"
amends "lib/simple-command.pkl"
name = "swh.git-loader.from-disk.postgres"
// we do not modify the loaded project
copy_data_env = false
local db_name = "poulpe-swh-git-loader"
local batch_item_size = 100
local batch_bytes_size = 52428800
local loader_conf_path = "loader-conf.yml"
local storage_conf_path = "storage-conf.yml"
local pids_file = "rpc-serve-pid"
environment {
["TARGET_REPO"] = "DATA-VARS:swh.swh-loader-git.target-repo-path"
}
command = """
swh -l ERROR loader -C "$POULPE_TMP_DIR/\(loader_conf_path)" run git "$TARGET_REPO" > /dev/null
"""
// TODO Use a separate pgcluster for this to make sure we don't do anything stupid
// TODO This requires CREATE DATABASE privileges anyway and no authentication,
// TODO so no huge risk right now
gather_info = """
dropdb \(db_name) || true
createdb \(db_name)
postgres_server_version=$(psql \(db_name) -t -c 'SHOW server_version;')
postgres_client_version=$(psql --version)
dropdb \(db_name)
cat > "$POULPE_TMP_DIR/gathered_info.json" << EOF
{
"postgres": {
"server_version": "$postgres_server_version",
"client_version": "$postgres_client_version"
}
}
EOF
"""
prepare_run = """
dropdb \(db_name) 2>/dev/null || true
createdb \(db_name)
swh -l ERROR db init-admin -d postgresql://localhost/\(db_name) storage >/dev/null 2>&1
swh -l ERROR db init -d postgresql://localhost/\(db_name) storage >/dev/null 2>&1
cat > "$POULPE_TMP_DIR/\(loader_conf_path)" << EOF
storage:
cls: pipeline
steps:
- cls: buffer
min_batch_size:
content: \(batch_item_size)
content_bytes: \(batch_bytes_size)
directory: \(batch_item_size)
directory_entries: \(batch_item_size * 5)
extid: \(batch_item_size)
release: \(batch_item_size)
release_bytes: \(batch_bytes_size)
revision: \(batch_item_size)
revision_bytes: \(batch_bytes_size)
revision_parents: \(batch_item_size * 2)
- cls: filter
- cls: retry
- cls: remote
url: http://127.0.0.1:5002
EOF
mkdir "$POULPE_TMP_DIR/objstorage"
# XXX run that one with python3.11 not pypy
cat > "$POULPE_TMP_DIR/\(storage_conf_path)" << EOF
storage:
cls: postgresql
db: postgresql://localhost/\(db_name)
objstorage:
cls: pathslicing
root: $POULPE_TMP_DIR/objstorage
slicing: "0:2/2:4/4:6"
EOF
# TODO process groups?
swh -l ERROR storage -C "$POULPE_TMP_DIR/\(storage_conf_path)" rpc-serve >/dev/null 2>&1 &
echo "$!" > "$POULPE_TMP_DIR/\(pids_file)"
# in the preload case, load the data so that the benchmark has nothing left to do
if [ PRELOAD == "all" ]; then
swh -l ERROR loader -C "$POULPE_TMP_DIR/\(loader_conf_path)" run git "$TARGET_REPO" > /dev/null
fi
sync
"""
cleanup_run = """
cat "$POULPE_TMP_DIR/\(pids_file)" | xargs kill
dropdb \(db_name) 2>/dev/null || true
rm -rf "$POULPE_TMP_DIR"
mkdir --parents "$POULPE_TMP_DIR"
"""
variants {
["preload"] {
default_key = "none"
cases {
for (preload in List("none", "all")) {
[preload] {
environment {
["PRELOAD"] = preload
}
}
}
}
}
}
amends "../../../_common/benchmarks/simple-command.pkl"
// just a local re-export for Mercurial, might contains mercurial specific content later.
// just a local re-export for SWH, might contains SWH specific content later.
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment