Skip to content
Snippets Groups Projects
Commit c07f9e0b authored by Raphaël Gomès's avatar Raphaël Gomès
Browse files

swh: add a benchmark for the git loader backed with a postgres storage

This is still loading from disk, but will at least use the postgres storage
that we're interested in.
parent c434d2bb
No related branches found
No related tags found
1 merge request!44Suite agnostic tooling, improvements to simple-command, SWH suite
Pipeline #95627 passed
[meta]
format = "0"
method = "simple-command"
name = "swh.git-loader.from-disk.postgres"
[run]
copy-data-env = false
[environment]
TARGET_REPO = "DATA-VARS:swh.swh-loader-git.target-repo-path"
[simple-command]
command = """swh -l ERROR loader -C "$POULPE_TMP_DIR/loader-conf.yml" run git "$TARGET_REPO" > /dev/null"""
gather-info = """
# TODO Use a separate pgcluster for this to make sure we don't do anything stupid
# TODO This requires CREATE DATABASE privileges anyway and no authentication,
# TODO so no huge risk right now
dropdb poulpe-swh-git-loader || true
createdb poulpe-swh-git-loader
postgres_server_version=$(psql poulpe-swh-git-loader -t -c 'SHOW server_version;')
postgres_client_version=$(psql --version)
dropdb poulpe-swh-git-loader
cat > "$POULPE_TMP_DIR/gathered_info.json" << EOF
{
"postgres": {
"server_version": "$postgres_server_version",
"client_version": "$postgres_client_version"
}
}
EOF
"""
prepare-run = """
dropdb poulpe-swh-git-loader 2>/dev/null || true
createdb poulpe-swh-git-loader
swh -l ERROR db init-admin -d postgresql://localhost/poulpe-swh-git-loader storage >/dev/null 2>&1
swh -l ERROR db init -d postgresql://localhost/poulpe-swh-git-loader storage >/dev/null 2>&1
cat > "$POULPE_TMP_DIR/loader-conf.yml" << EOF
storage:
cls: pipeline
steps:
- cls: buffer
min_batch_size:
content: 100
content_bytes: 52428800
directory: 100
directory_entries: 500
extid: 100
release: 100
release_bytes: 52428800
revision: 100
revision_bytes: 52428800
revision_parents: 200
- cls: filter
- cls: retry
- cls: remote
url: http://127.0.0.1:5002
EOF
mkdir "$POULPE_TMP_DIR/objstorage"
cat > "$POULPE_TMP_DIR/storage-conf.yml" << EOF
storage:
cls: postgresql
db: postgresql://localhost/poulpe-swh-git-loader
objstorage:
cls: pathslicing
root: $POULPE_TMP_DIR/objstorage
slicing: "0:2/2:4/4:6"
EOF
# TODO process groups?
swh -l ERROR storage -C "$POULPE_TMP_DIR/storage-conf.yml" rpc-serve >/dev/null 2>&1 &
echo "$!" > "$POULPE_TMP_DIR/rpc-serve-pid"
"""
cleanup-run = """
cat "$POULPE_TMP_DIR/rpc-serve-pid" | xargs kill
dropdb poulpe-swh-git-loader 2>/dev/null || true
rm -rf "$POULPE_TMP_DIR"
mkdir --parents "$POULPE_TMP_DIR"
"""
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment