# HG changeset patch # User Raphaël Gomès <rgomes@octobus.net> # Date 1734606426 -3600 # Thu Dec 19 12:07:06 2024 +0100 # Node ID 0caadc880915764764b9c97324465fc03e3c23a7 # Parent 048b43c5ca908664e2c92ec8bfa79dd5859ba394 swh: add a benchmark for the git loader backed with a postgres storage This is still loading from disk, but will at least use the postgres storage that we're interested in. diff --git a/suites/swh/benchmarks/from-disk-postgres.pbd b/suites/swh/benchmarks/from-disk-postgres.pbd new file mode 100644 --- /dev/null +++ b/suites/swh/benchmarks/from-disk-postgres.pbd @@ -0,0 +1,85 @@ +[meta] +format = "0" +method = "simple-command" +name = "swh.git-loader.from-disk.postgres" + +[run] +copy-data-env = false + +[environment] +TARGET_REPO = "DATA-VARS:swh.swh-loader-git.target-repo-path" + +[simple-command] +command = """swh -l ERROR loader -C "$POULPE_TMP_DIR/loader-conf.yml" run git "$TARGET_REPO" > /dev/null""" + +gather-info = """ +# TODO Use a separate pgcluster for this to make sure we don't do anything stupid +# TODO This requires CREATE DATABASE privileges anyway and no authentication, +# TODO so no huge risk right now +dropdb poulpe-swh-git-loader || true +createdb poulpe-swh-git-loader +postgres_server_version=$(psql poulpe-swh-git-loader -t -c 'SHOW server_version;') +postgres_client_version=$(psql --version) +dropdb poulpe-swh-git-loader +cat > "$POULPE_TMP_DIR/gathered_info.json" << EOF +{ + "postgres": { + "server_version": "$postgres_server_version", + "client_version": "$postgres_client_version" + } +} +EOF +""" + +prepare-run = """ +dropdb poulpe-swh-git-loader 2>/dev/null || true +createdb poulpe-swh-git-loader +swh -l ERROR db init-admin -d postgresql://localhost/poulpe-swh-git-loader storage >/dev/null 2>&1 +swh -l ERROR db init -d postgresql://localhost/poulpe-swh-git-loader storage >/dev/null 2>&1 + +cat > "$POULPE_TMP_DIR/loader-conf.yml" << EOF +storage: + cls: pipeline + steps: + - cls: buffer + min_batch_size: + content: 100 + content_bytes: 52428800 + directory: 100 + directory_entries: 500 + extid: 100 + release: 100 + release_bytes: 52428800 + revision: 100 + revision_bytes: 52428800 + revision_parents: 200 + - cls: filter + - cls: retry + - cls: remote + url: http://127.0.0.1:5002 +EOF + +mkdir "$POULPE_TMP_DIR/objstorage" + +cat > "$POULPE_TMP_DIR/storage-conf.yml" << EOF +storage: + cls: postgresql + db: postgresql://localhost/poulpe-swh-git-loader + objstorage: + cls: pathslicing + root: $POULPE_TMP_DIR/objstorage + slicing: "0:2/2:4/4:6" +EOF + +# TODO process groups? + +swh -l ERROR storage -C "$POULPE_TMP_DIR/storage-conf.yml" rpc-serve >/dev/null 2>&1 & +echo "$!" > "$POULPE_TMP_DIR/rpc-serve-pid" +""" + +cleanup-run = """ +cat "$POULPE_TMP_DIR/rpc-serve-pid" | xargs kill +dropdb poulpe-swh-git-loader 2>/dev/null || true +rm -rf "$POULPE_TMP_DIR" +mkdir --parents "$POULPE_TMP_DIR" +""" diff --git a/tests/test-benchmark-listing.t b/tests/test-benchmark-listing.t --- a/tests/test-benchmark-listing.t +++ b/tests/test-benchmark-listing.t @@ -49,6 +49,7 @@ hg.perf.perf-bundle hg.perf.unbundle swh.git-loader.from-disk.in-memory + swh.git-loader.from-disk.postgres Showing path ------------ @@ -70,6 +71,8 @@ benchmarks/hg/perf-unbundle.pbd swh.git-loader.from-disk.in-memory benchmarks/swh/from-disk-in-memory.pbd + swh.git-loader.from-disk.postgres + benchmarks/swh/from-disk-postgres.pbd listing variants ---------------- @@ -195,6 +198,7 @@ - default (default) - push swh.git-loader.from-disk.in-memory + swh.git-loader.from-disk.postgres with a patterns -------------- diff --git a/tests/test-setup-poulpe-den.t b/tests/test-setup-poulpe-den.t --- a/tests/test-setup-poulpe-den.t +++ b/tests/test-setup-poulpe-den.t @@ -120,7 +120,7 @@ data environments: [OKAY] available: 0 benchmarks: [OKAY] - available: 27 + available: 28 results: [OKAY] poulpe suites: [OKAY] available bin setup: 2 @@ -137,7 +137,7 @@ data environments: [OKAY] available: 0 benchmarks: [OKAY] - available: 27 + available: 28 results: [OKAY] poulpe suites: [OKAY] available bin setup: 2 diff --git a/tests/test-tasks-generation.t b/tests/test-tasks-generation.t --- a/tests/test-tasks-generation.t +++ b/tests/test-tasks-generation.t @@ -63,28 +63,30 @@ $ poulpe den-generate-tasks hg:MERCURIAL_VERSION=6.9 > tasks $ wc -l < tasks - 8 + 10 $ sort --unique < tasks | wc -l - 8 + 10 (we grep to skip later introduced benchmark) $ head tasks RUN hg:MERCURIAL_VERSION=6.9 bar hg/cat.pbd RUN hg:MERCURIAL_VERSION=6.9 bar hg/debug-update-cache.pbd RUN hg:MERCURIAL_VERSION=6.9 bar hg/perf-read-revisions.pbd RUN hg:MERCURIAL_VERSION=6.9 bar swh/from-disk-in-memory.pbd + RUN hg:MERCURIAL_VERSION=6.9 bar swh/from-disk-postgres.pbd RUN hg:MERCURIAL_VERSION=6.9 foo hg/cat.pbd RUN hg:MERCURIAL_VERSION=6.9 foo hg/debug-update-cache.pbd RUN hg:MERCURIAL_VERSION=6.9 foo hg/perf-read-revisions.pbd RUN hg:MERCURIAL_VERSION=6.9 foo swh/from-disk-in-memory.pbd + RUN hg:MERCURIAL_VERSION=6.9 foo swh/from-disk-postgres.pbd no-filter-run using all variants -------------------------------- $ poulpe den-generate-tasks --all-variants hg:MERCURIAL_VERSION=6.9 > tasks $ wc -l < tasks - 112 + 114 $ sort --unique < tasks | wc -l - 112 + 114 $ head tasks RUN hg:MERCURIAL_VERSION=6.9 bar hg/cat.pbd:files=all-root:output=plain:rev=working-copy RUN hg:MERCURIAL_VERSION=6.9 bar hg/cat.pbd:files=all-root:output=plain:rev=tip @@ -104,9 +106,9 @@ > --data-env-filter mercurial.repo.graph.visible-revision-count="re:^1....$" \ > > tasks $ wc -l < tasks - 56 + 57 $ sort --unique < tasks | wc -l - 56 + 57 $ head tasks RUN hg:MERCURIAL_VERSION=6.9 foo hg/cat.pbd:files=all-root:output=plain:rev=working-copy RUN hg:MERCURIAL_VERSION=6.9 foo hg/cat.pbd:files=all-root:output=plain:rev=tip @@ -123,9 +125,9 @@ > --data-env-filter name="ba-bar" \ > > tasks $ wc -l < tasks - 56 + 57 $ sort --unique < tasks | wc -l - 56 + 57 $ head tasks RUN hg:MERCURIAL_VERSION=6.9 bar hg/cat.pbd:files=all-root:output=plain:rev=working-copy RUN hg:MERCURIAL_VERSION=6.9 bar hg/cat.pbd:files=all-root:output=plain:rev=tip @@ -169,9 +171,9 @@ > --benchmark-variant-filter files="re:^.*shuffled$" \ > > tasks $ wc -l < tasks - 52 + 54 $ sort --unique < tasks | wc -l - 52 + 54 $ head tasks RUN hg:MERCURIAL_VERSION=6.9 bar hg/cat.pbd:files=all-list-path-shuffled:output=plain:rev=working-copy RUN hg:MERCURIAL_VERSION=6.9 bar hg/cat.pbd:files=all-list-path-shuffled:output=plain:rev=tip