# HG changeset patch
# User Raphaël Gomès <rgomes@octobus.net>
# Date 1734606426 -3600
#      Thu Dec 19 12:07:06 2024 +0100
# Node ID 0caadc880915764764b9c97324465fc03e3c23a7
# Parent  048b43c5ca908664e2c92ec8bfa79dd5859ba394
swh: add a benchmark for the git loader backed with a postgres storage

This is still loading from disk, but will at least use the postgres storage
that we're interested in.

diff --git a/suites/swh/benchmarks/from-disk-postgres.pbd b/suites/swh/benchmarks/from-disk-postgres.pbd
new file mode 100644
--- /dev/null
+++ b/suites/swh/benchmarks/from-disk-postgres.pbd
@@ -0,0 +1,85 @@
+[meta]
+format = "0"
+method = "simple-command"
+name = "swh.git-loader.from-disk.postgres"
+
+[run]
+copy-data-env = false
+
+[environment]
+TARGET_REPO = "DATA-VARS:swh.swh-loader-git.target-repo-path"
+
+[simple-command]
+command = """swh -l ERROR loader -C "$POULPE_TMP_DIR/loader-conf.yml" run git "$TARGET_REPO" > /dev/null"""
+
+gather-info = """
+# TODO Use a separate pgcluster for this to make sure we don't do anything stupid
+# TODO This requires CREATE DATABASE privileges anyway and no authentication,
+# TODO so no huge risk right now
+dropdb poulpe-swh-git-loader || true
+createdb poulpe-swh-git-loader
+postgres_server_version=$(psql poulpe-swh-git-loader -t -c 'SHOW server_version;')
+postgres_client_version=$(psql --version)
+dropdb poulpe-swh-git-loader
+cat > "$POULPE_TMP_DIR/gathered_info.json" << EOF
+{
+    "postgres": {
+        "server_version": "$postgres_server_version",
+        "client_version": "$postgres_client_version"
+    }
+}
+EOF
+"""
+
+prepare-run = """
+dropdb poulpe-swh-git-loader 2>/dev/null || true
+createdb poulpe-swh-git-loader
+swh -l ERROR db init-admin -d postgresql://localhost/poulpe-swh-git-loader storage >/dev/null 2>&1
+swh -l ERROR db init -d postgresql://localhost/poulpe-swh-git-loader storage >/dev/null 2>&1
+
+cat > "$POULPE_TMP_DIR/loader-conf.yml" << EOF
+storage:
+  cls: pipeline
+  steps:
+  - cls: buffer
+    min_batch_size:
+      content: 100
+      content_bytes: 52428800
+      directory: 100
+      directory_entries: 500
+      extid: 100
+      release: 100
+      release_bytes: 52428800
+      revision: 100
+      revision_bytes: 52428800
+      revision_parents: 200
+  - cls: filter
+  - cls: retry
+  - cls: remote
+    url: http://127.0.0.1:5002
+EOF
+
+mkdir "$POULPE_TMP_DIR/objstorage"
+
+cat > "$POULPE_TMP_DIR/storage-conf.yml" << EOF
+storage:
+  cls: postgresql
+  db: postgresql://localhost/poulpe-swh-git-loader
+  objstorage:
+    cls: pathslicing
+    root: $POULPE_TMP_DIR/objstorage
+    slicing: "0:2/2:4/4:6"
+EOF
+
+# TODO process groups?
+
+swh -l ERROR storage -C "$POULPE_TMP_DIR/storage-conf.yml" rpc-serve >/dev/null 2>&1 &
+echo "$!" > "$POULPE_TMP_DIR/rpc-serve-pid"
+"""
+
+cleanup-run = """
+cat "$POULPE_TMP_DIR/rpc-serve-pid" | xargs kill
+dropdb poulpe-swh-git-loader 2>/dev/null || true
+rm -rf "$POULPE_TMP_DIR"
+mkdir --parents "$POULPE_TMP_DIR"
+"""
diff --git a/tests/test-benchmark-listing.t b/tests/test-benchmark-listing.t
--- a/tests/test-benchmark-listing.t
+++ b/tests/test-benchmark-listing.t
@@ -49,6 +49,7 @@
   hg.perf.perf-bundle
   hg.perf.unbundle
   swh.git-loader.from-disk.in-memory
+  swh.git-loader.from-disk.postgres
 
 Showing path
 ------------
@@ -70,6 +71,8 @@
    benchmarks/hg/perf-unbundle.pbd
   swh.git-loader.from-disk.in-memory
    benchmarks/swh/from-disk-in-memory.pbd
+  swh.git-loader.from-disk.postgres
+   benchmarks/swh/from-disk-postgres.pbd
 
 listing variants
 ----------------
@@ -195,6 +198,7 @@
       - default (default)
       - push
   swh.git-loader.from-disk.in-memory
+  swh.git-loader.from-disk.postgres
 
 with a patterns
 --------------
diff --git a/tests/test-setup-poulpe-den.t b/tests/test-setup-poulpe-den.t
--- a/tests/test-setup-poulpe-den.t
+++ b/tests/test-setup-poulpe-den.t
@@ -120,7 +120,7 @@
   data environments:     [OKAY]
     available: 0
   benchmarks:            [OKAY]
-    available: 27
+    available: 28
   results:               [OKAY]
   poulpe suites:         [OKAY]
     available bin setup: 2
@@ -137,7 +137,7 @@
   data environments:     [OKAY]
     available: 0
   benchmarks:            [OKAY]
-    available: 27
+    available: 28
   results:               [OKAY]
   poulpe suites:         [OKAY]
     available bin setup: 2
diff --git a/tests/test-tasks-generation.t b/tests/test-tasks-generation.t
--- a/tests/test-tasks-generation.t
+++ b/tests/test-tasks-generation.t
@@ -63,28 +63,30 @@
 
   $ poulpe den-generate-tasks hg:MERCURIAL_VERSION=6.9 > tasks
   $ wc -l < tasks
-  8
+  10
   $ sort --unique < tasks | wc -l
-  8
+  10
 (we grep to skip later introduced benchmark)
   $ head tasks
   RUN hg:MERCURIAL_VERSION=6.9 bar hg/cat.pbd
   RUN hg:MERCURIAL_VERSION=6.9 bar hg/debug-update-cache.pbd
   RUN hg:MERCURIAL_VERSION=6.9 bar hg/perf-read-revisions.pbd
   RUN hg:MERCURIAL_VERSION=6.9 bar swh/from-disk-in-memory.pbd
+  RUN hg:MERCURIAL_VERSION=6.9 bar swh/from-disk-postgres.pbd
   RUN hg:MERCURIAL_VERSION=6.9 foo hg/cat.pbd
   RUN hg:MERCURIAL_VERSION=6.9 foo hg/debug-update-cache.pbd
   RUN hg:MERCURIAL_VERSION=6.9 foo hg/perf-read-revisions.pbd
   RUN hg:MERCURIAL_VERSION=6.9 foo swh/from-disk-in-memory.pbd
+  RUN hg:MERCURIAL_VERSION=6.9 foo swh/from-disk-postgres.pbd
 
 no-filter-run using all variants
 --------------------------------
 
   $ poulpe den-generate-tasks --all-variants hg:MERCURIAL_VERSION=6.9 > tasks
   $ wc -l < tasks
-  112
+  114
   $ sort --unique < tasks | wc -l
-  112
+  114
   $ head tasks
   RUN hg:MERCURIAL_VERSION=6.9 bar hg/cat.pbd:files=all-root:output=plain:rev=working-copy
   RUN hg:MERCURIAL_VERSION=6.9 bar hg/cat.pbd:files=all-root:output=plain:rev=tip
@@ -104,9 +106,9 @@
   >   --data-env-filter mercurial.repo.graph.visible-revision-count="re:^1....$" \
   >   > tasks
   $ wc -l < tasks
-  56
+  57
   $ sort --unique < tasks | wc -l
-  56
+  57
   $ head tasks
   RUN hg:MERCURIAL_VERSION=6.9 foo hg/cat.pbd:files=all-root:output=plain:rev=working-copy
   RUN hg:MERCURIAL_VERSION=6.9 foo hg/cat.pbd:files=all-root:output=plain:rev=tip
@@ -123,9 +125,9 @@
   >   --data-env-filter name="ba-bar" \
   >   > tasks
   $ wc -l < tasks
-  56
+  57
   $ sort --unique < tasks | wc -l
-  56
+  57
   $ head tasks
   RUN hg:MERCURIAL_VERSION=6.9 bar hg/cat.pbd:files=all-root:output=plain:rev=working-copy
   RUN hg:MERCURIAL_VERSION=6.9 bar hg/cat.pbd:files=all-root:output=plain:rev=tip
@@ -169,9 +171,9 @@
   >   --benchmark-variant-filter files="re:^.*shuffled$" \
   >   > tasks
   $ wc -l < tasks
-  52
+  54
   $ sort --unique < tasks | wc -l
-  52
+  54
   $ head tasks
   RUN hg:MERCURIAL_VERSION=6.9 bar hg/cat.pbd:files=all-list-path-shuffled:output=plain:rev=working-copy
   RUN hg:MERCURIAL_VERSION=6.9 bar hg/cat.pbd:files=all-list-path-shuffled:output=plain:rev=tip