diff --git a/Makefile b/Makefile index 1db916d2565448c26659e8f7b4508d5cb9ca12d1_TWFrZWZpbGU=..b10ae44f19bf99dd82c4f4054537aca94dbbf0c9_TWFrZWZpbGU= 100644 --- a/Makefile +++ b/Makefile @@ -1,2 +1,3 @@ benchmarks: make -C suites/hg/benchmarks/ + make -C suites/swh/benchmarks/ diff --git a/suites/hg/benchmarks/Makefile b/suites/swh/benchmarks/Makefile similarity index 100% copy from suites/hg/benchmarks/Makefile copy to suites/swh/benchmarks/Makefile diff --git a/suites/swh/benchmarks/from-disk-in-memory.pkl b/suites/swh/benchmarks/from-disk-in-memory.pkl new file mode 100644 index 0000000000000000000000000000000000000000..b10ae44f19bf99dd82c4f4054537aca94dbbf0c9_c3VpdGVzL3N3aC9iZW5jaG1hcmtzL2Zyb20tZGlzay1pbi1tZW1vcnkucGts --- /dev/null +++ b/suites/swh/benchmarks/from-disk-in-memory.pkl @@ -0,0 +1,14 @@ +amends "lib/simple-command.pkl" + +name = "swh.git-loader.from-disk.in-memory" + +// we do not modify the loaded project +copy_data_env = false + +environment { + ["TARGET_REPO"] = "DATA-VARS:swh.swh-loader-git.target-repo-path" +} + +command = """ + swh loader run git $TARGET_REPO + """ diff --git a/suites/swh/benchmarks/from-disk-postgres.pbd b/suites/swh/benchmarks/from-disk-postgres.pbd index 1db916d2565448c26659e8f7b4508d5cb9ca12d1_c3VpdGVzL3N3aC9iZW5jaG1hcmtzL2Zyb20tZGlzay1wb3N0Z3Jlcy5wYmQ=..b10ae44f19bf99dd82c4f4054537aca94dbbf0c9_c3VpdGVzL3N3aC9iZW5jaG1hcmtzL2Zyb20tZGlzay1wb3N0Z3Jlcy5wYmQ= 100644 --- a/suites/swh/benchmarks/from-disk-postgres.pbd +++ b/suites/swh/benchmarks/from-disk-postgres.pbd @@ -1,3 +1,4 @@ [meta] format = "0" +name = "swh.git-loader.from-disk.postgres" method = "simple-command" @@ -3,5 +4,4 @@ method = "simple-command" -name = "swh.git-loader.from-disk.postgres" [run] copy-data-env = false @@ -10,6 +10,4 @@ TARGET_REPO = "DATA-VARS:swh.swh-loader-git.target-repo-path" [simple-command] -command = """swh -l ERROR loader -C "$POULPE_TMP_DIR/loader-conf.yml" run git "$TARGET_REPO" > /dev/null""" - gather-info = """ @@ -15,9 +13,6 @@ gather-info = """ -# TODO Use a separate pgcluster for this to make sure we don't do anything stupid -# TODO This requires CREATE DATABASE privileges anyway and no authentication, -# TODO so no huge risk right now dropdb poulpe-swh-git-loader || true createdb poulpe-swh-git-loader postgres_server_version=$(psql poulpe-swh-git-loader -t -c 'SHOW server_version;') postgres_client_version=$(psql --version) dropdb poulpe-swh-git-loader @@ -19,7 +14,7 @@ dropdb poulpe-swh-git-loader || true createdb poulpe-swh-git-loader postgres_server_version=$(psql poulpe-swh-git-loader -t -c 'SHOW server_version;') postgres_client_version=$(psql --version) dropdb poulpe-swh-git-loader -cat > "$POULPE_TMP_DIR/gathered_info.json" << EOF +cat > \"$POULPE_TMP_DIR/gathered_info.json\" << EOF { @@ -25,6 +20,6 @@ { - "postgres": { - "server_version": "$postgres_server_version", - "client_version": "$postgres_client_version" + \"postgres\": { + \"server_version\": \"$postgres_server_version\", + \"client_version\": \"$postgres_client_version\" } } @@ -29,11 +24,10 @@ } } -EOF -""" - +EOF""" +command = "swh -l ERROR loader -C \"$POULPE_TMP_DIR/loader-conf.yml\" run git \"$TARGET_REPO\" > /dev/null" prepare-run = """ dropdb poulpe-swh-git-loader 2>/dev/null || true createdb poulpe-swh-git-loader swh -l ERROR db init-admin -d postgresql://localhost/poulpe-swh-git-loader storage >/dev/null 2>&1 swh -l ERROR db init -d postgresql://localhost/poulpe-swh-git-loader storage >/dev/null 2>&1 @@ -34,10 +28,10 @@ prepare-run = """ dropdb poulpe-swh-git-loader 2>/dev/null || true createdb poulpe-swh-git-loader swh -l ERROR db init-admin -d postgresql://localhost/poulpe-swh-git-loader storage >/dev/null 2>&1 swh -l ERROR db init -d postgresql://localhost/poulpe-swh-git-loader storage >/dev/null 2>&1 -cat > "$POULPE_TMP_DIR/loader-conf.yml" << EOF +cat > \"$POULPE_TMP_DIR/loader-conf.yml\" << EOF storage: cls: pipeline steps: @@ -59,5 +53,5 @@ url: http://127.0.0.1:5002 EOF -mkdir "$POULPE_TMP_DIR/objstorage" +mkdir \"$POULPE_TMP_DIR/objstorage\" @@ -63,8 +57,10 @@ -cat > "$POULPE_TMP_DIR/storage-conf.yml" << EOF +# XXX run that one with python3.11 not pypy + +cat > \"$POULPE_TMP_DIR/storage-conf.yml\" << EOF storage: cls: postgresql db: postgresql://localhost/poulpe-swh-git-loader objstorage: cls: pathslicing root: $POULPE_TMP_DIR/objstorage @@ -65,11 +61,11 @@ storage: cls: postgresql db: postgresql://localhost/poulpe-swh-git-loader objstorage: cls: pathslicing root: $POULPE_TMP_DIR/objstorage - slicing: "0:2/2:4/4:6" + slicing: \"0:2/2:4/4:6\" EOF # TODO process groups? @@ -72,8 +68,8 @@ EOF # TODO process groups? -swh -l ERROR storage -C "$POULPE_TMP_DIR/storage-conf.yml" rpc-serve >/dev/null 2>&1 & -echo "$!" > "$POULPE_TMP_DIR/rpc-serve-pid" +swh -l ERROR storage -C \"$POULPE_TMP_DIR/storage-conf.yml\" rpc-serve >/dev/null 2>&1 & +echo \"$!\" > \"$POULPE_TMP_DIR/rpc-serve-pid\" # in the preload case, load the data so that the benchmark has nothing left to do @@ -78,5 +74,5 @@ # in the preload case, load the data so that the benchmark has nothing left to do -if [ PRELOAD == "all" ]; then - swh -l ERROR loader -C "$POULPE_TMP_DIR/loader-conf.yml" run git "$TARGET_REPO" > /dev/null +if [ PRELOAD == \"all\" ]; then + swh -l ERROR loader -C \"$POULPE_TMP_DIR/loader-conf.yml\" run git \"$TARGET_REPO\" > /dev/null fi @@ -82,5 +78,3 @@ fi -sync -""" - +sync""" cleanup-run = """ @@ -86,3 +80,3 @@ cleanup-run = """ -cat "$POULPE_TMP_DIR/rpc-serve-pid" | xargs kill +cat \"$POULPE_TMP_DIR/rpc-serve-pid\" | xargs kill dropdb poulpe-swh-git-loader 2>/dev/null || true @@ -88,7 +82,6 @@ dropdb poulpe-swh-git-loader 2>/dev/null || true -rm -rf "$POULPE_TMP_DIR" -mkdir --parents "$POULPE_TMP_DIR" -""" +rm -rf \"$POULPE_TMP_DIR\" +mkdir --parents \"$POULPE_TMP_DIR\"""" [simple-command.variants.dimensions.preload.none] default = true @@ -92,5 +85,4 @@ [simple-command.variants.dimensions.preload.none] default = true -environment.PRELOAD = "none" @@ -96,3 +88,6 @@ -[simple-command.variants.dimensions.preload.all] -environment.PRELOAD = "all" +[simple-command.variants.dimensions.preload.none.environment] +PRELOAD = "none" + +[simple-command.variants.dimensions.preload.all.environment] +PRELOAD = "all" diff --git a/suites/swh/benchmarks/from-disk-postgres.pkl b/suites/swh/benchmarks/from-disk-postgres.pkl new file mode 100644 index 0000000000000000000000000000000000000000..b10ae44f19bf99dd82c4f4054537aca94dbbf0c9_c3VpdGVzL3N3aC9iZW5jaG1hcmtzL2Zyb20tZGlzay1wb3N0Z3Jlcy5wa2w= --- /dev/null +++ b/suites/swh/benchmarks/from-disk-postgres.pkl @@ -0,0 +1,116 @@ +amends "lib/simple-command.pkl" + +name = "swh.git-loader.from-disk.postgres" + +// we do not modify the loaded project +copy_data_env = false + +local db_name = "poulpe-swh-git-loader" +local batch_item_size = 100 +local batch_bytes_size = 52428800 +local loader_conf_path = "loader-conf.yml" +local storage_conf_path = "storage-conf.yml" +local pids_file = "rpc-serve-pid" + +environment { + ["TARGET_REPO"] = "DATA-VARS:swh.swh-loader-git.target-repo-path" +} + +command = """ +swh -l ERROR loader -C "$POULPE_TMP_DIR/\(loader_conf_path)" run git "$TARGET_REPO" > /dev/null +""" + +// TODO Use a separate pgcluster for this to make sure we don't do anything stupid +// TODO This requires CREATE DATABASE privileges anyway and no authentication, +// TODO so no huge risk right now +gather_info = """ +dropdb \(db_name) || true +createdb \(db_name) +postgres_server_version=$(psql \(db_name) -t -c 'SHOW server_version;') +postgres_client_version=$(psql --version) +dropdb \(db_name) +cat > "$POULPE_TMP_DIR/gathered_info.json" << EOF +{ + "postgres": { + "server_version": "$postgres_server_version", + "client_version": "$postgres_client_version" + } +} +EOF +""" + +prepare_run = """ +dropdb \(db_name) 2>/dev/null || true +createdb \(db_name) +swh -l ERROR db init-admin -d postgresql://localhost/\(db_name) storage >/dev/null 2>&1 +swh -l ERROR db init -d postgresql://localhost/\(db_name) storage >/dev/null 2>&1 + +cat > "$POULPE_TMP_DIR/\(loader_conf_path)" << EOF +storage: + cls: pipeline + steps: + - cls: buffer + min_batch_size: + content: \(batch_item_size) + content_bytes: \(batch_bytes_size) + directory: \(batch_item_size) + directory_entries: \(batch_item_size * 5) + extid: \(batch_item_size) + release: \(batch_item_size) + release_bytes: \(batch_bytes_size) + revision: \(batch_item_size) + revision_bytes: \(batch_bytes_size) + revision_parents: \(batch_item_size * 2) + - cls: filter + - cls: retry + - cls: remote + url: http://127.0.0.1:5002 +EOF + +mkdir "$POULPE_TMP_DIR/objstorage" + +# XXX run that one with python3.11 not pypy + +cat > "$POULPE_TMP_DIR/\(storage_conf_path)" << EOF +storage: + cls: postgresql + db: postgresql://localhost/\(db_name) + objstorage: + cls: pathslicing + root: $POULPE_TMP_DIR/objstorage + slicing: "0:2/2:4/4:6" +EOF + +# TODO process groups? + +swh -l ERROR storage -C "$POULPE_TMP_DIR/\(storage_conf_path)" rpc-serve >/dev/null 2>&1 & +echo "$!" > "$POULPE_TMP_DIR/\(pids_file)" + +# in the preload case, load the data so that the benchmark has nothing left to do +if [ PRELOAD == "all" ]; then + swh -l ERROR loader -C "$POULPE_TMP_DIR/\(loader_conf_path)" run git "$TARGET_REPO" > /dev/null +fi +sync +""" + +cleanup_run = """ +cat "$POULPE_TMP_DIR/\(pids_file)" | xargs kill +dropdb \(db_name) 2>/dev/null || true +rm -rf "$POULPE_TMP_DIR" +mkdir --parents "$POULPE_TMP_DIR" +""" + +variants { + ["preload"] { + default_key = "none" + cases { + for (preload in List("none", "all")) { + [preload] { + environment { + ["PRELOAD"] = preload + } + } + } + } + } +} diff --git a/suites/hg/benchmarks/lib/simple-command.pkl b/suites/swh/benchmarks/lib/simple-command.pkl similarity index 40% copy from suites/hg/benchmarks/lib/simple-command.pkl copy to suites/swh/benchmarks/lib/simple-command.pkl index 1db916d2565448c26659e8f7b4508d5cb9ca12d1_c3VpdGVzL2hnL2JlbmNobWFya3MvbGliL3NpbXBsZS1jb21tYW5kLnBrbA==..b10ae44f19bf99dd82c4f4054537aca94dbbf0c9_c3VpdGVzL3N3aC9iZW5jaG1hcmtzL2xpYi9zaW1wbGUtY29tbWFuZC5wa2w= 100644 --- a/suites/hg/benchmarks/lib/simple-command.pkl +++ b/suites/swh/benchmarks/lib/simple-command.pkl @@ -1,2 +1,2 @@ amends "../../../_common/benchmarks/simple-command.pkl" -// just a local re-export for Mercurial, might contains mercurial specific content later. +// just a local re-export for SWH, might contains SWH specific content later.