Skip to content
Snippets Groups Projects
Commit 772ecf5b authored by Pierre-Yves David's avatar Pierre-Yves David :octopus:
Browse files

benchmark: rework the perf::unbundle benchmark

We no longer build the bundle on the fly, applying them on a stripped
repository. Instead we rely on a set of available bundle pre-generated from the
repository with extra changesets.
parent 3cfcc3aa
No related branches found
No related tags found
No related merge requests found
[meta]
format="0"
name="hg.perf.perf-unbundle"
name="hg.perf.unbundle"
method="mercurial-perf-extension"
......@@ -4,5 +4,6 @@
method="mercurial-perf-extension"
[run]
copy-data-env = true
......@@ -6,5 +7,8 @@
[run]
copy-data-env = true
[environment]
MAIN_REPO='DATA-VARS:mercurial.main-repo-path'
[hg-perf-ext]
command="perf::unbundle"
......@@ -9,7 +13,8 @@
[hg-perf-ext]
command="perf::unbundle"
cwd="DATA-VARS:mercurial.main-repo-path"
args = ['../tmp-bundle.hg']
setup-script = "echo $MAIN_REPO"
#setup-script = "echo $REPO; hg debugupdatecache -R $REPO"
args = ['--repository', 'DATA-VARS:mercurial.main-repo-path']
[hg-perf-ext.variants.dimensions.issue6528.default]
default=true
......@@ -22,6 +27,6 @@
args = ['--config', 'storage.revlog.issue6528.fix-incoming=no']
constraints.compatible-revision-range="2813d406b036::"
[hg-perf-ext.variants.dimensions.revs.last-100000]
setup-script = "hg bundle --type none-v2 ../tmp-bundle.hg --rev 'last(all(), 100000)' --base 'not last(all(), 100000)' ; hg --config extensions.strip= strip --no-backup 'last(all(), 100000)'; hg debugupdatecache"
[hg-perf-ext.variants.dimensions.revs.any-1-extra-rev]
args = ['DATA-VARS:mercurial.bundles.any-1-revs']
......@@ -27,4 +32,4 @@
[hg-perf-ext.variants.dimensions.revs.last-100000.constraints.graph]
visible-revision-count.min=100000
[hg-perf-ext.variants.dimensions.revs.any-10-extra-rev]
args = ['DATA-VARS:mercurial.bundles.any-10-revs']
......@@ -30,14 +35,7 @@
[hg-perf-ext.variants.dimensions.revs.last-40000]
default=true
setup-script = "hg bundle --type none-v2 ../tmp-bundle.hg --rev 'last(all(), 40000)' --base 'not last(all(), 40000)' ; hg --config extensions.strip= strip --no-backup 'last(all(), 40000)'; hg debugupdatecache"
[hg-perf-ext.variants.dimensions.revs.last-40000.constraints.graph]
visible-revision-count.min=40000
[hg-perf-ext.variants.dimensions.revs.last-10000]
setup-script = "hg bundle --type none-v2 ../tmp-bundle.hg --rev 'last(all(), 10000)' --base 'not last(all(), 10000)' ; hg --config extensions.strip= strip --no-backup 'last(all(), 10000)'; hg debugupdatecache"
[hg-perf-ext.variants.dimensions.revs.last-10000.constraints.graph]
visible-revision-count.min=10000
[hg-perf-ext.variants.dimensions.revs.any-100-extra-rev]
args = ['DATA-VARS:mercurial.bundles.any-100-revs']
# some 2**N behavior prevent reasonable pulling time for set of revision
# introduced in 23950e39281f, fixed in c9ceb4f60256
compatible-revision-range= 'not (23950e39281f:: and not c9ceb4f60256::)'
......@@ -43,11 +41,7 @@
[hg-perf-ext.variants.dimensions.revs.last-1000]
setup-script = "hg bundle --type none-v2 ../tmp-bundle.hg --rev 'last(all(), 1000)' --base 'not last(all(), 1000)' ; hg --config extensions.strip= strip --no-backup 'last(all(), 1000)'; hg debugupdatecache"
[hg-perf-ext.variants.dimensions.revs.last-1000.constraints.graph]
visible-revision-count.min=1000
[hg-perf-ext.variants.dimensions.revs.last-100]
setup-script = "hg bundle --type none-v2 ../tmp-bundle.hg --rev 'last(all(), 100)' --base 'not last(all(), 100)' ; hg --config extensions.strip= strip --no-backup 'last(all(), 100)'; hg debugupdatecache"
[hg-perf-ext.variants.dimensions.revs.last-100.constraints.graph]
visible-revision-count.min=100
[hg-perf-ext.variants.dimensions.revs.any-1000-extra-rev]
args = ['DATA-VARS:mercurial.bundles.any-1000-revs']
# some 2**N behavior prevent reasonable pulling time for set of revision
# introduced in 23950e39281f, fixed in c9ceb4f60256
compatible-revision-range= 'not (23950e39281f:: and not c9ceb4f60256::)'
......@@ -53,6 +47,8 @@
[hg-perf-ext.variants.dimensions.revs.last-10]
setup-script = "hg bundle --type none-v2 ../tmp-bundle.hg --rev 'last(all(), 10)' --base 'not last(all(), 10)' ; hg --config extensions.strip= strip --no-backup 'last(all(), 10)'; hg debugupdatecache"
[hg-perf-ext.variants.dimensions.revs.last-10.constraints.graph]
visible-revision-count.min=10
[hg-perf-ext.variants.dimensions.revs.any-10000-extra-rev]
args = ['DATA-VARS:mercurial.bundles.any-10000-revs']
default=true
# some 2**N behavior prevent reasonable pulling time for set of revision
# introduced in 23950e39281f, fixed in c9ceb4f60256
compatible-revision-range= 'not (23950e39281f:: and not c9ceb4f60256::)'
......@@ -58,5 +54,6 @@
[hg-perf-ext.variants.dimensions.revs.last-1]
setup-script = "hg bundle --type none-v2 ../tmp-bundle.hg --rev 'last(all(), 1)' --base 'not last(all(), 1)' ; hg --config extensions.strip= strip --no-backup 'last(all(), 1)'; hg debugupdatecache"
[hg-perf-ext.variants.dimensions.revs.last-1.constraints.graph]
visible-revision-count.min=1
[hg-perf-ext.variants.dimensions.revs.any-100000-extra-rev]
args = ['DATA-VARS:mercurial.bundles.any-100000-revs']
# some 2**N behavior prevent reasonable pulling time for set of revision
# introduced in 23950e39281f, fixed in c9ceb4f60256
compatible-revision-range= 'not (23950e39281f:: and not c9ceb4f60256::)'
#!/bin/bash
#
# create bundles data-brick for a repository
#
# It take three arguments
#
# 1) the directory where to create the data brick
# 2) the repository to will receive the bundle
# 3) the repository with extra changeset that will be used to create the
# bundles.
#
set -euo pipefail
export HGRCPATH=
directory=$1
source_repo=$2
extra_rev_repo=$3
all_extra_rev="$directory/all-extra-rev.txt"
brick_dir="$directory/bundles.brick"
if [ ! -d "$brick_dir" ]; then
mkdir $brick_dir
fi
brick_file="$brick_dir/bundle-bricks.toml"
echo "finding all extra revisions"
hg -R "$source_repo" incoming --quiet "$extra_rev_repo" -T '{node}\n' > "$all_extra_rev"
total_extra=`cat "$all_extra_rev" | wc -l`
echo "$total_extra revisions found"
cat << EOF > "$brick_file"
[poulpe.brick]
type="bundles"
version=0
[bundles]
EOF
for rev in 1 10 100 1000 10000 100000; do
if [ $total_extra -lt $rev ]; then
break;
fi
echo "generating bundle for $rev revisions"
tmp_file="$directory/$rev-extra-rev.txt"
head -n $rev "$all_extra_rev" > "$tmp_file"
hg -R "$extra_rev_repo" bundle --exact \
--rev 'nodefromfile("'"$tmp_file"'")' \
"$brick_dir/any-$rev-extra-rev.hg"
rm "$tmp_file"
echo "any-$rev-revs = "'"'"any-$rev-extra-rev.hg"'"' >> $brick_file
done
rm "$all_extra_rev"
#!/usr/bin/env python3
"""Script to setup a data environment for bundle related benchmarks
It consumes a "bundles" bricks.
Such brick can be created with the `suites/hg/brick-setup/create-bundle-brick`
script.
"""
import os
import pathlib
import shutil
import click
import poulpe.basics
class BundlesBrick:
def __init__(self, path):
self._path = path
desc_file = self._path / "brick-description.poulpe"
self._data = poulpe.basics.get_data(desc_file)
assert self._data['poulpe']['brick']['type'] == "bundles"
assert self._data['poulpe']['brick']['version'] == 0
def all_bundles(self):
for key, path in self._data["bundles"].items():
yield key, self._path / path
BUNDLE_DIR = "bundles"
KEY = "mercurial.bundles.%s"
PATH = f"{BUNDLE_DIR}/%s"
def process_brick(data_env, brick):
bundle_dir = data_env.path / BUNDLE_DIR
os.makedirs(bundle_dir, exist_ok=True)
for key, path in brick.all_bundles():
shutil.copyfile(path, bundle_dir / path.name)
data_env.set_benchmark_input_var(KEY % key, PATH % path.name)
@click.command()
@click.argument("data-env-path")
@click.argument("data-brick-path")
def setup_bundles(data_env_path, data_brick_path):
data_env_path = pathlib.Path(data_env_path)
data_env = poulpe.get_data_env(data_env_path)
brick_path = pathlib.Path(data_brick_path)
brick = BundlesBrick(brick_path)
process_brick(data_env, brick)
data_env.write()
if __name__ == "__main__":
setup_bundles()
......@@ -43,6 +43,5 @@
hg.perf.perf-delta-find
hg.perf.perf-startup
hg.perf.perf-stream-locked-section
hg.perf.perf-unbundle
hg.perf.phase-cache
hg.perf.read-revisions
......@@ -47,5 +46,6 @@
hg.perf.phase-cache
hg.perf.read-revisions
hg.perf.unbundle
Showing path
------------
......@@ -87,9 +87,7 @@
benchmarks/hg/perf-startup.pbd
hg.perf.perf-stream-locked-section
benchmarks/hg/perf-stream-locked.pbd
hg.perf.perf-unbundle
benchmarks/hg/perf-unbundle.pbd
hg.perf.phase-cache
benchmarks/hg/perf-phases-cache.pbd
hg.perf.read-revisions
benchmarks/hg/perf-read-revisions.pbd
......@@ -92,7 +90,9 @@
hg.perf.phase-cache
benchmarks/hg/perf-phases-cache.pbd
hg.perf.read-revisions
benchmarks/hg/perf-read-revisions.pbd
hg.perf.unbundle
benchmarks/hg/perf-unbundle.pbd
listing variants
----------------
......@@ -265,19 +265,6 @@
- manifest-merge-long-chain-both-e
hg.perf.perf-startup
hg.perf.perf-stream-locked-section
hg.perf.perf-unbundle
* issue6528:
- default (default)
- fix-incoming
- disabled
* revs:
- last-100000
- last-40000 (default)
- last-10000
- last-1000
- last-100
- last-10
- last-1
hg.perf.phase-cache
* full:
- no (default)
......@@ -286,6 +273,18 @@
* order:
- default (default)
- reverse
hg.perf.unbundle
* issue6528:
- default (default)
- fix-incoming
- disabled
* revs:
- any-1-extra-rev
- any-10-extra-rev
- any-100-extra-rev
- any-1000-extra-rev
- any-10000-extra-rev (default)
- any-100000-extra-rev
with a patterns
--------------
......@@ -299,6 +298,5 @@
hg.perf.perf-delta-find
hg.perf.perf-startup
hg.perf.perf-stream-locked-section
hg.perf.perf-unbundle
hg.perf.phase-cache
hg.perf.read-revisions
......@@ -303,5 +301,6 @@
hg.perf.phase-cache
hg.perf.read-revisions
hg.perf.unbundle
multiple pattern
......@@ -317,6 +316,5 @@
hg.perf.perf-delta-find
hg.perf.perf-startup
hg.perf.perf-stream-locked-section
hg.perf.perf-unbundle
hg.perf.phase-cache
hg.perf.read-revisions
......@@ -321,7 +319,8 @@
hg.perf.phase-cache
hg.perf.read-revisions
hg.perf.unbundle
regex pattern
$ poulpe den-list-benchmarks 're:.*bundle$'
hg.perf.perf-bundle
......@@ -323,7 +322,7 @@
regex pattern
$ poulpe den-list-benchmarks 're:.*bundle$'
hg.perf.perf-bundle
hg.perf.perf-unbundle
hg.perf.unbundle
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment