Skip to content
Snippets Groups Projects
Commit f08ad5d1 authored by Pierre-Yves David's avatar Pierre-Yves David
Browse files

revsetbenchmarks: allow running multiple variants per revset

The current benchmarks were only testing the whole iteration. This is suboptimal
because some changes are meaningful for things like first result, minimum or
sorting.

We introduce a "variants" feature that let you systematically add some variants
to all revsets tested.

A typical variants value would be 'plain,min,last,sort'. When testing 'all()' it
will also provide testing for:
- all()
- min(all())
- last(all())
- sort(sort)

and output:

   plain         min            last           sort
0) 0.034568      0.037857       0.000074       0.034238
1) 0.011358  32% 0.020181  53%  0.000080 108%  0.011405  33%

Using revsets (who hit the API) instead of the internal API add some overhead,
but the overhead should be the same everywhere so it still allow comparison.
This is is more simple to implement and allows comparison with older versions
who do not have the same API.
parent 460922c9
No related branches found
No related tags found
No related merge requests found
......@@ -16,6 +16,8 @@
# cannot use argparse, python 2.7 only
from optparse import OptionParser
DEFAULTVARIANTS = ['plain']
def check_output(*args, **kwargs):
kwargs.setdefault('stderr', PIPE)
kwargs.setdefault('stdout', PIPE)
......@@ -144,6 +146,6 @@
return 'x%ix%i' % (factor, order)
_marker = object()
def printresult(idx, data, maxidx, verbose=False, reference=_marker):
def printresult(variants, idx, data, maxidx, verbose=False, reference=_marker):
"""print a line of result to stdout"""
mask = '%%0%ii) %%s' % idxwidth(maxidx)
......@@ -148,15 +150,17 @@
"""print a line of result to stdout"""
mask = '%%0%ii) %%s' % idxwidth(maxidx)
out = ['%10.6f' % data['wall']]
if reference is not _marker:
factor = None
if reference is not None:
factor = getfactor(reference, data, 'wall')
out.append(formatfactor(factor))
if verbose:
out.append('%10.6f' % data['comb'])
out.append('%10.6f' % data['user'])
out.append('%10.6f' % data['sys'])
out.append('%6d' % data['count'])
out = []
for var in variants:
out.append('%10.6f' % data[var]['wall'])
if reference is not _marker:
factor = None
if reference is not None:
factor = getfactor(reference[var], data[var], 'wall')
out.append(formatfactor(factor))
if verbose:
out.append('%10.6f' % data[var]['comb'])
out.append('%10.6f' % data[var]['user'])
out.append('%10.6f' % data[var]['sys'])
out.append('%6d' % data[var]['count'])
print mask % (idx, ' '.join(out))
......@@ -161,15 +165,18 @@
print mask % (idx, ' '.join(out))
def printheader(maxidx, verbose=False, relative=False):
header = [' ' * (idxwidth(maxidx) + 1),
' %-8s' % 'time']
if relative:
header.append(' ')
if verbose:
header.append(' %-8s' % 'comb')
header.append(' %-8s' % 'user')
header.append(' %-8s' % 'sys')
header.append('%6s' % 'count')
def printheader(variants, maxidx, verbose=False, relative=False):
header = [' ' * (idxwidth(maxidx) + 1)]
for var in variants:
if not var:
var = 'iter'
header.append(' %-8s' % var)
if relative:
header.append(' ')
if verbose:
header.append(' %-8s' % 'comb')
header.append(' %-8s' % 'user')
header.append(' %-8s' % 'sys')
header.append('%6s' % 'count')
print ' '.join(header)
def getrevs(spec):
......@@ -182,6 +189,12 @@
return [r for r in out.split() if r]
def applyvariants(revset, variant):
if variant == 'plain':
return revset
return '%s(%s)' % (variant, revset)
parser = OptionParser(usage="usage: %prog [options] <revs>")
parser.add_option("-f", "--file",
help="read revset from FILE (stdin if omitted)",
......@@ -193,6 +206,11 @@
action='store_true',
help="display all timing data (not just best total time)")
parser.add_option("", "--variants",
default=','.join(DEFAULTVARIANTS),
help="comma separated list of variant to test "
"(eg: plain,min,sorted) (plain = no modification)")
(options, args) = parser.parse_args()
if not args:
......@@ -221,6 +239,8 @@
for a in args:
revs.extend(getrevs(a))
variants = options.variants.split(',')
results = []
for r in revs:
print "----------------------------"
......@@ -229,5 +249,5 @@
update(r)
res = []
results.append(res)
printheader(len(revsets), verbose=options.verbose)
printheader(variants, len(revsets), verbose=options.verbose)
for idx, rset in enumerate(revsets):
......@@ -233,7 +253,12 @@
for idx, rset in enumerate(revsets):
data = perf(rset, target=options.repo)
res.append(data)
printresult(idx, data, len(revsets), verbose=options.verbose)
varres = {}
for var in variants:
varrset = applyvariants(rset, var)
data = perf(varrset, target=options.repo)
varres[var] = data
res.append(varres)
printresult(variants, idx, varres, len(revsets),
verbose=options.verbose)
sys.stdout.flush()
print "----------------------------"
......@@ -256,6 +281,6 @@
for ridx, rset in enumerate(revsets):
print "revset #%i: %s" % (ridx, rset)
printheader(len(results), verbose=options.verbose, relative=True)
printheader(variants, len(results), verbose=options.verbose, relative=True)
ref = None
for idx, data in enumerate(results):
......@@ -260,6 +285,6 @@
ref = None
for idx, data in enumerate(results):
printresult(idx, data[ridx], len(results), verbose=options.verbose,
reference=ref)
printresult(variants, idx, data[ridx], len(results),
verbose=options.verbose, reference=ref)
ref = data[ridx]
print
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment