Skip to content
Snippets Groups Projects
Commit d99e66b6e060 authored by Boris Feld's avatar Boris Feld
Browse files

Try reducing the memory consumption by reducing the number of clones

parent b18455e246a6
No related branches found
No related tags found
No related merge requests found
......@@ -6,5 +6,6 @@
import tempfile
import shutil
import subprocess
import shlex
from .utils import (BaseNChangesetsTestSuite, BaseTimeTestSuite,
......@@ -9,6 +10,6 @@
from .utils import (BaseNChangesetsTestSuite, BaseTimeTestSuite,
BaseTrackTestSuite, median)
BaseTrackTestSuite, median, REPOS_DIR)
class TestSuite(BaseTrackTestSuite):
......@@ -137,8 +138,56 @@
class ExchangeTrackSuite(BaseTrackTestSuite):
param_names = BaseTrackTestSuite.param_names + ['single_rev']
params = BaseTrackTestSuite.params + [[True, False]]
param_names = BaseTrackTestSuite.param_names + ['single_rev', 'revset']
params = BaseTrackTestSuite.params + [[True, False]] + [['default~10']]
def setup(self, repo_name, single_rev, revset, *args, **kwargs):
super(ExchangeTrackSuite, self).setup(repo_name, *args, **kwargs)
self.single_rev = single_rev
from base64 import b64encode
self.safe_range = b64encode(revset)
self.tempdir = tempfile.mkdtemp(dir=os.environ.get('ASVTEMPDIR'))
if single_rev is False:
self.cache_name = "%s.tar" % repo_name
self.cache_path = os.path.join(self.tempdir, self.cache_name)
if not os.path.isfile(self.cache_path):
tar_cmd = "tar cf %s -C %s ." % (self.cache_path, self.repo_path)
self._single_execute(shlex.split(tar_cmd))
else:
self.cache_name = "%s-%s-%s.tar" % (repo_name, single_rev, self.safe_range)
self.cache_path = os.path.join(self.tempdir, self.cache_name)
if not os.path.isfile(self.cache_path):
# First clone it
clone_path = os.path.join(self.tempdir, "%s-%s-clone" % (repo_name, self.safe_range))
rev = self._identify_id(revset)
cmd = [self.hgpath, '--cwd', self.repo_path,
'clone', '--noupdate', '.', clone_path,
'-r', rev]
self._single_execute(cmd)
tar_cmd = "tar cf %s -C %s ." % (self.cache_path, clone_path)
self._single_execute(shlex.split(tar_cmd))
shutil.rmtree(clone_path)
def teardown(self, *args, **kwargs):
try:
shutil.rmtree(self.tempdir)
except Exception:
pass
def _identify_id(self, revset):
rev = subprocess.check_output([
self.hgpath, '--cwd', self.repo_path, 'identify', '-i', "-r", revset], env=self.environ)
return rev.strip()
@contextlib.contextmanager
def clone(self, revset=None):
......@@ -142,5 +191,4 @@
@contextlib.contextmanager
def clone(self, revset=None):
tempdir = tempfile.mkdtemp(dir=os.environ.get('ASVTEMPDIR'))
try:
......@@ -146,5 +194,5 @@
try:
clone_path = os.path.join(tempdir, 'clone')
clone_path = os.path.join(self.tempdir, 'clone')
cmd = [self.hgpath, '--cwd', self.repo_path,
'clone', '--noupdate', '.', clone_path]
if revset is not None:
......@@ -148,10 +196,8 @@
cmd = [self.hgpath, '--cwd', self.repo_path,
'clone', '--noupdate', '.', clone_path]
if revset is not None:
rev = subprocess.check_output([
self.hgpath, '--cwd', self.repo_path, 'log', '-T',
'{node}', '-r', revset, '--limit', '1'], env=self.environ)
rev = self._identify_id(revset)
cmd.extend(['-r', rev])
clone_time = self._single_execute(cmd)
yield clone_path, clone_time
finally:
......@@ -154,6 +200,14 @@
cmd.extend(['-r', rev])
clone_time = self._single_execute(cmd)
yield clone_path, clone_time
finally:
shutil.rmtree(tempdir)
shutil.rmtree(clone_path)
@contextlib.contextmanager
def clone_from_cache(self):
try:
import time
clone_path = os.path.join(self.tempdir, 'clone-%s' % time.time())
os.mkdir(clone_path)
......@@ -159,6 +213,14 @@
def track_incoming(self, repo_name, single_rev):
with self.clone(revset='default~10') as (clone_path, _):
cmd = [self.hgpath, '--cwd', clone_path, 'incoming']
cmd = "tar xf %s -C %s" % (self.cache_path, clone_path)
clone_time = self._single_execute(shlex.split(cmd))
yield clone_path
finally:
shutil.rmtree(clone_path)
def track_incoming(self, repo_name, single_rev, revset):
with self.clone_from_cache() as clone_path:
cmd = [self.hgpath, '--cwd', clone_path, 'incoming', self.repo_path]
if single_rev:
cmd.extend(['-r', 'default'])
......@@ -163,4 +225,3 @@
if single_rev:
cmd.extend(['-r', 'default'])
return self._timeit(self._single_execute, (cmd,))
......@@ -166,7 +227,10 @@
def track_outgoing(self, repo_name, single_rev):
with self.clone(revset='default~10') as (clone_path, _):
# Ignore the return code
return self._timeit(self._single_execute, (cmd, False))
def track_outgoing(self, repo_name, single_rev, revset):
with self.clone_from_cache() as clone_path:
cmd = [self.hgpath, '--cwd', self.repo_path, 'outgoing',
clone_path]
if single_rev:
cmd.extend(['-r', 'default'])
......@@ -169,6 +233,8 @@
cmd = [self.hgpath, '--cwd', self.repo_path, 'outgoing',
clone_path]
if single_rev:
cmd.extend(['-r', 'default'])
return self._timeit(self._single_execute, (cmd,))
# Ignore the return code
return self._timeit(self._single_execute, (cmd, False))
......@@ -174,3 +240,3 @@
def track_push(self, repo_name, single_rev):
def track_push(self, repo_name, single_rev, revset):
def clone_and_push():
......@@ -176,6 +242,6 @@
def clone_and_push():
with self.clone(revset='default~10') as (clone_path, _):
with self.clone_from_cache() as clone_path:
cmd = [self.hgpath, '--cwd', self.repo_path, 'push', '-f',
clone_path]
if single_rev:
cmd.extend(['-r', 'default'])
......@@ -178,7 +244,7 @@
cmd = [self.hgpath, '--cwd', self.repo_path, 'push', '-f',
clone_path]
if single_rev:
cmd.extend(['-r', 'default'])
return self._single_execute(cmd)
return self._single_execute(cmd, False)
return self._timeit(clone_and_push)
......@@ -183,6 +249,6 @@
return self._timeit(clone_and_push)
def track_clone(self, repo_name, single_rev):
def track_clone(self, repo_name, single_rev, revset):
revset = 'default' if single_rev else None
def clone():
......
......@@ -164,5 +164,5 @@
times.append(func(*args, **kwargs))
return median(times)
def _execute(self, command, *args):
def _execute(self, command, check_returncode=True, *args):
cmd = self._prepare_cmd(command, *args)
......@@ -168,5 +168,5 @@
cmd = self._prepare_cmd(command, *args)
return self._timeit(self._single_execute, (cmd,))
return self._timeit(self._single_execute, (cmd, check_returncode))
def _prepare_cmd(self, command, *args):
cmd = [
......@@ -177,9 +177,19 @@
return cmd
def _single_execute(self, cmd):
try:
before = time.time()
subprocess.check_output(cmd, stderr=subprocess.STDOUT,
env=self.environ)
def _single_execute(self, cmd, check_returncode=True):
before = time.time()
if check_returncode is True:
try:
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT,
env=self.environ)
after = time.time()
duration = after - before
return duration
except subprocess.CalledProcessError as e:
raise
else:
subprocess.call(cmd, stderr=subprocess.STDOUT,
env=self.environ)
after = time.time()
......@@ -185,8 +195,8 @@
after = time.time()
return after - before
except subprocess.CalledProcessError as e:
print("OUTPUT", e.output)
raise
duration = after - before
return duration
class BaseTimeTestSuite(BaseTestSuite):
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment