Commit 6fa5f68b authored by Augie Fackler's avatar Augie Fackler
Browse files

formatting: blacken the codebase

This is using my patch to black
(https://github.com/psf/black/pull/826) so we don't un-wrap collection
literals.

Done with:

  hg files 'set:**.py - mercurial/thirdparty/** - "contrib/python-zstandard/**"' | xargs black -S

# skip-blame mass-reformatting only

# no-check-commit reformats foo_bar functions

Differential Revision: https://phab.mercurial-scm.org/D6971
parent 57875cf423c9
......@@ -36,8 +36,13 @@ def bootstrap():
pip = venv_bin / 'pip'
python = venv_bin / 'python'
args = [str(pip), 'install', '-r', str(REQUIREMENTS_TXT),
'--disable-pip-version-check']
args = [
str(pip),
'install',
'-r',
str(REQUIREMENTS_TXT),
'--disable-pip-version-check',
]
if not venv_created:
args.append('-q')
......@@ -45,8 +50,7 @@ def bootstrap():
subprocess.run(args, check=True)
os.environ['HGAUTOMATION_BOOTSTRAPPED'] = '1'
os.environ['PATH'] = '%s%s%s' % (
venv_bin, os.pathsep, os.environ['PATH'])
os.environ['PATH'] = '%s%s%s' % (venv_bin, os.pathsep, os.environ['PATH'])
subprocess.run([str(python), __file__] + sys.argv[1:], check=True)
......
......@@ -10,9 +10,7 @@
import pathlib
import secrets
from .aws import (
AWSConnection,
)
from .aws import AWSConnection
class HGAutomation:
......@@ -53,7 +51,7 @@ class HGAutomation:
return password
def aws_connection(self, region: str, ensure_ec2_state: bool=True):
def aws_connection(self, region: str, ensure_ec2_state: bool = True):
"""Obtain an AWSConnection instance bound to a specific region."""
return AWSConnection(self, region, ensure_ec2_state=ensure_ec2_state)
This diff is collapsed.
......@@ -22,12 +22,15 @@ from . import (
)
SOURCE_ROOT = pathlib.Path(os.path.abspath(__file__)).parent.parent.parent.parent
SOURCE_ROOT = pathlib.Path(
os.path.abspath(__file__)
).parent.parent.parent.parent
DIST_PATH = SOURCE_ROOT / 'dist'
def bootstrap_linux_dev(hga: HGAutomation, aws_region, distros=None,
parallel=False):
def bootstrap_linux_dev(
hga: HGAutomation, aws_region, distros=None, parallel=False
):
c = hga.aws_connection(aws_region)
if distros:
......@@ -59,8 +62,9 @@ def bootstrap_windows_dev(hga: HGAutomation, aws_region, base_image_name):
print('Windows development AMI available as %s' % image.id)
def build_inno(hga: HGAutomation, aws_region, arch, revision, version,
base_image_name):
def build_inno(
hga: HGAutomation, aws_region, arch, revision, version, base_image_name
):
c = hga.aws_connection(aws_region)
image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
DIST_PATH.mkdir(exist_ok=True)
......@@ -71,13 +75,14 @@ def build_inno(hga: HGAutomation, aws_region, arch, revision, version,
windows.synchronize_hg(SOURCE_ROOT, revision, instance)
for a in arch:
windows.build_inno_installer(instance.winrm_client, a,
DIST_PATH,
version=version)
windows.build_inno_installer(
instance.winrm_client, a, DIST_PATH, version=version
)
def build_wix(hga: HGAutomation, aws_region, arch, revision, version,
base_image_name):
def build_wix(
hga: HGAutomation, aws_region, arch, revision, version, base_image_name
):
c = hga.aws_connection(aws_region)
image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
DIST_PATH.mkdir(exist_ok=True)
......@@ -88,12 +93,14 @@ def build_wix(hga: HGAutomation, aws_region, arch, revision, version,
windows.synchronize_hg(SOURCE_ROOT, revision, instance)
for a in arch:
windows.build_wix_installer(instance.winrm_client, a,
DIST_PATH, version=version)
windows.build_wix_installer(
instance.winrm_client, a, DIST_PATH, version=version
)
def build_windows_wheel(hga: HGAutomation, aws_region, arch, revision,
base_image_name):
def build_windows_wheel(
hga: HGAutomation, aws_region, arch, revision, base_image_name
):
c = hga.aws_connection(aws_region)
image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
DIST_PATH.mkdir(exist_ok=True)
......@@ -107,8 +114,9 @@ def build_windows_wheel(hga: HGAutomation, aws_region, arch, revision,
windows.build_wheel(instance.winrm_client, a, DIST_PATH)
def build_all_windows_packages(hga: HGAutomation, aws_region, revision,
version, base_image_name):
def build_all_windows_packages(
hga: HGAutomation, aws_region, revision, version, base_image_name
):
c = hga.aws_connection(aws_region)
image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
DIST_PATH.mkdir(exist_ok=True)
......@@ -124,11 +132,13 @@ def build_all_windows_packages(hga: HGAutomation, aws_region, revision,
windows.purge_hg(winrm_client)
windows.build_wheel(winrm_client, arch, DIST_PATH)
windows.purge_hg(winrm_client)
windows.build_inno_installer(winrm_client, arch, DIST_PATH,
version=version)
windows.build_inno_installer(
winrm_client, arch, DIST_PATH, version=version
)
windows.purge_hg(winrm_client)
windows.build_wix_installer(winrm_client, arch, DIST_PATH,
version=version)
windows.build_wix_installer(
winrm_client, arch, DIST_PATH, version=version
)
def terminate_ec2_instances(hga: HGAutomation, aws_region):
......@@ -141,8 +151,15 @@ def purge_ec2_resources(hga: HGAutomation, aws_region):
aws.remove_resources(c)
def run_tests_linux(hga: HGAutomation, aws_region, instance_type,
python_version, test_flags, distro, filesystem):
def run_tests_linux(
hga: HGAutomation,
aws_region,
instance_type,
python_version,
test_flags,
distro,
filesystem,
):
c = hga.aws_connection(aws_region)
image = aws.ensure_linux_dev_ami(c, distro=distro)
......@@ -151,17 +168,17 @@ def run_tests_linux(hga: HGAutomation, aws_region, instance_type,
ensure_extra_volume = filesystem not in ('default', 'tmpfs')
with aws.temporary_linux_dev_instances(
c, image, instance_type,
ensure_extra_volume=ensure_extra_volume) as insts:
c, image, instance_type, ensure_extra_volume=ensure_extra_volume
) as insts:
instance = insts[0]
linux.prepare_exec_environment(instance.ssh_client,
filesystem=filesystem)
linux.prepare_exec_environment(
instance.ssh_client, filesystem=filesystem
)
linux.synchronize_hg(SOURCE_ROOT, instance, '.')
t_prepared = time.time()
linux.run_tests(instance.ssh_client, python_version,
test_flags)
linux.run_tests(instance.ssh_client, python_version, test_flags)
t_done = time.time()
t_setup = t_prepared - t_start
......@@ -169,29 +186,48 @@ def run_tests_linux(hga: HGAutomation, aws_region, instance_type,
print(
'total time: %.1fs; setup: %.1fs; tests: %.1fs; setup overhead: %.1f%%'
% (t_all, t_setup, t_done - t_prepared, t_setup / t_all * 100.0))
% (t_all, t_setup, t_done - t_prepared, t_setup / t_all * 100.0)
)
def run_tests_windows(hga: HGAutomation, aws_region, instance_type,
python_version, arch, test_flags, base_image_name):
def run_tests_windows(
hga: HGAutomation,
aws_region,
instance_type,
python_version,
arch,
test_flags,
base_image_name,
):
c = hga.aws_connection(aws_region)
image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
with aws.temporary_windows_dev_instances(c, image, instance_type,
disable_antivirus=True) as insts:
with aws.temporary_windows_dev_instances(
c, image, instance_type, disable_antivirus=True
) as insts:
instance = insts[0]
windows.synchronize_hg(SOURCE_ROOT, '.', instance)
windows.run_tests(instance.winrm_client, python_version, arch,
test_flags)
windows.run_tests(
instance.winrm_client, python_version, arch, test_flags
)
def publish_windows_artifacts(hg: HGAutomation, aws_region, version: str,
pypi: bool, mercurial_scm_org: bool,
ssh_username: str):
windows.publish_artifacts(DIST_PATH, version,
pypi=pypi, mercurial_scm_org=mercurial_scm_org,
ssh_username=ssh_username)
def publish_windows_artifacts(
hg: HGAutomation,
aws_region,
version: str,
pypi: bool,
mercurial_scm_org: bool,
ssh_username: str,
):
windows.publish_artifacts(
DIST_PATH,
version,
pypi=pypi,
mercurial_scm_org=mercurial_scm_org,
ssh_username=ssh_username,
)
def run_try(hga: HGAutomation, aws_region: str, rev: str):
......@@ -208,25 +244,21 @@ def get_parser():
help='Path for local state files',
)
parser.add_argument(
'--aws-region',
help='AWS region to use',
default='us-west-2',
'--aws-region', help='AWS region to use', default='us-west-2',
)
subparsers = parser.add_subparsers()
sp = subparsers.add_parser(
'bootstrap-linux-dev',
help='Bootstrap Linux development environments',
'bootstrap-linux-dev', help='Bootstrap Linux development environments',
)
sp.add_argument(
'--distros',
help='Comma delimited list of distros to bootstrap',
'--distros', help='Comma delimited list of distros to bootstrap',
)
sp.add_argument(
'--parallel',
action='store_true',
help='Generate AMIs in parallel (not CTRL-c safe)'
help='Generate AMIs in parallel (not CTRL-c safe)',
)
sp.set_defaults(func=bootstrap_linux_dev)
......@@ -242,17 +274,13 @@ def get_parser():
sp.set_defaults(func=bootstrap_windows_dev)
sp = subparsers.add_parser(
'build-all-windows-packages',
help='Build all Windows packages',
'build-all-windows-packages', help='Build all Windows packages',
)
sp.add_argument(
'--revision',
help='Mercurial revision to build',
default='.',
'--revision', help='Mercurial revision to build', default='.',
)
sp.add_argument(
'--version',
help='Mercurial version string to use',
'--version', help='Mercurial version string to use',
)
sp.add_argument(
'--base-image-name',
......@@ -262,8 +290,7 @@ def get_parser():
sp.set_defaults(func=build_all_windows_packages)
sp = subparsers.add_parser(
'build-inno',
help='Build Inno Setup installer(s)',
'build-inno', help='Build Inno Setup installer(s)',
)
sp.add_argument(
'--arch',
......@@ -273,13 +300,10 @@ def get_parser():
default=['x64'],
)
sp.add_argument(
'--revision',
help='Mercurial revision to build',
default='.',
'--revision', help='Mercurial revision to build', default='.',
)
sp.add_argument(
'--version',
help='Mercurial version string to use in installer',
'--version', help='Mercurial version string to use in installer',
)
sp.add_argument(
'--base-image-name',
......@@ -289,8 +313,7 @@ def get_parser():
sp.set_defaults(func=build_inno)
sp = subparsers.add_parser(
'build-windows-wheel',
help='Build Windows wheel(s)',
'build-windows-wheel', help='Build Windows wheel(s)',
)
sp.add_argument(
'--arch',
......@@ -300,9 +323,7 @@ def get_parser():
default=['x64'],
)
sp.add_argument(
'--revision',
help='Mercurial revision to build',
default='.',
'--revision', help='Mercurial revision to build', default='.',
)
sp.add_argument(
'--base-image-name',
......@@ -311,10 +332,7 @@ def get_parser():
)
sp.set_defaults(func=build_windows_wheel)
sp = subparsers.add_parser(
'build-wix',
help='Build WiX installer(s)'
)
sp = subparsers.add_parser('build-wix', help='Build WiX installer(s)')
sp.add_argument(
'--arch',
help='Architecture to build for',
......@@ -323,13 +341,10 @@ def get_parser():
default=['x64'],
)
sp.add_argument(
'--revision',
help='Mercurial revision to build',
default='.',
'--revision', help='Mercurial revision to build', default='.',
)
sp.add_argument(
'--version',
help='Mercurial version string to use in installer',
'--version', help='Mercurial version string to use in installer',
)
sp.add_argument(
'--base-image-name',
......@@ -345,15 +360,11 @@ def get_parser():
sp.set_defaults(func=terminate_ec2_instances)
sp = subparsers.add_parser(
'purge-ec2-resources',
help='Purge all EC2 resources managed by us',
'purge-ec2-resources', help='Purge all EC2 resources managed by us',
)
sp.set_defaults(func=purge_ec2_resources)
sp = subparsers.add_parser(
'run-tests-linux',
help='Run tests on Linux',
)
sp = subparsers.add_parser('run-tests-linux', help='Run tests on Linux',)
sp.add_argument(
'--distro',
help='Linux distribution to run tests on',
......@@ -374,8 +385,18 @@ def get_parser():
sp.add_argument(
'--python-version',
help='Python version to use',
choices={'system2', 'system3', '2.7', '3.5', '3.6', '3.7', '3.8',
'pypy', 'pypy3.5', 'pypy3.6'},
choices={
'system2',
'system3',
'2.7',
'3.5',
'3.6',
'3.7',
'3.8',
'pypy',
'pypy3.5',
'pypy3.6',
},
default='system2',
)
sp.add_argument(
......@@ -386,13 +407,10 @@ def get_parser():
sp.set_defaults(func=run_tests_linux)
sp = subparsers.add_parser(
'run-tests-windows',
help='Run tests on Windows',
'run-tests-windows', help='Run tests on Windows',
)
sp.add_argument(
'--instance-type',
help='EC2 instance type to use',
default='t3.medium',
'--instance-type', help='EC2 instance type to use', default='t3.medium',
)
sp.add_argument(
'--python-version',
......@@ -407,8 +425,7 @@ def get_parser():
default='x64',
)
sp.add_argument(
'--test-flags',
help='Extra command line flags to pass to run-tests.py',
'--test-flags', help='Extra command line flags to pass to run-tests.py',
)
sp.add_argument(
'--base-image-name',
......@@ -419,7 +436,7 @@ def get_parser():
sp = subparsers.add_parser(
'publish-windows-artifacts',
help='Publish built Windows artifacts (wheels, installers, etc)'
help='Publish built Windows artifacts (wheels, installers, etc)',
)
sp.add_argument(
'--no-pypi',
......@@ -436,22 +453,17 @@ def get_parser():
help='Skip uploading to www.mercurial-scm.org',
)
sp.add_argument(
'--ssh-username',
help='SSH username for mercurial-scm.org',
'--ssh-username', help='SSH username for mercurial-scm.org',
)
sp.add_argument(
'version',
help='Mercurial version string to locate local packages',
'version', help='Mercurial version string to locate local packages',
)
sp.set_defaults(func=publish_windows_artifacts)
sp = subparsers.add_parser(
'try',
help='Run CI automation against a custom changeset'
'try', help='Run CI automation against a custom changeset'
)
sp.add_argument('-r', '--rev',
default='.',
help='Revision to run CI on')
sp.add_argument('-r', '--rev', default='.', help='Revision to run CI on')
sp.set_defaults(func=run_try)
return parser
......
......@@ -13,9 +13,7 @@ import shlex
import subprocess
import tempfile
from .ssh import (
exec_command,
)
from .ssh import exec_command
# Linux distributions that are supported.
......@@ -62,7 +60,9 @@ for v in ${PYENV3_VERSIONS}; do
done
pyenv global ${PYENV2_VERSIONS} ${PYENV3_VERSIONS} system
'''.lstrip().replace('\r\n', '\n')
'''.lstrip().replace(
'\r\n', '\n'
)
INSTALL_RUST = r'''
......@@ -87,10 +87,13 @@ wget -O ${HG_TARBALL} --progress dot:mega https://www.mercurial-scm.org/release/
echo "${HG_SHA256} ${HG_TARBALL}" | sha256sum --check -
/hgdev/venv-bootstrap/bin/pip install ${HG_TARBALL}
'''.lstrip().replace('\r\n', '\n')
'''.lstrip().replace(
'\r\n', '\n'
)
BOOTSTRAP_DEBIAN = r'''
BOOTSTRAP_DEBIAN = (
r'''
#!/bin/bash
set -ex
......@@ -323,11 +326,14 @@ publish = false
EOF
sudo chown -R hg:hg /hgdev
'''.lstrip().format(
install_rust=INSTALL_RUST,
install_pythons=INSTALL_PYTHONS,
bootstrap_virtualenv=BOOTSTRAP_VIRTUALENV
).replace('\r\n', '\n')
'''.lstrip()
.format(
install_rust=INSTALL_RUST,
install_pythons=INSTALL_PYTHONS,
bootstrap_virtualenv=BOOTSTRAP_VIRTUALENV,
)
.replace('\r\n', '\n')
)
# Prepares /hgdev for operations.
......@@ -409,7 +415,9 @@ mkdir /hgwork/tmp
chown hg:hg /hgwork/tmp
rsync -a /hgdev/src /hgwork/
'''.lstrip().replace('\r\n', '\n')
'''.lstrip().replace(
'\r\n', '\n'
)
HG_UPDATE_CLEAN = '''
......@@ -421,7 +429,9 @@ cd /hgwork/src
${HG} --config extensions.purge= purge --all
${HG} update -C $1
${HG} log -r .
'''.lstrip().replace('\r\n', '\n')
'''.lstrip().replace(
'\r\n', '\n'
)
def prepare_exec_environment(ssh_client, filesystem='default'):
......@@ -456,11 +466,12 @@ def prepare_exec_environment(ssh_client, filesystem='default'):
res = chan.recv_exit_status()
if res:
raise Exception('non-0 exit code updating working directory; %d'
% res)
raise Exception('non-0 exit code updating working directory; %d' % res)
def synchronize_hg(source_path: pathlib.Path, ec2_instance, revision: str=None):
def synchronize_hg(
source_path: pathlib.Path, ec2_instance, revision: str = None
):
"""Synchronize a local Mercurial source path to remote EC2 instance."""
with tempfile.TemporaryDirectory() as temp_dir:
......@@ -482,8 +493,10 @@ def synchronize_hg(source_path: pathlib.Path, ec2_instance, revision: str=None):
fh.write(' IdentityFile %s\n' % ec2_instance.ssh_private_key_path)
if not (source_path / '.hg').is_dir():
raise Exception('%s is not a Mercurial repository; synchronization '
'not yet supported' % source_path)
raise Exception(
'%s is not a Mercurial repository; synchronization '
'not yet supported' % source_path
)
env = dict(os.environ)
env['HGPLAIN'] = '1'
......@@ -493,17 +506,29 @@ def synchronize_hg(source_path: pathlib.Path, ec2_instance, revision: str=None):
res = subprocess.run(
['python2.7', str(hg_bin), 'log', '-r', revision, '-T', '{node}'],
cwd=str(source_path), env=env, check=True, capture_output=True)
cwd=str(source_path),
env=env,
check=True,
capture_output=True,
)
full_revision = res.stdout.decode('ascii')
args = [
'python2.7', str(hg_bin),
'--config', 'ui.ssh=ssh -F %s' % ssh_config,
'--config', 'ui.remotecmd=/hgdev/venv-bootstrap/bin/hg',
'python2.7',
str(hg_bin),
'--config',
'ui.ssh=ssh -F %s' % ssh_config,
'--config',
'ui.remotecmd=/hgdev/venv-bootstrap/bin/hg',
# Also ensure .hgtags changes are present so auto version
# calculation works.
'push', '-f', '-r', full_revision, '-r', 'file(.hgtags)',
'push',
'-f',
'-r',
full_revision,
'-r',
'file(.hgtags)',
'ssh://%s//hgwork/src' % public_ip,
]
......@@ -522,7 +547,8 @@ def synchronize_hg(source_path: pathlib.Path, ec2_instance, revision: str=None):
fh.chmod(0o0700)