Commit 2372284d945 by Augie Fackler

formatting: blacken the codebase

This is using my patch to black
(https://github.com/psf/black/pull/826) so we don't un-wrap collection
literals.

Done with:

  hg files 'set:**.py - mercurial/thirdparty/** - "contrib/python-zstandard/**"' | xargs black -S

# skip-blame mass-reformatting only

# no-check-commit reformats foo_bar functions

Differential Revision: https://phab.mercurial-scm.org/D6971
parent 57875cf423c
......@@ -36,8 +36,13 @@ def bootstrap():
pip = venv_bin / 'pip'
python = venv_bin / 'python'
args = [str(pip), 'install', '-r', str(REQUIREMENTS_TXT),
'--disable-pip-version-check']
args = [
str(pip),
'install',
'-r',
str(REQUIREMENTS_TXT),
'--disable-pip-version-check',
]
if not venv_created:
args.append('-q')
......@@ -45,8 +50,7 @@ def bootstrap():
subprocess.run(args, check=True)
os.environ['HGAUTOMATION_BOOTSTRAPPED'] = '1'
os.environ['PATH'] = '%s%s%s' % (
venv_bin, os.pathsep, os.environ['PATH'])
os.environ['PATH'] = '%s%s%s' % (venv_bin, os.pathsep, os.environ['PATH'])
subprocess.run([str(python), __file__] + sys.argv[1:], check=True)
......
......@@ -10,9 +10,7 @@
import pathlib
import secrets
from .aws import (
AWSConnection,
)
from .aws import AWSConnection
class HGAutomation:
......@@ -53,7 +51,7 @@ class HGAutomation:
return password
def aws_connection(self, region: str, ensure_ec2_state: bool=True):
def aws_connection(self, region: str, ensure_ec2_state: bool = True):
"""Obtain an AWSConnection instance bound to a specific region."""
return AWSConnection(self, region, ensure_ec2_state=ensure_ec2_state)
......@@ -13,9 +13,7 @@ import shlex
import subprocess
import tempfile
from .ssh import (
exec_command,
)
from .ssh import exec_command
# Linux distributions that are supported.
......@@ -62,7 +60,9 @@ for v in ${PYENV3_VERSIONS}; do
done
pyenv global ${PYENV2_VERSIONS} ${PYENV3_VERSIONS} system
'''.lstrip().replace('\r\n', '\n')
'''.lstrip().replace(
'\r\n', '\n'
)
INSTALL_RUST = r'''
......@@ -87,10 +87,13 @@ wget -O ${HG_TARBALL} --progress dot:mega https://www.mercurial-scm.org/release/
echo "${HG_SHA256} ${HG_TARBALL}" | sha256sum --check -
/hgdev/venv-bootstrap/bin/pip install ${HG_TARBALL}
'''.lstrip().replace('\r\n', '\n')
'''.lstrip().replace(
'\r\n', '\n'
)
BOOTSTRAP_DEBIAN = r'''
BOOTSTRAP_DEBIAN = (
r'''
#!/bin/bash
set -ex
......@@ -323,11 +326,14 @@ publish = false
EOF
sudo chown -R hg:hg /hgdev
'''.lstrip().format(
install_rust=INSTALL_RUST,
install_pythons=INSTALL_PYTHONS,
bootstrap_virtualenv=BOOTSTRAP_VIRTUALENV
).replace('\r\n', '\n')
'''.lstrip()
.format(
install_rust=INSTALL_RUST,
install_pythons=INSTALL_PYTHONS,
bootstrap_virtualenv=BOOTSTRAP_VIRTUALENV,
)
.replace('\r\n', '\n')
)
# Prepares /hgdev for operations.
......@@ -409,7 +415,9 @@ mkdir /hgwork/tmp
chown hg:hg /hgwork/tmp
rsync -a /hgdev/src /hgwork/
'''.lstrip().replace('\r\n', '\n')
'''.lstrip().replace(
'\r\n', '\n'
)
HG_UPDATE_CLEAN = '''
......@@ -421,7 +429,9 @@ cd /hgwork/src
${HG} --config extensions.purge= purge --all
${HG} update -C $1
${HG} log -r .
'''.lstrip().replace('\r\n', '\n')
'''.lstrip().replace(
'\r\n', '\n'
)
def prepare_exec_environment(ssh_client, filesystem='default'):
......@@ -456,11 +466,12 @@ def prepare_exec_environment(ssh_client, filesystem='default'):
res = chan.recv_exit_status()
if res:
raise Exception('non-0 exit code updating working directory; %d'
% res)
raise Exception('non-0 exit code updating working directory; %d' % res)
def synchronize_hg(source_path: pathlib.Path, ec2_instance, revision: str=None):
def synchronize_hg(
source_path: pathlib.Path, ec2_instance, revision: str = None
):
"""Synchronize a local Mercurial source path to remote EC2 instance."""
with tempfile.TemporaryDirectory() as temp_dir:
......@@ -482,8 +493,10 @@ def synchronize_hg(source_path: pathlib.Path, ec2_instance, revision: str=None):
fh.write(' IdentityFile %s\n' % ec2_instance.ssh_private_key_path)
if not (source_path / '.hg').is_dir():
raise Exception('%s is not a Mercurial repository; synchronization '
'not yet supported' % source_path)
raise Exception(
'%s is not a Mercurial repository; synchronization '
'not yet supported' % source_path
)
env = dict(os.environ)
env['HGPLAIN'] = '1'
......@@ -493,17 +506,29 @@ def synchronize_hg(source_path: pathlib.Path, ec2_instance, revision: str=None):
res = subprocess.run(
['python2.7', str(hg_bin), 'log', '-r', revision, '-T', '{node}'],
cwd=str(source_path), env=env, check=True, capture_output=True)
cwd=str(source_path),
env=env,
check=True,
capture_output=True,
)
full_revision = res.stdout.decode('ascii')
args = [
'python2.7', str(hg_bin),
'--config', 'ui.ssh=ssh -F %s' % ssh_config,
'--config', 'ui.remotecmd=/hgdev/venv-bootstrap/bin/hg',
'python2.7',
str(hg_bin),
'--config',
'ui.ssh=ssh -F %s' % ssh_config,
'--config',
'ui.remotecmd=/hgdev/venv-bootstrap/bin/hg',
# Also ensure .hgtags changes are present so auto version
# calculation works.
'push', '-f', '-r', full_revision, '-r', 'file(.hgtags)',
'push',
'-f',
'-r',
full_revision,
'-r',
'file(.hgtags)',
'ssh://%s//hgwork/src' % public_ip,
]
......@@ -522,7 +547,8 @@ def synchronize_hg(source_path: pathlib.Path, ec2_instance, revision: str=None):
fh.chmod(0o0700)
chan, stdin, stdout = exec_command(
ec2_instance.ssh_client, '/hgdev/hgup %s' % full_revision)
ec2_instance.ssh_client, '/hgdev/hgup %s' % full_revision
)
stdin.close()
for line in stdout:
......@@ -531,8 +557,9 @@ def synchronize_hg(source_path: pathlib.Path, ec2_instance, revision: str=None):
res = chan.recv_exit_status()
if res:
raise Exception('non-0 exit code updating working directory; %d'
% res)
raise Exception(
'non-0 exit code updating working directory; %d' % res
)
def run_tests(ssh_client, python_version, test_flags=None):
......@@ -554,8 +581,8 @@ def run_tests(ssh_client, python_version, test_flags=None):
command = (
'/bin/sh -c "export TMPDIR=/hgwork/tmp; '
'cd /hgwork/src/tests && %s run-tests.py %s"' % (
python, test_flags))
'cd /hgwork/src/tests && %s run-tests.py %s"' % (python, test_flags)
)
chan, stdin, stdout = exec_command(ssh_client, command)
......
......@@ -7,12 +7,8 @@
# no-check-code because Python 3 native.
from twine.commands.upload import (
upload as twine_upload,
)
from twine.settings import (
Settings,
)
from twine.commands.upload import upload as twine_upload
from twine.settings import Settings
def upload(paths):
......
......@@ -11,14 +11,13 @@ import socket
import time
import warnings
from cryptography.utils import (
CryptographyDeprecationWarning,
)
from cryptography.utils import CryptographyDeprecationWarning
import paramiko
def wait_for_ssh(hostname, port, timeout=60, username=None, key_filename=None):
"""Wait for an SSH server to start on the specified host and port."""
class IgnoreHostKeyPolicy(paramiko.MissingHostKeyPolicy):
def missing_host_key(self, client, hostname, key):
return
......@@ -28,17 +27,23 @@ def wait_for_ssh(hostname, port, timeout=60, username=None, key_filename=None):
# paramiko triggers a CryptographyDeprecationWarning in the cryptography
# package. Let's suppress
with warnings.catch_warnings():
warnings.filterwarnings('ignore',
category=CryptographyDeprecationWarning)
warnings.filterwarnings(
'ignore', category=CryptographyDeprecationWarning
)
while True:
client = paramiko.SSHClient()
client.set_missing_host_key_policy(IgnoreHostKeyPolicy())
try:
client.connect(hostname, port=port, username=username,
key_filename=key_filename,
timeout=5.0, allow_agent=False,
look_for_keys=False)
client.connect(
hostname,
port=port,
username=username,
key_filename=key_filename,
timeout=5.0,
allow_agent=False,
look_for_keys=False,
)
return client
except socket.error:
......
......@@ -15,12 +15,8 @@ import re
import subprocess
import tempfile
from .pypi import (
upload as pypi_upload,
)
from .winrm import (
run_powershell,
)
from .pypi import upload as pypi_upload
from .winrm import run_powershell
# PowerShell commands to activate a Visual Studio 2008 environment.
......@@ -117,14 +113,21 @@ MERCURIAL_SCM_BASE_URL = 'https://mercurial-scm.org/release/windows'
X86_USER_AGENT_PATTERN = '.*Windows.*'
X64_USER_AGENT_PATTERN = '.*Windows.*(WOW|x)64.*'
X86_EXE_DESCRIPTION = ('Mercurial {version} Inno Setup installer - x86 Windows '
'- does not require admin rights')
X64_EXE_DESCRIPTION = ('Mercurial {version} Inno Setup installer - x64 Windows '
'- does not require admin rights')
X86_MSI_DESCRIPTION = ('Mercurial {version} MSI installer - x86 Windows '
'- requires admin rights')
X64_MSI_DESCRIPTION = ('Mercurial {version} MSI installer - x64 Windows '
'- requires admin rights')
X86_EXE_DESCRIPTION = (
'Mercurial {version} Inno Setup installer - x86 Windows '
'- does not require admin rights'
)
X64_EXE_DESCRIPTION = (
'Mercurial {version} Inno Setup installer - x64 Windows '
'- does not require admin rights'
)
X86_MSI_DESCRIPTION = (
'Mercurial {version} MSI installer - x86 Windows ' '- requires admin rights'
)
X64_MSI_DESCRIPTION = (
'Mercurial {version} MSI installer - x64 Windows ' '- requires admin rights'
)
def get_vc_prefix(arch):
if arch == 'x86':
......@@ -158,10 +161,21 @@ def synchronize_hg(hg_repo: pathlib.Path, revision: str, ec2_instance):
ssh_dir.chmod(0o0700)
# Generate SSH key to use for communication.
subprocess.run([
'ssh-keygen', '-t', 'rsa', '-b', '4096', '-N', '',
'-f', str(ssh_dir / 'id_rsa')],
check=True, capture_output=True)
subprocess.run(
[
'ssh-keygen',
'-t',
'rsa',
'-b',
'4096',
'-N',
'',
'-f',
str(ssh_dir / 'id_rsa'),
],
check=True,
capture_output=True,
)
# Add it to ~/.ssh/authorized_keys on remote.
# This assumes the file doesn't already exist.
......@@ -182,8 +196,10 @@ def synchronize_hg(hg_repo: pathlib.Path, revision: str, ec2_instance):
fh.write(' IdentityFile %s\n' % (ssh_dir / 'id_rsa'))
if not (hg_repo / '.hg').is_dir():
raise Exception('%s is not a Mercurial repository; '
'synchronization not yet supported' % hg_repo)
raise Exception(
'%s is not a Mercurial repository; '
'synchronization not yet supported' % hg_repo
)
env = dict(os.environ)
env['HGPLAIN'] = '1'
......@@ -193,17 +209,29 @@ def synchronize_hg(hg_repo: pathlib.Path, revision: str, ec2_instance):
res = subprocess.run(
['python2.7', str(hg_bin), 'log', '-r', revision, '-T', '{node}'],
cwd=str(hg_repo), env=env, check=True, capture_output=True)
cwd=str(hg_repo),
env=env,
check=True,
capture_output=True,
)
full_revision = res.stdout.decode('ascii')
args = [
'python2.7', hg_bin,
'--config', 'ui.ssh=ssh -F %s' % ssh_config,
'--config', 'ui.remotecmd=c:/hgdev/venv-bootstrap/Scripts/hg.exe',
'python2.7',
hg_bin,
'--config',
'ui.ssh=ssh -F %s' % ssh_config,
'--config',
'ui.remotecmd=c:/hgdev/venv-bootstrap/Scripts/hg.exe',
# Also ensure .hgtags changes are present so auto version
# calculation works.
'push', '-f', '-r', full_revision, '-r', 'file(.hgtags)',
'push',
'-f',
'-r',
full_revision,
'-r',
'file(.hgtags)',
'ssh://%s/c:/hgdev/src' % public_ip,
]
......@@ -213,8 +241,9 @@ def synchronize_hg(hg_repo: pathlib.Path, revision: str, ec2_instance):
if res.returncode not in (0, 1):
res.check_returncode()
run_powershell(winrm_client,
HG_UPDATE_CLEAN.format(revision=full_revision))
run_powershell(
winrm_client, HG_UPDATE_CLEAN.format(revision=full_revision)
)
# TODO detect dirty local working directory and synchronize accordingly.
......@@ -250,8 +279,9 @@ def copy_latest_dist(winrm_client, pattern, dest_path):
winrm_client.fetch(source, str(dest))
def build_inno_installer(winrm_client, arch: str, dest_path: pathlib.Path,
version=None):
def build_inno_installer(
winrm_client, arch: str, dest_path: pathlib.Path, version=None
):
"""Build the Inno Setup installer on a remote machine.
Using a WinRM client, remote commands are executed to build
......@@ -263,8 +293,9 @@ def build_inno_installer(winrm_client, arch: str, dest_path: pathlib.Path,
if version:
extra_args.extend(['--version', version])
ps = get_vc_prefix(arch) + BUILD_INNO.format(arch=arch,
extra_args=' '.join(extra_args))
ps = get_vc_prefix(arch) + BUILD_INNO.format(
arch=arch, extra_args=' '.join(extra_args)
)
run_powershell(winrm_client, ps)
copy_latest_dist(winrm_client, '*.exe', dest_path)
......@@ -281,8 +312,9 @@ def build_wheel(winrm_client, arch: str, dest_path: pathlib.Path):
copy_latest_dist(winrm_client, '*.whl', dest_path)
def build_wix_installer(winrm_client, arch: str, dest_path: pathlib.Path,
version=None):
def build_wix_installer(
winrm_client, arch: str, dest_path: pathlib.Path, version=None
):
"""Build the WiX installer on a remote machine.
Using a WinRM client, remote commands are executed to build a WiX installer.
......@@ -292,8 +324,9 @@ def build_wix_installer(winrm_client, arch: str, dest_path: pathlib.Path,
if version:
extra_args.extend(['--version', version])
ps = get_vc_prefix(arch) + BUILD_WIX.format(arch=arch,
extra_args=' '.join(extra_args))
ps = get_vc_prefix(arch) + BUILD_WIX.format(
arch=arch, extra_args=' '.join(extra_args)
)
run_powershell(winrm_client, ps)
copy_latest_dist(winrm_client, '*.msi', dest_path)
......@@ -307,18 +340,16 @@ def run_tests(winrm_client, python_version, arch, test_flags=''):
``run-tests.py``.
"""
if not re.match(r'\d\.\d', python_version):
raise ValueError(r'python_version must be \d.\d; got %s' %
python_version)
raise ValueError(
r'python_version must be \d.\d; got %s' % python_version
)
if arch not in ('x86', 'x64'):
raise ValueError('arch must be x86 or x64; got %s' % arch)
python_path = 'python%s-%s' % (python_version.replace('.', ''), arch)
ps = RUN_TESTS.format(
python_path=python_path,
test_flags=test_flags or '',
)
ps = RUN_TESTS.format(python_path=python_path, test_flags=test_flags or '',)
run_powershell(winrm_client, ps)
......@@ -374,8 +405,8 @@ def generate_latest_dat(version: str):
version,
X64_USER_AGENT_PATTERN,
'%s/%s' % (MERCURIAL_SCM_BASE_URL, x64_msi_filename),
X64_MSI_DESCRIPTION.format(version=version)
)
X64_MSI_DESCRIPTION.format(version=version),
),
)
lines = ['\t'.join(e) for e in entries]
......@@ -396,8 +427,9 @@ def publish_artifacts_pypi(dist_path: pathlib.Path, version: str):
pypi_upload(wheel_paths)
def publish_artifacts_mercurial_scm_org(dist_path: pathlib.Path, version: str,
ssh_username=None):
def publish_artifacts_mercurial_scm_org(
dist_path: pathlib.Path, version: str, ssh_username=None
):
"""Publish Windows release artifacts to mercurial-scm.org."""
all_paths = resolve_all_artifacts(dist_path, version)
......@@ -436,7 +468,8 @@ def publish_artifacts_mercurial_scm_org(dist_path: pathlib.Path, version: str,
now = datetime.datetime.utcnow()
backup_path = dist_path / (
'latest-windows-%s.dat' % now.strftime('%Y%m%dT%H%M%S'))
'latest-windows-%s.dat' % now.strftime('%Y%m%dT%H%M%S')
)
print('backing up %s to %s' % (latest_dat_path, backup_path))
with sftp.open(latest_dat_path, 'rb') as fh:
......@@ -453,9 +486,13 @@ def publish_artifacts_mercurial_scm_org(dist_path: pathlib.Path, version: str,
fh.write(latest_dat_content.encode('ascii'))