Commit 9788b635 authored by Augie Fackler's avatar Augie Fackler

merge default into stable for 5.2 release

--HG--
branch : stable
......@@ -113,7 +113,7 @@ check: tests
tests:
# Run Rust tests if cargo is installed
if command -v $(CARGO) >/dev/null 2>&1; then \
cd $(HGROOT)/rust/hg-cpython && $(CARGO) test --quiet --all; \
$(MAKE) rust-tests; \
fi
cd tests && $(PYTHON) run-tests.py $(TESTFLAGS)
......@@ -127,6 +127,13 @@ testpy-%:
$(MAKE) -f $(HGROOT)/contrib/Makefile.python PYTHONVER=$* PREFIX=$(HGPYTHONS)/$* python )
cd tests && $(HGPYTHONS)/$*/bin/python run-tests.py $(TESTFLAGS)
rust-tests: py_feature = $(shell $(PYTHON) -c \
'import sys; print(["python27-bin", "python3-bin"][sys.version_info[0] >= 3])')
rust-tests:
cd $(HGROOT)/rust/hg-cpython \
&& $(CARGO) test --quiet --all \
--no-default-features --features "$(py_feature)"
check-code:
hg manifest | xargs python contrib/check-code.py
......@@ -248,6 +255,7 @@ osx:
.PHONY: help all local build doc cleanbutpackages clean install install-bin \
install-doc install-home install-home-bin install-home-doc \
dist dist-notests check tests check-code format-c update-pot \
dist dist-notests check tests rust-tests check-code format-c \
update-pot \
$(packaging_targets) \
osx
[tool.black]
line-length = 80
exclude = '''
build/
| wheelhouse/
| dist/
| packages/
| \.hg/
| \.mypy_cache/
| \.venv/
| mercurial/thirdparty/
| hgext/fsmonitor/pywatchman/
| contrib/python-zstandard/
| contrib/grey.py
'''
skip-string-normalization = true
quiet = true
......@@ -33,6 +33,46 @@ side-effects so they don't impact the local system. e.g. when we SSH
into a remote machine, we create a temporary directory for the SSH
config so the user's known hosts file isn't updated.
Try Server
==========
There exists a *Try Server* which allows automation to run against
an arbitrary Mercurial changeset and displays results via the web.
.. note::
The *Try Server* is still experimental infrastructure.
To use the *Try Server*::
$ ./automation.py try
With a custom AWS profile::
$ AWS_PROFILE=hg contrib/automation/automation.py try
By default, the ``.`` revision is submitted. **Any uncommitted changes
are not submitted.**
To switch which revision is used::
$ ./automation.py try -r abcdef
Access to the *Try Server* requires access to a special AWS account.
This account is currently run by Gregory Szorc. Here is the procedure
for accessing the *Try Server*:
1. Email Gregory Szorc at gregory.szorc@gmail.com and request a
username. This username will be stored in the public domain.
2. Wait for an email reply containing your temporary AWS credentials.
3. Log in at https://gregoryszorc-hg.signin.aws.amazon.com/console
and set a new, secure password.
4. Go to https://console.aws.amazon.com/iam/home?region=us-west-2#/security_credentials
5. Under ``Access keys for CLI, SDK, & API access``, click the
``Create access key`` button.
6. See the ``AWS Integration`` section for instructions on
configuring your local client to use the generated credentials.
AWS Integration
===============
......@@ -47,12 +87,25 @@ https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html
for how ``boto3`` works. Once you have configured your environment such
that ``boto3`` can find credentials, interaction with AWS should *just work*.
.. hint::
To configure ``boto3``, you can use the ``aws configure`` command to
write out configuration files. (The ``aws`` command is typically provided
by an ``awscli`` package available in your package manager, including
``pip``.) Alternatively, you can write out files in ``~/.aws/`` directly.
e.g.::
Typically you have a ``~/.aws/credentials`` file containing AWS
credentials. If you manage multiple credentials, you can override which
*profile* to use at run-time by setting the ``AWS_PROFILE`` environment
variable.
# ~/.aws/config
[default]
region = us-west-2
# ~/.aws/credentials
[default]
aws_access_key_id = XXXX
aws_secret_access_key = YYYY
If you have multiple AWS accounts, you can name the profile something
different from ``default``. e.g. ``hg``. You can influence which profile
is used by ``boto3`` by setting the ``AWS_PROFILE`` environment variable.
e.g. ``AWS_PROFILE=hg``.
Resource Management
-------------------
......@@ -181,3 +234,25 @@ Various dependencies to run the Mercurial test harness are also required.
Documenting them is beyond the scope of this document. Various tests
also require other optional dependencies and missing dependencies will
be printed by the test runner when a test is skipped.
Releasing Windows Artifacts
===========================
The `automation.py` script can be used to automate the release of Windows
artifacts::
$ ./automation.py build-all-windows-packages --revision 5.1.1
$ ./automation.py publish-windows-artifacts 5.1.1
The first command will launch an EC2 instance to build all Windows packages
and copy them into the `dist` directory relative to the repository root. The
second command will then attempt to upload these files to PyPI (via `twine`)
and to `mercurial-scm.org` (via SSH).
Uploading to PyPI requires a PyPI account with write access to the `Mercurial`
package. You can skip PyPI uploading by passing `--no-pypi`.
Uploading to `mercurial-scm.org` requires an SSH account on that server
with `windows` group membership and for the SSH key for that account to be the
default SSH key (e.g. `~/.ssh/id_rsa`) or in a running SSH agent. You can
skip `mercurial-scm.org` uploading by passing `--no-mercurial-scm-org`.
......@@ -36,8 +36,13 @@ def bootstrap():
pip = venv_bin / 'pip'
python = venv_bin / 'python'
args = [str(pip), 'install', '-r', str(REQUIREMENTS_TXT),
'--disable-pip-version-check']
args = [
str(pip),
'install',
'-r',
str(REQUIREMENTS_TXT),
'--disable-pip-version-check',
]
if not venv_created:
args.append('-q')
......@@ -45,8 +50,7 @@ def bootstrap():
subprocess.run(args, check=True)
os.environ['HGAUTOMATION_BOOTSTRAPPED'] = '1'
os.environ['PATH'] = '%s%s%s' % (
venv_bin, os.pathsep, os.environ['PATH'])
os.environ['PATH'] = '%s%s%s' % (venv_bin, os.pathsep, os.environ['PATH'])
subprocess.run([str(python), __file__] + sys.argv[1:], check=True)
......
......@@ -10,9 +10,7 @@
import pathlib
import secrets
from .aws import (
AWSConnection,
)
from .aws import AWSConnection
class HGAutomation:
......@@ -53,7 +51,7 @@ class HGAutomation:
return password
def aws_connection(self, region: str, ensure_ec2_state: bool=True):
def aws_connection(self, region: str, ensure_ec2_state: bool = True):
"""Obtain an AWSConnection instance bound to a specific region."""
return AWSConnection(self, region, ensure_ec2_state=ensure_ec2_state)
This diff is collapsed.
......@@ -17,16 +17,20 @@ from . import (
aws,
HGAutomation,
linux,
try_server,
windows,
)
SOURCE_ROOT = pathlib.Path(os.path.abspath(__file__)).parent.parent.parent.parent
SOURCE_ROOT = pathlib.Path(
os.path.abspath(__file__)
).parent.parent.parent.parent
DIST_PATH = SOURCE_ROOT / 'dist'
def bootstrap_linux_dev(hga: HGAutomation, aws_region, distros=None,
parallel=False):
def bootstrap_linux_dev(
hga: HGAutomation, aws_region, distros=None, parallel=False
):
c = hga.aws_connection(aws_region)
if distros:
......@@ -58,8 +62,9 @@ def bootstrap_windows_dev(hga: HGAutomation, aws_region, base_image_name):
print('Windows development AMI available as %s' % image.id)
def build_inno(hga: HGAutomation, aws_region, arch, revision, version,
base_image_name):
def build_inno(
hga: HGAutomation, aws_region, arch, revision, version, base_image_name
):
c = hga.aws_connection(aws_region)
image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
DIST_PATH.mkdir(exist_ok=True)
......@@ -70,13 +75,14 @@ def build_inno(hga: HGAutomation, aws_region, arch, revision, version,
windows.synchronize_hg(SOURCE_ROOT, revision, instance)
for a in arch:
windows.build_inno_installer(instance.winrm_client, a,
DIST_PATH,
version=version)
windows.build_inno_installer(
instance.winrm_client, a, DIST_PATH, version=version
)
def build_wix(hga: HGAutomation, aws_region, arch, revision, version,
base_image_name):
def build_wix(
hga: HGAutomation, aws_region, arch, revision, version, base_image_name
):
c = hga.aws_connection(aws_region)
image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
DIST_PATH.mkdir(exist_ok=True)
......@@ -87,12 +93,14 @@ def build_wix(hga: HGAutomation, aws_region, arch, revision, version,
windows.synchronize_hg(SOURCE_ROOT, revision, instance)
for a in arch:
windows.build_wix_installer(instance.winrm_client, a,
DIST_PATH, version=version)
windows.build_wix_installer(
instance.winrm_client, a, DIST_PATH, version=version
)
def build_windows_wheel(hga: HGAutomation, aws_region, arch, revision,
base_image_name):
def build_windows_wheel(
hga: HGAutomation, aws_region, arch, revision, base_image_name
):
c = hga.aws_connection(aws_region)
image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
DIST_PATH.mkdir(exist_ok=True)
......@@ -106,8 +114,9 @@ def build_windows_wheel(hga: HGAutomation, aws_region, arch, revision,
windows.build_wheel(instance.winrm_client, a, DIST_PATH)
def build_all_windows_packages(hga: HGAutomation, aws_region, revision,
version, base_image_name):
def build_all_windows_packages(
hga: HGAutomation, aws_region, revision, version, base_image_name
):
c = hga.aws_connection(aws_region)
image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
DIST_PATH.mkdir(exist_ok=True)
......@@ -123,11 +132,13 @@ def build_all_windows_packages(hga: HGAutomation, aws_region, revision,
windows.purge_hg(winrm_client)
windows.build_wheel(winrm_client, arch, DIST_PATH)
windows.purge_hg(winrm_client)
windows.build_inno_installer(winrm_client, arch, DIST_PATH,
version=version)
windows.build_inno_installer(
winrm_client, arch, DIST_PATH, version=version
)
windows.purge_hg(winrm_client)
windows.build_wix_installer(winrm_client, arch, DIST_PATH,
version=version)
windows.build_wix_installer(
winrm_client, arch, DIST_PATH, version=version
)
def terminate_ec2_instances(hga: HGAutomation, aws_region):
......@@ -140,8 +151,15 @@ def purge_ec2_resources(hga: HGAutomation, aws_region):
aws.remove_resources(c)
def run_tests_linux(hga: HGAutomation, aws_region, instance_type,
python_version, test_flags, distro, filesystem):
def run_tests_linux(
hga: HGAutomation,
aws_region,
instance_type,
python_version,
test_flags,
distro,
filesystem,