Commit 08a7faae authored by Pulkit Goyal's avatar Pulkit Goyal

merge default into stable for 5.5rc0

--HG--
branch : stable
Pipeline #8173 failed with stage
in 29 minutes and 7 seconds
......@@ -52,6 +52,7 @@ cscope.*
.idea/*
.asv/*
.pytype/*
.mypy_cache
i18n/hg.pot
locale/*/LC_MESSAGES/hg.mo
hgext/__index__.py
......
......@@ -232,7 +232,7 @@ static void execcmdserver(const struct cmdserveropts *opts)
abortmsgerrno("failed to putenv CHG_CLEAR_LC_CTYPE");
} else {
if (setenv("CHGORIG_LC_CTYPE", lc_ctype_env, 1) != 0) {
abortmsgerrno("failed to setenv CHGORIG_LC_CTYYPE");
abortmsgerrno("failed to setenv CHGORIG_LC_CTYPE");
}
}
......
......@@ -28,7 +28,7 @@ binopen.options = {}
def printb(data, end=b'\n'):
sys.stdout.flush()
pycompat.stdout.write(data + end)
procutil.stdout.write(data + end)
for f in sys.argv[1:]:
......
......@@ -11,6 +11,7 @@ CXX = clang++
LIB_FUZZING_ENGINE ?= standalone_fuzz_target_runner.o
PYTHON_CONFIG ?= $$OUT/sanpy/bin/python-config
PYTHON_CONFIG_FLAGS ?= --ldflags
CXXFLAGS += -Wno-deprecated-register
......@@ -67,7 +68,7 @@ dirs_fuzzer: dirs.cc pyutil.o $(PARSERS_OBJS) $$OUT/dirs_fuzzer_seed_corpus.zip
-Wno-register -Wno-macro-redefined \
-I../../mercurial dirs.cc \
pyutil.o $(PARSERS_OBJS) \
$(LIB_FUZZING_ENGINE) `$(PYTHON_CONFIG) --ldflags` \
$(LIB_FUZZING_ENGINE) `$(PYTHON_CONFIG) $(PYTHON_CONFIG_FLAGS)` \
-o $$OUT/dirs_fuzzer
fncache_fuzzer: fncache.cc
......@@ -75,7 +76,7 @@ fncache_fuzzer: fncache.cc
-Wno-register -Wno-macro-redefined \
-I../../mercurial fncache.cc \
pyutil.o $(PARSERS_OBJS) \
$(LIB_FUZZING_ENGINE) `$(PYTHON_CONFIG) --ldflags` \
$(LIB_FUZZING_ENGINE) `$(PYTHON_CONFIG) $(PYTHON_CONFIG_FLAGS)` \
-o $$OUT/fncache_fuzzer
jsonescapeu8fast_fuzzer: jsonescapeu8fast.cc pyutil.o $(PARSERS_OBJS)
......@@ -83,7 +84,7 @@ jsonescapeu8fast_fuzzer: jsonescapeu8fast.cc pyutil.o $(PARSERS_OBJS)
-Wno-register -Wno-macro-redefined \
-I../../mercurial jsonescapeu8fast.cc \
pyutil.o $(PARSERS_OBJS) \
$(LIB_FUZZING_ENGINE) `$(PYTHON_CONFIG) --ldflags` \
$(LIB_FUZZING_ENGINE) `$(PYTHON_CONFIG) $(PYTHON_CONFIG_FLAGS)` \
-o $$OUT/jsonescapeu8fast_fuzzer
manifest_fuzzer: manifest.cc pyutil.o $(PARSERS_OBJS) $$OUT/manifest_fuzzer_seed_corpus.zip
......@@ -91,7 +92,7 @@ manifest_fuzzer: manifest.cc pyutil.o $(PARSERS_OBJS) $$OUT/manifest_fuzzer_seed
-Wno-register -Wno-macro-redefined \
-I../../mercurial manifest.cc \
pyutil.o $(PARSERS_OBJS) \
$(LIB_FUZZING_ENGINE) `$(PYTHON_CONFIG) --ldflags` \
$(LIB_FUZZING_ENGINE) `$(PYTHON_CONFIG) $(PYTHON_CONFIG_FLAGS)` \
-o $$OUT/manifest_fuzzer
revlog_fuzzer: revlog.cc pyutil.o $(PARSERS_OBJS) $$OUT/revlog_fuzzer_seed_corpus.zip
......@@ -99,7 +100,7 @@ revlog_fuzzer: revlog.cc pyutil.o $(PARSERS_OBJS) $$OUT/revlog_fuzzer_seed_corpu
-Wno-register -Wno-macro-redefined \
-I../../mercurial revlog.cc \
pyutil.o $(PARSERS_OBJS) \
$(LIB_FUZZING_ENGINE) `$(PYTHON_CONFIG) --ldflags` \
$(LIB_FUZZING_ENGINE) `$(PYTHON_CONFIG) $(PYTHON_CONFIG_FLAGS)` \
-o $$OUT/revlog_fuzzer
dirstate_fuzzer: dirstate.cc pyutil.o $(PARSERS_OBJS) $$OUT/dirstate_fuzzer_seed_corpus.zip
......@@ -107,7 +108,7 @@ dirstate_fuzzer: dirstate.cc pyutil.o $(PARSERS_OBJS) $$OUT/dirstate_fuzzer_seed
-Wno-register -Wno-macro-redefined \
-I../../mercurial dirstate.cc \
pyutil.o $(PARSERS_OBJS) \
$(LIB_FUZZING_ENGINE) `$(PYTHON_CONFIG) --ldflags` \
$(LIB_FUZZING_ENGINE) `$(PYTHON_CONFIG) $(PYTHON_CONFIG_FLAGS)` \
-o $$OUT/dirstate_fuzzer
fm1readmarkers_fuzzer: fm1readmarkers.cc pyutil.o $(PARSERS_OBJS) $$OUT/fm1readmarkers_fuzzer_seed_corpus.zip
......@@ -115,7 +116,7 @@ fm1readmarkers_fuzzer: fm1readmarkers.cc pyutil.o $(PARSERS_OBJS) $$OUT/fm1readm
-Wno-register -Wno-macro-redefined \
-I../../mercurial fm1readmarkers.cc \
pyutil.o $(PARSERS_OBJS) \
$(LIB_FUZZING_ENGINE) `$(PYTHON_CONFIG) --ldflags` \
$(LIB_FUZZING_ENGINE) `$(PYTHON_CONFIG) $(PYTHON_CONFIG_FLAGS)` \
-o $$OUT/fm1readmarkers_fuzzer
clean:
......
......@@ -3,6 +3,7 @@
#include <stdlib.h>
#include <unistd.h>
#include "FuzzedDataProvider.h"
#include "pyutil.h"
#include <string>
......@@ -24,7 +25,7 @@ try:
lm[e]
e in lm
(e + 'nope') in lm
lm[b'xyzzy'] = (b'\0' * 20, 'x')
lm[b'xyzzy'] = (b'\0' * nlen, 'x')
# do an insert, text should change
assert lm.text() != mdata, "insert should change text and didn't: %r %r" % (lm.text(), mdata)
cloned = lm.filtercopy(lambda x: x != 'xyzzy')
......@@ -51,10 +52,14 @@ int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
if (Size > 100000) {
return 0;
}
FuzzedDataProvider provider(Data, Size);
Py_ssize_t nodelength = provider.ConsumeBool() ? 20 : 32;
PyObject *nlen = PyLong_FromSsize_t(nodelength);
PyObject *mtext =
PyBytes_FromStringAndSize((const char *)Data, (Py_ssize_t)Size);
PyObject *locals = PyDict_New();
PyDict_SetItemString(locals, "mdata", mtext);
PyDict_SetItemString(locals, "nlen", nlen);
PyObject *res = PyEval_EvalCode(code, contrib::pyglobals(), locals);
if (!res) {
PyErr_Print();
......
......@@ -10,7 +10,7 @@ args = ap.parse_args()
with zipfile.ZipFile(args.out[0], "w", zipfile.ZIP_STORED) as zf:
zf.writestr(
"manifest_zero",
'''PKG-INFO\09b3ed8f2b81095a13064402e930565f083346e9a
'''\0PKG-INFO\09b3ed8f2b81095a13064402e930565f083346e9a
README\080b6e76643dcb44d4bc729e932fc464b3e36dbe3
hg\0b6444347c629cc058d478023905cfb83b7f5bb9d
mercurial/__init__.py\0b80de5d138758541c5f05265ad144ab9fa86d1db
......@@ -25,9 +25,14 @@ setup.py\0ccf3f6daf0f13101ca73631f7a1769e328b472c9
tkmerge\03c922edb43a9c143682f7bc7b00f98b3c756ebe7
''',
)
zf.writestr("badmanifest_shorthashes", "narf\0aa\nnarf2\0aaa\n")
zf.writestr("badmanifest_shorthashes", "\0narf\0aa\nnarf2\0aaa\n")
zf.writestr(
"badmanifest_nonull",
"narf\0cccccccccccccccccccccccccccccccccccccccc\n"
"\0narf\0cccccccccccccccccccccccccccccccccccccccc\n"
"narf2aaaaaaaaaaaaaaaaaaaa\n",
)
zf.writestr(
"manifest_long_nodes",
"\1a\0ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff\n",
)
......@@ -21,7 +21,7 @@ static PyObject *globals;
void initpy(const char *cselfpath)
{
#ifdef HG_FUZZER_PY3
const std::string subdir = "/sanpy/lib/python3.7";
const std::string subdir = "/sanpy/lib/python3.8";
#else
const std::string subdir = "/sanpy/lib/python2.7";
#endif
......
......@@ -5,6 +5,8 @@ image: octobus/ci-mercurial-core
before_script:
- hg clone . /tmp/mercurial-ci/ --noupdate
- hg -R /tmp/mercurial-ci/ update `hg log --rev '.' --template '{node}'`
- cd /tmp/mercurial-ci/rust/rhg
- cargo build
- cd /tmp/mercurial-ci/
- ls -1 tests/test-check-*.* > /tmp/check-tests.txt
......@@ -79,3 +81,9 @@ test-py3-rust:
RUNTEST_ARGS: "--rust --blacklist /tmp/check-tests.txt"
PYTHON: python3
TEST_HGMODULEPOLICY: "rust+c"
test-py2-chg:
<<: *runtests
variables:
RUNTEST_ARGS: "--blacklist /tmp/check-tests.txt --chg"
TEST_HGMODULEPOLICY: "c"
......@@ -2,14 +2,51 @@
# Uncomment this to turn on verbose mode.
# export DH_VERBOSE=1
# By default we build a .deb where the native components are built with the
# current "default" version of py3 on the build machine. If you wish to build a
# .deb that has native components built for multiple versions of py3:
#
# 1. install python3.x and python3.x-dev for each version you want
# 2. set DEB_HG_MULTI_VERSION=1 or DEB_HG_PYTHON_VERSIONS in your environment
# (if both are set, DEB_HG_PYTHON_VERSIONS has precedence)
#
# If you choose `DEB_HG_MULTI_VERSION=1`, it will build for every "supported"
# version of py3 that's installed on the build machine. This may not be equal to
# the actual versions that are installed, see the comment above where we set
# DEB_HG_PYTHON_VERSIONS below. If you choose to set `DEB_HG_PYTHON_VERSIONS`
# yourself, set it to a space-separated string of python version numbers, like:
# DEB_HG_PYTHON_VERSIONS="3.7 3.8" make deb
DEB_HG_MULTI_VERSION?=0
CPUS=$(shell cat /proc/cpuinfo | grep -E ^processor | wc -l)
# By default, only build for the version of python3 that the system considers
# the 'default' (which should be the one invoked by just running 'python3'
# without a minor version). If DEB_HG_PYTHON_VERSIONS is set, this is ignored.
ifeq ($(DEB_HG_MULTI_VERSION), 1)
# If we're building for multiple versions, use all of the "supported" versions
# on the build machine. Note: the mechanism in use here (`py3versions`) is the
# recommended one, but it relies on a file written by the python3-minimal
# package, and this file is not dynamic and does not account for manual
# installations, just the ones that would be installed by `python3-all`. This
# includes the `-i` flag, which claims it's to list all "installed" versions,
# but it doesn't. This was quite confusing, hence this tale of woe. :)
DEB_HG_PYTHON_VERSIONS?=$(shell py3versions -vs)
else
# If we're building for only one version, identify the "default" version on
# the build machine and use that when building; this is just so that we don't
# have to duplicate the rules below for multi-version vs. single-version. The
# shebang line will still be /usr/bin/python3 (no minor version).
DEB_HG_PYTHON_VERSIONS?=$(shell py3versions -vd)
endif
export HGPYTHON3=1
export PYTHON=python3
%:
dh $@ --with python3
# Note: testing can be disabled using the standard `DEB_BUILD_OPTIONS=nocheck`
override_dh_auto_test:
http_proxy='' dh_auto_test -- TESTFLAGS="-j$(CPUS)"
......@@ -24,8 +61,15 @@ override_dh_auto_build:
$(MAKE) all
$(MAKE) -C contrib/chg all
override_dh_auto_install:
python3 setup.py install --root "$(CURDIR)"/debian/mercurial --install-layout=deb
# Build the native extensions for a specfic python3 version (which must be
# installed on the build machine).
install-python%:
python$* setup.py install --root "$(CURDIR)"/debian/mercurial --install-layout=deb
# Build the final package. This rule has a dependencies section that causes the
# native extensions to be compiled for every version of python3 listed in
# DEB_HG_PYTHON_VERSIONS.
override_dh_auto_install: $(DEB_HG_PYTHON_VERSIONS:%=install-python%)
# chg
make -C contrib/chg \
DESTDIR="$(CURDIR)"/debian/mercurial \
......
......@@ -3794,19 +3794,47 @@ def perflrucache(
fm.end()
@command(b'perfwrite', formatteropts)
@command(
b'perfwrite',
formatteropts
+ [
(b'', b'write-method', b'write', b'ui write method'),
(b'', b'nlines', 100, b'number of lines'),
(b'', b'nitems', 100, b'number of items (per line)'),
(b'', b'item', b'x', b'item that is written'),
(b'', b'batch-line', None, b'pass whole line to write method at once'),
(b'', b'flush-line', None, b'flush after each line'),
],
)
def perfwrite(ui, repo, **opts):
"""microbenchmark ui.write
"""microbenchmark ui.write (and others)
"""
opts = _byteskwargs(opts)
timer, fm = gettimer(ui, opts)
write = getattr(ui, _sysstr(opts[b'write_method']))
nlines = int(opts[b'nlines'])
nitems = int(opts[b'nitems'])
item = opts[b'item']
batch_line = opts.get(b'batch_line')
flush_line = opts.get(b'flush_line')
if batch_line:
line = item * nitems + b'\n'
def write():
for i in range(100000):
ui.writenoi18n(b'Testing write performance\n')
def benchmark():
for i in pycompat.xrange(nlines):
if batch_line:
write(line)
else:
for i in pycompat.xrange(nitems):
write(item)
write(b'\n')
if flush_line:
ui.flush()
ui.flush()
timer(write)
timer, fm = gettimer(ui, opts)
timer(benchmark)
fm.end()
......
......@@ -45,8 +45,8 @@ class ParseError(Exception):
def showhelp():
pycompat.stdout.write(usage)
pycompat.stdout.write(b'\noptions:\n')
procutil.stdout.write(usage)
procutil.stdout.write(b'\noptions:\n')
out_opts = []
for shortopt, longopt, default, desc in options:
......@@ -62,11 +62,11 @@ def showhelp():
)
opts_len = max([len(opt[0]) for opt in out_opts])
for first, second in out_opts:
pycompat.stdout.write(b' %-*s %s\n' % (opts_len, first, second))
procutil.stdout.write(b' %-*s %s\n' % (opts_len, first, second))
try:
for fp in (sys.stdin, pycompat.stdout, sys.stderr):
for fp in (sys.stdin, procutil.stdout, sys.stderr):
procutil.setbinary(fp)
opts = {}
......@@ -92,11 +92,11 @@ try:
)
except ParseError as e:
e = stringutil.forcebytestr(e)
pycompat.stdout.write(b"%s: %s\n" % (sys.argv[0].encode('utf8'), e))
procutil.stdout.write(b"%s: %s\n" % (sys.argv[0].encode('utf8'), e))
showhelp()
sys.exit(1)
except error.Abort as e:
pycompat.stderr.write(b"abort: %s\n" % e)
procutil.stderr.write(b"abort: %s\n" % e)
sys.exit(255)
except KeyboardInterrupt:
sys.exit(255)
......@@ -9,7 +9,6 @@ import sys
from mercurial import (
encoding,
node,
pycompat,
revlog,
transaction,
vfs as vfsmod,
......@@ -30,7 +29,7 @@ while True:
if l.startswith("file:"):
f = encoding.strtolocal(l[6:-1])
r = revlog.revlog(opener, f)
pycompat.stdout.write(b'%s\n' % f)
procutil.stdout.write(b'%s\n' % f)
elif l.startswith("node:"):
n = node.bin(l[6:-1])
elif l.startswith("linkrev:"):
......
......@@ -50,6 +50,7 @@ from mercurial import (
phases,
pycompat,
registrar,
rewriteutil,
scmutil,
util,
)
......@@ -782,7 +783,10 @@ class fixupstate(object):
# nothing changed, nothing commited
nextp1 = ctx
continue
if self._willbecomenoop(memworkingcopy, ctx, nextp1):
willbecomenoop = ctx.files() and self._willbecomenoop(
memworkingcopy, ctx, nextp1
)
if self.skip_empty_successor and willbecomenoop:
# changeset is no longer necessary
self.replacemap[ctx.node()] = None
msg = _(b'became empty and was dropped')
......@@ -793,7 +797,11 @@ class fixupstate(object):
nextp1 = lastcommitted
self.replacemap[ctx.node()] = lastcommitted.node()
if memworkingcopy:
msg = _(b'%d file(s) changed, became %s') % (
if willbecomenoop:
msg = _(b'%d file(s) changed, became empty as %s')
else:
msg = _(b'%d file(s) changed, became %s')
msg = msg % (
len(memworkingcopy),
self._ctx2str(lastcommitted),
)
......@@ -887,6 +895,10 @@ class fixupstate(object):
if len(parents) != 1:
return False
pctx = parents[0]
if ctx.branch() != pctx.branch():
return False
if ctx.extra().get(b'close'):
return False
# ctx changes more files (not a subset of memworkingcopy)
if not set(ctx.files()).issubset(set(memworkingcopy)):
return False
......@@ -929,6 +941,10 @@ class fixupstate(object):
self.repo, replacements, operation=b'absorb', fixphase=True
)
@util.propertycache
def skip_empty_successor(self):
return rewriteutil.skip_empty_successor(self.ui, b'absorb')
def _parsechunk(hunk):
"""(crecord.uihunk or patch.recordhunk) -> (path, (a1, a2, [bline]))"""
......@@ -1045,7 +1061,7 @@ def absorb(ui, repo, stack=None, targetctx=None, pats=None, opts=None):
not opts.get(b'apply_changes')
and state.ctxaffected
and ui.promptchoice(
b"apply changes (yn)? $$ &Yes $$ &No", default=1
b"apply changes (y/N)? $$ &Yes $$ &No", default=1
)
):
raise error.Abort(_(b'absorb cancelled\n'))
......
......@@ -226,8 +226,7 @@ class convert_cvs(converter_source):
cmd = [rsh, host] + cmd
# popen2 does not support argument lists under Windows
cmd = [procutil.shellquote(arg) for arg in cmd]
cmd = procutil.quotecommand(b' '.join(cmd))
cmd = b' '.join(procutil.shellquote(arg) for arg in cmd)
self.writep, self.readp = procutil.popen2(cmd)
self.realroot = root
......
......@@ -217,7 +217,7 @@ class gnuarch_source(common.converter_source, common.commandline):
cmdline = [procutil.shellquote(arg) for arg in cmdline]
bdevnull = pycompat.bytestr(os.devnull)
cmdline += [b'>', bdevnull, b'2>', bdevnull]
cmdline = procutil.quotecommand(b' '.join(cmdline))
cmdline = b' '.join(cmdline)
self.ui.debug(cmdline, b'\n')
return os.system(pycompat.rapply(procutil.tonativestr, cmdline))
......
......@@ -1366,7 +1366,7 @@ class svn_source(converter_source):
arg = encodeargs(args)
hgexe = procutil.hgexecutable()
cmd = b'%s debugsvnlog' % procutil.shellquote(hgexe)
stdin, stdout = procutil.popen2(procutil.quotecommand(cmd))
stdin, stdout = procutil.popen2(cmd)
stdin.write(arg)
try:
stdin.close()
......
......@@ -233,7 +233,6 @@ def _systembackground(cmd, environ=None, cwd=None):
''' like 'procutil.system', but returns the Popen object directly
so we don't have to wait on it.
'''
cmd = procutil.quotecommand(cmd)
env = procutil.shellenviron(environ)
proc = subprocess.Popen(
procutil.tonativestr(cmd),
......@@ -351,6 +350,187 @@ def _runperfilediff(
proc.wait()
def diffpatch(ui, repo, node1, node2, tmproot, matcher, cmdline):
template = b'hg-%h.patch'
# write patches to temporary files
with formatter.nullformatter(ui, b'extdiff', {}) as fm:
cmdutil.export(
repo,
[repo[node1].rev(), repo[node2].rev()],
fm,
fntemplate=repo.vfs.reljoin(tmproot, template),
match=matcher,
)
label1 = cmdutil.makefilename(repo[node1], template)
label2 = cmdutil.makefilename(repo[node2], template)
file1 = repo.vfs.reljoin(tmproot, label1)
file2 = repo.vfs.reljoin(tmproot, label2)
cmdline = formatcmdline(
cmdline,
repo.root,
# no 3way while comparing patches
do3way=False,
parent1=file1,
plabel1=label1,
# while comparing patches, there is no second parent
parent2=None,
plabel2=None,
child=file2,
clabel=label2,
)
ui.debug(b'running %r in %s\n' % (pycompat.bytestr(cmdline), tmproot))
ui.system(cmdline, cwd=tmproot, blockedtag=b'extdiff')
return 1
def diffrevs(
ui,
repo,
node1a,
node1b,
node2,
matcher,
tmproot,
cmdline,
do3way,
guitool,
opts,
):
subrepos = opts.get(b'subrepos')
# calculate list of files changed between both revs
st = repo.status(node1a, node2, matcher, listsubrepos=subrepos)
mod_a, add_a, rem_a = set(st.modified), set(st.added), set(st.removed)
if do3way:
stb = repo.status(node1b, node2, matcher, listsubrepos=subrepos)
mod_b, add_b, rem_b = (
set(stb.modified),
set(stb.added),
set(stb.removed),
)
else:
mod_b, add_b, rem_b = set(), set(), set()
modadd = mod_a | add_a | mod_b | add_b
common = modadd | rem_a | rem_b
if not common:
return 0
# Always make a copy of node1a (and node1b, if applicable)
# dir1a should contain files which are: