pycompat.py 15.5 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
# pycompat.py - portability shim for python 3
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.

"""Mercurial portability shim for python 3.

This contains aliases to hide python version-specific details from the core.
"""

from __future__ import absolute_import

13
import getopt
14
import inspect
15
import json
16
import os
17
import shlex
18
import sys
19
import tempfile
20

21
ispy3 = sys.version_info[0] >= 3
22
ispypy = '__pypy__' in sys.builtin_module_names
23
24
25
26
27
28
TYPE_CHECKING = False

if not globals():  # hide this from non-pytype users
    import typing

    TYPE_CHECKING = typing.TYPE_CHECKING
29
30

if not ispy3:
31
    import cookielib
32
    import cPickle as pickle
33
    import httplib
34
    import Queue as queue
35
    import SocketServer as socketserver
36
    import xmlrpclib
37
38

    from .thirdparty.concurrent import futures
39
40
41

    def future_set_exception_info(f, exc_info):
        f.set_exception_info(*exc_info)
42
43


44
else:
45
    import concurrent.futures as futures
46
    import http.cookiejar as cookielib
47
48
    import http.client as httplib
    import pickle
49
    import queue as queue
50
    import socketserver
51
    import xmlrpc.client as xmlrpclib
52

53
54
55
    def future_set_exception_info(f, exc_info):
        f.set_exception(exc_info[0])

56

57
58
59
def identity(a):
    return a

60

61
62
63
64
65
66
67
68
69
70
def _rapply(f, xs):
    if xs is None:
        # assume None means non-value of optional data
        return xs
    if isinstance(xs, (list, set, tuple)):
        return type(xs)(_rapply(f, x) for x in xs)
    if isinstance(xs, dict):
        return type(xs)((_rapply(f, k), _rapply(f, v)) for k, v in xs.items())
    return f(xs)

71

72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
def rapply(f, xs):
    """Apply function recursively to every item preserving the data structure

    >>> def f(x):
    ...     return 'f(%s)' % x
    >>> rapply(f, None) is None
    True
    >>> rapply(f, 'a')
    'f(a)'
    >>> rapply(f, {'a'}) == {'f(a)'}
    True
    >>> rapply(f, ['a', 'b', None, {'c': 'd'}, []])
    ['f(a)', 'f(b)', None, {'f(c)': 'f(d)'}, []]

    >>> xs = [object()]
    >>> rapply(identity, xs) is xs
    True
    """
    if f is identity:
        # fast path mainly for py2
        return xs
    return _rapply(f, xs)

95

96
if ispy3:
97
    import builtins
98
    import codecs
99
    import functools
100
    import io
101
    import struct
102

103
104
105
106
107
    if os.name == r'nt' and sys.version_info >= (3, 6):
        # MBCS (or ANSI) filesystem encoding must be used as before.
        # Otherwise non-ASCII filenames in existing repositories would be
        # corrupted.
        # This must be set once prior to any fsencode/fsdecode calls.
108
        sys._enablelegacywindowsfsencoding()  # pytype: disable=module-attr
109

110
    fsencode = os.fsencode
111
    fsdecode = os.fsdecode
Yuya Nishihara's avatar
Yuya Nishihara committed
112
    oscurdir = os.curdir.encode('ascii')
113
    oslinesep = os.linesep.encode('ascii')
114
    osname = os.name.encode('ascii')
115
    ospathsep = os.pathsep.encode('ascii')
Yuya Nishihara's avatar
Yuya Nishihara committed
116
    ospardir = os.pardir.encode('ascii')
117
    ossep = os.sep.encode('ascii')
118
119
120
    osaltsep = os.altsep
    if osaltsep:
        osaltsep = osaltsep.encode('ascii')
121
    osdevnull = os.devnull.encode('ascii')
122

123
    sysplatform = sys.platform.encode('ascii')
124
125
126
    sysexecutable = sys.executable
    if sysexecutable:
        sysexecutable = os.fsencode(sysexecutable)
127
128
129
    bytesio = io.BytesIO
    # TODO deprecate stringio name, as it is a lie on Python 3.
    stringio = bytesio
130
131
132
133

    def maplist(*args):
        return list(map(*args))

134
135
136
    def rangelist(*args):
        return list(range(*args))

137
138
139
    def ziplist(*args):
        return list(zip(*args))

140
    rawinput = input
141
    getargspec = inspect.getfullargspec
142

143
144
    long = int

145
    if getattr(sys, 'argv', None) is not None:
146
        # On POSIX, the char** argv array is converted to Python str using
147
148
149
150
        # Py_DecodeLocale(). The inverse of this is Py_EncodeLocale(), which
        # isn't directly callable from Python code. In practice, os.fsencode()
        # can be used instead (this is recommended by Python's documentation
        # for sys.argv).
151
152
153
154
155
156
157
158
159
160
161
162
        #
        # On Windows, the wchar_t **argv is passed into the interpreter as-is.
        # Like POSIX, we need to emulate what Py_EncodeLocale() would do. But
        # there's an additional wrinkle. What we really want to access is the
        # ANSI codepage representation of the arguments, as this is what
        # `int main()` would receive if Python 3 didn't define `int wmain()`
        # (this is how Python 2 worked). To get that, we encode with the mbcs
        # encoding, which will pass CP_ACP to the underlying Windows API to
        # produce bytes.
        if os.name == r'nt':
            sysargv = [a.encode("mbcs", "ignore") for a in sys.argv]
        else:
163
            sysargv = [fsencode(a) for a in sys.argv]
164

165
    bytechr = struct.Struct('>B').pack
166
    byterepr = b'%r'.__mod__
167

168
169
170
171
    class bytestr(bytes):
        """A bytes which mostly acts as a Python 2 str

        >>> bytestr(), bytestr(bytearray(b'foo')), bytestr(u'ascii'), bytestr(1)
172
        ('', 'foo', 'ascii', '1')
173
174
175
        >>> s = bytestr(b'foo')
        >>> assert s is bytestr(s)

176
177
178
179
180
181
        __bytes__() should be called if provided:

        >>> class bytesable(object):
        ...     def __bytes__(self):
        ...         return b'bytes'
        >>> bytestr(bytesable())
182
        'bytes'
183

184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
        There's no implicit conversion from non-ascii str as its encoding is
        unknown:

        >>> bytestr(chr(0x80)) # doctest: +ELLIPSIS
        Traceback (most recent call last):
          ...
        UnicodeEncodeError: ...

        Comparison between bytestr and bytes should work:

        >>> assert bytestr(b'foo') == b'foo'
        >>> assert b'foo' == bytestr(b'foo')
        >>> assert b'f' in bytestr(b'foo')
        >>> assert bytestr(b'f') in b'foo'

        Sliced elements should be bytes, not integer:

        >>> s[1], s[:2]
        (b'o', b'fo')
        >>> list(s), list(reversed(s))
        ([b'f', b'o', b'o'], [b'o', b'o', b'f'])

        As bytestr type isn't propagated across operations, you need to cast
        bytes to bytestr explicitly:

        >>> s = bytestr(b'foo').upper()
        >>> t = bytestr(s)
        >>> s[0], t[0]
        (70, b'F')

        Be careful to not pass a bytestr object to a function which expects
        bytearray-like behavior.

        >>> t = bytes(t)  # cast to bytes
        >>> assert type(t) is bytes
        """

        def __new__(cls, s=b''):
            if isinstance(s, bytestr):
                return s
224
225
226
227
228
            if not isinstance(
                s, (bytes, bytearray)
            ) and not hasattr(  # hasattr-py3-only
                s, u'__bytes__'
            ):
229
                s = str(s).encode('ascii')
230
231
232
233
234
235
236
237
238
239
240
            return bytes.__new__(cls, s)

        def __getitem__(self, key):
            s = bytes.__getitem__(self, key)
            if not isinstance(s, bytes):
                s = bytechr(s)
            return s

        def __iter__(self):
            return iterbytestr(bytes.__iter__(self))

241
242
243
        def __repr__(self):
            return bytes.__repr__(self)[1:]  # drop b''

244
245
    def iterbytestr(s):
        """Iterate bytes as if it were a str object of Python 2"""
246
        return map(bytechr, s)
247

248
249
250
251
252
253
    def maybebytestr(s):
        """Promote bytes to bytestr"""
        if isinstance(s, bytes):
            return bytestr(s)
        return s

254
255
256
257
258
259
    def sysbytes(s):
        """Convert an internal str (e.g. keyword, __doc__) back to bytes

        This never raises UnicodeEncodeError, but only ASCII characters
        can be round-trip by sysstr(sysbytes(s)).
        """
260
261
        if isinstance(s, bytes):
            return s
262
        return s.encode('utf-8')
263

264
265
266
267
268
269
270
271
272
273
    def sysstr(s):
        """Return a keyword str to be passed to Python functions such as
        getattr() and str.encode()

        This never raises UnicodeDecodeError. Non-ascii characters are
        considered invalid and mapped to arbitrary but unique code points
        such that 'sysstr(a) != sysstr(b)' for all 'a != b'.
        """
        if isinstance(s, builtins.str):
            return s
274
        return s.decode('latin-1')
275

276
277
    def strurl(url):
        """Converts a bytes url back to str"""
278
        if isinstance(url, bytes):
279
            return url.decode('ascii')
280
        return url
281

282
283
    def bytesurl(url):
        """Converts a str url to bytes by encoding in ascii"""
284
        if isinstance(url, str):
285
            return url.encode('ascii')
286
        return url
287

288
289
290
291
    def raisewithtb(exc, tb):
        """Raise exception with the given traceback"""
        raise exc.with_traceback(tb)

292
293
294
    def getdoc(obj):
        """Get docstring as bytes; may be None so gettext() won't confuse it
        with _('')"""
295
        doc = getattr(obj, '__doc__', None)
296
297
298
299
        if doc is None:
            return doc
        return sysbytes(doc)

300
301
302
    def _wrapattrfunc(f):
        @functools.wraps(f)
        def w(object, name, *args):
303
            return f(object, sysstr(name), *args)
304

305
306
        return w

307
    # these wrappers are automagically imported by hgloader
308
309
310
311
    delattr = _wrapattrfunc(builtins.delattr)
    getattr = _wrapattrfunc(builtins.getattr)
    hasattr = _wrapattrfunc(builtins.hasattr)
    setattr = _wrapattrfunc(builtins.setattr)
312
    xrange = builtins.range
313
    unicode = str
314

315
    def open(name, mode=b'r', buffering=-1, encoding=None):
316
        return builtins.open(name, sysstr(mode), buffering, encoding)
317

318
319
    safehasattr = _wrapattrfunc(builtins.hasattr)

320
    def _getoptbwrapper(orig, args, shortlist, namelist):
321
322
323
324
325
326
        """
        Takes bytes arguments, converts them to unicode, pass them to
        getopt.getopt(), convert the returned values back to bytes and then
        return them for Python 3 compatibility as getopt.getopt() don't accepts
        bytes on Python 3.
        """
327
328
329
        args = [a.decode('latin-1') for a in args]
        shortlist = shortlist.decode('latin-1')
        namelist = [a.decode('latin-1') for a in namelist]
330
        opts, args = orig(args, shortlist, namelist)
331
        opts = [(a[0].encode('latin-1'), a[1].encode('latin-1')) for a in opts]
332
333
334
        args = [a.encode('latin-1') for a in args]
        return opts, args

335
    def strkwargs(dic):
336
337
338
339
340
        """
        Converts the keys of a python dictonary to str i.e. unicodes so that
        they can be passed as keyword arguments as dictonaries with bytes keys
        can't be passed as keyword arguments to functions on Python 3.
        """
341
        dic = {k.decode('latin-1'): v for k, v in dic.items()}
342
343
344
        return dic

    def byteskwargs(dic):
345
346
347
348
        """
        Converts keys of python dictonaries to bytes as they were converted to
        str to pass that dictonary as a keyword argument on Python 3.
        """
349
        dic = {k.encode('latin-1'): v for k, v in dic.items()}
350
351
        return dic

352
    # TODO: handle shlex.shlex().
353
    def shlexsplit(s, comments=False, posix=True):
354
355
356
357
358
        """
        Takes bytes argument, convert it to str i.e. unicodes, pass that into
        shlex.split(), convert the returned value to bytes and return that for
        Python 3 compatibility as shelx.split() don't accept bytes on Python 3.
        """
359
        ret = shlex.split(s.decode('latin-1'), comments, posix)
360
361
        return [a.encode('latin-1') for a in ret]

362
    iteritems = lambda x: x.items()
363
    itervalues = lambda x: x.values()
364

365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
    # Python 3.5's json.load and json.loads require str. We polyfill its
    # code for detecting encoding from bytes.
    if sys.version_info[0:2] < (3, 6):

        def _detect_encoding(b):
            bstartswith = b.startswith
            if bstartswith((codecs.BOM_UTF32_BE, codecs.BOM_UTF32_LE)):
                return 'utf-32'
            if bstartswith((codecs.BOM_UTF16_BE, codecs.BOM_UTF16_LE)):
                return 'utf-16'
            if bstartswith(codecs.BOM_UTF8):
                return 'utf-8-sig'

            if len(b) >= 4:
                if not b[0]:
                    # 00 00 -- -- - utf-32-be
                    # 00 XX -- -- - utf-16-be
                    return 'utf-16-be' if b[1] else 'utf-32-be'
                if not b[1]:
                    # XX 00 00 00 - utf-32-le
                    # XX 00 00 XX - utf-16-le
                    # XX 00 XX -- - utf-16-le
                    return 'utf-16-le' if b[2] or b[3] else 'utf-32-le'
            elif len(b) == 2:
                if not b[0]:
                    # 00 XX - utf-16-be
                    return 'utf-16-be'
                if not b[1]:
                    # XX 00 - utf-16-le
                    return 'utf-16-le'
            # default
            return 'utf-8'

        def json_loads(s, *args, **kwargs):
            if isinstance(s, (bytes, bytearray)):
                s = s.decode(_detect_encoding(s), 'surrogatepass')

            return json.loads(s, *args, **kwargs)

    else:
        json_loads = json.loads

407
else:
408
409
    import cStringIO

410
    xrange = xrange
411
    unicode = unicode
412
    bytechr = chr
413
    byterepr = repr
414
    bytestr = str
415
    iterbytestr = iter
416
    maybebytestr = identity
417
    sysbytes = identity
418
    sysstr = identity
419
    strurl = identity
420
    bytesurl = identity
421
    open = open
422
    delattr = delattr
423
    getattr = getattr
424
    hasattr = hasattr
425
    setattr = setattr
426

427
    # this can't be parsed on Python 3
428
    exec(b'def raisewithtb(exc, tb):\n    raise exc, None, tb\n')
429

430
    def fsencode(filename):
431
432
433
434
435
        """
        Partial backport from os.py in Python 3, which only accepts bytes.
        In Python 2, our paths should only ever be bytes, a unicode path
        indicates a bug.
        """
436
437
        if isinstance(filename, str):
            return filename
438
        else:
439
            raise TypeError("expect str, not %s" % type(filename).__name__)
440

441
442
    # In Python 2, fsdecode() has a very chance to receive bytes. So it's
    # better not to touch Python 2 part as it's already working fine.
443
    fsdecode = identity
444

445
446
447
    def getdoc(obj):
        return getattr(obj, '__doc__', None)

448
449
450
451
452
    _notset = object()

    def safehasattr(thing, attr):
        return getattr(thing, attr, _notset) is not _notset

453
454
    def _getoptbwrapper(orig, args, shortlist, namelist):
        return orig(args, shortlist, namelist)
455

456
457
    strkwargs = identity
    byteskwargs = identity
458

Yuya Nishihara's avatar
Yuya Nishihara committed
459
    oscurdir = os.curdir
460
    oslinesep = os.linesep
461
    osname = os.name
462
    ospathsep = os.pathsep
Yuya Nishihara's avatar
Yuya Nishihara committed
463
    ospardir = os.pardir
464
    ossep = os.sep
465
    osaltsep = os.altsep
466
    osdevnull = os.devnull
467
    long = long
468
469
    if getattr(sys, 'argv', None) is not None:
        sysargv = sys.argv
470
    sysplatform = sys.platform
471
    sysexecutable = sys.executable
472
    shlexsplit = shlex.split
473
474
    bytesio = cStringIO.StringIO
    stringio = bytesio
475
    maplist = map
476
    rangelist = range
477
    ziplist = zip
478
    rawinput = raw_input
479
    getargspec = inspect.getargspec
480
    iteritems = lambda x: x.iteritems()
481
    itervalues = lambda x: x.itervalues()
482
    json_loads = json.loads
483

484
isjython = sysplatform.startswith(b'java')
485

486
487
isdarwin = sysplatform.startswith(b'darwin')
islinux = sysplatform.startswith(b'linux')
488
489
isposix = osname == b'posix'
iswindows = osname == b'nt'
490

491

492
493
494
def getoptb(args, shortlist, namelist):
    return _getoptbwrapper(getopt.getopt, args, shortlist, namelist)

495

496
497
def gnugetoptb(args, shortlist, namelist):
    return _getoptbwrapper(getopt.gnu_getopt, args, shortlist, namelist)
498

499

500
501
502
def mkdtemp(suffix=b'', prefix=b'tmp', dir=None):
    return tempfile.mkdtemp(suffix, prefix, dir)

503

504
505
506
# text=True is not supported; use util.from/tonativeeol() instead
def mkstemp(suffix=b'', prefix=b'tmp', dir=None):
    return tempfile.mkstemp(suffix, prefix, dir)
507

508

509
# mode must include 'b'ytes as encoding= is not supported
510
511
512
def namedtempfile(
    mode=b'w+b', bufsize=-1, suffix=b'', prefix=b'tmp', dir=None, delete=True
):
513
    mode = sysstr(mode)
514
    assert 'b' in mode
515
516
517
    return tempfile.NamedTemporaryFile(
        mode, bufsize, suffix=suffix, prefix=prefix, dir=dir, delete=delete
    )