procutil.py 22.1 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
# procutil.py - utility for managing processes and executable environment
#
#  Copyright 2005 K. Thananchayan <thananck@yahoo.com>
#  Copyright 2005-2007 Matt Mackall <mpm@selenic.com>
#  Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.

from __future__ import absolute_import

12
import contextlib
13
import errno
14
15
16
17
18
import io
import os
import signal
import subprocess
import sys
19
import threading
20
21
22
import time

from ..i18n import _
23
24
25
26
from ..pycompat import (
    getattr,
    open,
)
27
28
29
30
31
32
33
34

from .. import (
    encoding,
    error,
    policy,
    pycompat,
)

35
36
37
# Import like this to keep import-checker happy
from ..utils import resourceutil

38
osutil = policy.importmod('osutil')
39

40
41
42
43
44
if pycompat.iswindows:
    from .. import windows as platform
else:
    from .. import posix as platform

45

46
47
48
49
50
51
def isatty(fp):
    try:
        return fp.isatty()
    except AttributeError:
        return False

52

53
54
55
class LineBufferedWrapper(object):
    def __init__(self, orig):
        self.orig = orig
56

57
58
    def __getattr__(self, attr):
        return getattr(self.orig, attr)
59

60
61
62
63
64
65
    def write(self, s):
        orig = self.orig
        res = orig.write(s)
        if s.endswith(b'\n'):
            orig.flush()
        return res
66

67
68

io.BufferedIOBase.register(LineBufferedWrapper)
69
70


71
72
73
74
75
76
77
78
79
80
81
82
def make_line_buffered(stream):
    if pycompat.ispy3 and not isinstance(stream, io.BufferedIOBase):
        # On Python 3, buffered streams can be expected to subclass
        # BufferedIOBase. This is definitively the case for the streams
        # initialized by the interpreter. For unbuffered streams, we don't need
        # to emulate line buffering.
        return stream
    if isinstance(stream, LineBufferedWrapper):
        return stream
    return LineBufferedWrapper(stream)


83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
class WriteAllWrapper(object):
    def __init__(self, orig):
        self.orig = orig

    def __getattr__(self, attr):
        return getattr(self.orig, attr)

    def write(self, s):
        write1 = self.orig.write
        m = memoryview(s)
        total_to_write = len(s)
        total_written = 0
        while total_written < total_to_write:
            total_written += write1(m[total_written:])
        return total_written


io.IOBase.register(WriteAllWrapper)


103
def _make_write_all(stream):
104
105
106
107
108
109
110
111
112
113
114
115
    assert pycompat.ispy3
    if isinstance(stream, WriteAllWrapper):
        return stream
    if isinstance(stream, io.BufferedIOBase):
        # The io.BufferedIOBase.write() contract guarantees that all data is
        # written.
        return stream
    # In general, the write() method of streams is free to write only part of
    # the data.
    return WriteAllWrapper(stream)


116
if pycompat.ispy3:
117
    # Python 3 implements its own I/O streams.
118
119
120
    # TODO: .buffer might not exist if std streams were replaced; we'll need
    # a silly wrapper to make a bytes stream backed by a unicode one.
    stdin = sys.stdin.buffer
121
    stdout = _make_write_all(sys.stdout.buffer)
122
123
124
125
126
    stderr = _make_write_all(sys.stderr.buffer)
    if pycompat.iswindows:
        # Work around Windows bugs.
        stdout = platform.winstdout(stdout)
        stderr = platform.winstdout(stderr)
127
128
129
    if isatty(stdout):
        # The standard library doesn't offer line-buffered binary streams.
        stdout = make_line_buffered(stdout)
130
else:
131
    # Python 2 uses the I/O streams provided by the C library.
132
133
    stdin = sys.stdin
    stdout = sys.stdout
134
135
136
137
138
    stderr = sys.stderr
    if pycompat.iswindows:
        # Work around Windows bugs.
        stdout = platform.winstdout(stdout)
        stderr = platform.winstdout(stderr)
139
140
141
142
143
144
145
146
147
    if isatty(stdout):
        if pycompat.iswindows:
            # The Windows C runtime library doesn't support line buffering.
            stdout = make_line_buffered(stdout)
        else:
            # glibc determines buffering on first write to stdout - if we
            # replace a TTY destined stdout with a pipe destined stdout (e.g.
            # pager), we want line buffering.
            stdout = os.fdopen(stdout.fileno(), 'wb', 1)
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174


findexe = platform.findexe
_gethgcmd = platform.gethgcmd
getuser = platform.getuser
getpid = os.getpid
hidewindow = platform.hidewindow
readpipe = platform.readpipe
setbinary = platform.setbinary
setsignalhandler = platform.setsignalhandler
shellquote = platform.shellquote
shellsplit = platform.shellsplit
spawndetached = platform.spawndetached
sshargs = platform.sshargs
testpid = platform.testpid

try:
    setprocname = osutil.setprocname
except AttributeError:
    pass
try:
    unblocksignal = osutil.unblocksignal
except AttributeError:
    pass

closefds = pycompat.isposix

175

176
def explainexit(code):
177
    """return a message describing a subprocess status
178
179
    (codes from kill are negative - not os.system/wait encoding)"""
    if code >= 0:
180
181
        return _(b"exited with status %d") % code
    return _(b"killed by signal %d") % -code
182

183

184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
class _pfile(object):
    """File-like wrapper for a stream opened by subprocess.Popen()"""

    def __init__(self, proc, fp):
        self._proc = proc
        self._fp = fp

    def close(self):
        # unlike os.popen(), this returns an integer in subprocess coding
        self._fp.close()
        return self._proc.wait()

    def __iter__(self):
        return iter(self._fp)

    def __getattr__(self, attr):
        return getattr(self._fp, attr)

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, exc_tb):
        self.close()

208

209
210
def popen(cmd, mode=b'rb', bufsize=-1):
    if mode == b'rb':
211
        return _popenreader(cmd, bufsize)
212
    elif mode == b'wb':
213
        return _popenwriter(cmd, bufsize)
214
    raise error.ProgrammingError(b'unsupported mode: %r' % mode)
215

216

217
def _popenreader(cmd, bufsize):
218
    p = subprocess.Popen(
219
        tonativestr(cmd),
220
221
222
223
224
        shell=True,
        bufsize=bufsize,
        close_fds=closefds,
        stdout=subprocess.PIPE,
    )
225
226
    return _pfile(p, p.stdout)

227

228
def _popenwriter(cmd, bufsize):
229
    p = subprocess.Popen(
230
        tonativestr(cmd),
231
232
233
234
235
        shell=True,
        bufsize=bufsize,
        close_fds=closefds,
        stdin=subprocess.PIPE,
    )
236
237
    return _pfile(p, p.stdin)

238

239
def popen2(cmd, env=None):
240
241
242
    # Setting bufsize to -1 lets the system decide the buffer size.
    # The default for bufsize is 0, meaning unbuffered. This leads to
    # poor performance on Mac OS X: http://bugs.python.org/issue4194
243
244
245
246
247
248
249
250
251
    p = subprocess.Popen(
        tonativestr(cmd),
        shell=True,
        bufsize=-1,
        close_fds=closefds,
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        env=tonativeenv(env),
    )
252
253
    return p.stdin, p.stdout

254

255
256
def popen3(cmd, env=None):
    stdin, stdout, stderr, p = popen4(cmd, env)
257
258
    return stdin, stdout, stderr

259

260
def popen4(cmd, env=None, bufsize=-1):
261
262
263
264
265
266
267
268
269
270
    p = subprocess.Popen(
        tonativestr(cmd),
        shell=True,
        bufsize=bufsize,
        close_fds=closefds,
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        env=tonativeenv(env),
    )
271
272
    return p.stdin, p.stdout, p.stderr, p

273

274
275
def pipefilter(s, cmd):
    '''filter string S through command CMD, returning its output'''
276
277
278
279
280
281
282
    p = subprocess.Popen(
        tonativestr(cmd),
        shell=True,
        close_fds=closefds,
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
    )
283
284
285
    pout, perr = p.communicate(s)
    return pout

286

287
288
289
290
291
292
293
def tempfilter(s, cmd):
    '''filter string S through a pair of temporary files with CMD.
    CMD is used as a template to create the real command to be run,
    with the strings INFILE and OUTFILE replaced by the real names of
    the temporary files generated.'''
    inname, outname = None, None
    try:
294
        infd, inname = pycompat.mkstemp(prefix=b'hg-filter-in-')
295
        fp = os.fdopen(infd, 'wb')
296
297
        fp.write(s)
        fp.close()
298
        outfd, outname = pycompat.mkstemp(prefix=b'hg-filter-out-')
299
        os.close(outfd)
300
301
        cmd = cmd.replace(b'INFILE', inname)
        cmd = cmd.replace(b'OUTFILE', outname)
302
        code = system(cmd)
303
        if pycompat.sysplatform == b'OpenVMS' and code & 1:
304
305
            code = 0
        if code:
306
            raise error.Abort(
307
                _(b"command '%s' failed: %s") % (cmd, explainexit(code))
308
            )
309
        with open(outname, b'rb') as fp:
310
311
312
313
314
315
316
317
318
319
320
321
322
            return fp.read()
    finally:
        try:
            if inname:
                os.unlink(inname)
        except OSError:
            pass
        try:
            if outname:
                os.unlink(outname)
        except OSError:
            pass

323

324
_filtertable = {
325
326
    b'tempfile:': tempfilter,
    b'pipe:': pipefilter,
327
328
}

329

330
def filter(s, cmd):
331
332
    """filter a string through a command that transforms its input to its
    output"""
333
    for name, fn in pycompat.iteritems(_filtertable):
334
        if cmd.startswith(name):
335
            return fn(s, cmd[len(name) :].lstrip())
336
337
    return pipefilter(s, cmd)

338

339
340
_hgexecutable = None

341

342
343
344
345
346
347
def hgexecutable():
    """return location of the 'hg' executable.

    Defaults to $HG or 'hg' in the search path.
    """
    if _hgexecutable is None:
348
        hg = encoding.environ.get(b'HG')
349
        mainmod = sys.modules['__main__']
350
351
        if hg:
            _sethgexecutable(hg)
352
        elif resourceutil.mainfrozen():
353
            if getattr(sys, 'frozen', None) == 'macosx_app':
354
                # Env variable set by py2app
355
                _sethgexecutable(encoding.environ[b'EXECUTABLEPATH'])
356
357
            else:
                _sethgexecutable(pycompat.sysexecutable)
358
359
        elif (
            not pycompat.iswindows
360
            and os.path.basename(getattr(mainmod, '__file__', '')) == 'hg'
361
        ):
362
363
            _sethgexecutable(pycompat.fsencode(mainmod.__file__))
        else:
364
            _sethgexecutable(
365
                findexe(b'hg') or os.path.basename(pycompat.sysargv[0])
366
            )
367
368
    return _hgexecutable

369

370
371
372
373
374
def _sethgexecutable(path):
    """set location of the 'hg' executable"""
    global _hgexecutable
    _hgexecutable = path

375

376
377
378
379
380
def _testfileno(f, stdf):
    fileno = getattr(f, 'fileno', None)
    try:
        return fileno and fileno() == stdf.fileno()
    except io.UnsupportedOperation:
381
382
        return False  # fileno() raised UnsupportedOperation

383
384
385
386

def isstdin(f):
    return _testfileno(f, sys.__stdin__)

387

388
389
390
def isstdout(f):
    return _testfileno(f, sys.__stdout__)

391

392
def protectstdio(uin, uout):
393
394
395
396
    """Duplicate streams and redirect original if (uin, uout) are stdio

    If uin is stdin, it's redirected to /dev/null. If uout is stdout, it's
    redirected to stderr so the output is still readable.
397
398
399
400
401
402

    Returns (fin, fout) which point to the original (uin, uout) fds, but
    may be copy of (uin, uout). The returned streams can be considered
    "owned" in that print(), exec(), etc. never reach to them.
    """
    uout.flush()
403
    fin, fout = uin, uout
404
    if _testfileno(uin, stdin):
405
        newfd = os.dup(uin.fileno())
406
        nullfd = os.open(os.devnull, os.O_RDONLY)
407
        os.dup2(nullfd, uin.fileno())
408
        os.close(nullfd)
409
        fin = os.fdopen(newfd, 'rb')
410
    if _testfileno(uout, stdout):
411
        newfd = os.dup(uout.fileno())
412
        os.dup2(stderr.fileno(), uout.fileno())
413
        fout = os.fdopen(newfd, 'wb')
414
    return fin, fout
415

416

417
418
419
420
421
422
423
424
def restorestdio(uin, uout, fin, fout):
    """Restore (uin, uout) streams from possibly duplicated (fin, fout)"""
    uout.flush()
    for f, uif in [(fin, uin), (fout, uout)]:
        if f is not uif:
            os.dup2(f.fileno(), uif.fileno())
            f.close()

425

426
427
def shellenviron(environ=None):
    """return environ with optional override, useful for shelling out"""
428

429
    def py2shell(val):
430
        """convert python object into string that is useful to shell"""
431
        if val is None or val is False:
432
            return b'0'
433
        if val is True:
434
            return b'1'
435
        return pycompat.bytestr(val)
436

437
438
    env = dict(encoding.environ)
    if environ:
439
        env.update((k, py2shell(v)) for k, v in pycompat.iteritems(environ))
440
    env[b'HG'] = hgexecutable()
441
442
    return env

443

444
if pycompat.iswindows:
445

446
    def shelltonative(cmd, env):
447
448
449
        return platform.shelltocmdexe(  # pytype: disable=module-attr
            cmd, shellenviron(env)
        )
450
451

    tonativestr = encoding.strfromlocal
452
else:
453

454
455
456
    def shelltonative(cmd, env):
        return cmd

457
458
    tonativestr = pycompat.identity

459

460
461
462
463
464
def tonativeenv(env):
    '''convert the environment from bytes to strings suitable for Popen(), etc.
    '''
    return pycompat.rapply(tonativestr, env)

465

466
467
468
469
470
471
472
473
474
475
476
477
def system(cmd, environ=None, cwd=None, out=None):
    '''enhanced shell command execution.
    run with environment maybe modified, maybe in different dir.

    if out is specified, it is assumed to be a file-like object that has a
    write() method. stdout and stderr will be redirected to out.'''
    try:
        stdout.flush()
    except Exception:
        pass
    env = shellenviron(environ)
    if out is None or isstdout(out):
478
479
480
481
482
483
484
        rc = subprocess.call(
            tonativestr(cmd),
            shell=True,
            close_fds=closefds,
            env=tonativeenv(env),
            cwd=pycompat.rapply(tonativestr, cwd),
        )
485
    else:
486
487
488
489
490
491
492
493
494
        proc = subprocess.Popen(
            tonativestr(cmd),
            shell=True,
            close_fds=closefds,
            env=tonativeenv(env),
            cwd=pycompat.rapply(tonativestr, cwd),
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT,
        )
495
        for line in iter(proc.stdout.readline, b''):
496
497
498
            out.write(line)
        proc.wait()
        rc = proc.returncode
499
    if pycompat.sysplatform == b'OpenVMS' and rc & 1:
500
501
502
        rc = 0
    return rc

503

504
505
506
507
_is_gui = None


def _gui():
508
509
    '''Are we running in a GUI?'''
    if pycompat.isdarwin:
510
        if b'SSH_CONNECTION' in encoding.environ:
511
512
513
514
515
516
517
518
519
            # handle SSH access to a box where the user is logged in
            return False
        elif getattr(osutil, 'isgui', None):
            # check if a CoreGraphics session is available
            return osutil.isgui()
        else:
            # pure build; use a safe default
            return True
    else:
520
        return pycompat.iswindows or encoding.environ.get(b"DISPLAY")
521

522

523
524
525
526
527
528
529
def gui():
    global _is_gui
    if _is_gui is None:
        _is_gui = _gui()
    return _is_gui


530
531
532
533
534
535
536
def hgcmd():
    """Return the command used to execute current hg

    This is different from hgexecutable() because on Windows we want
    to avoid things opening new shell windows like batch files, so we
    get either the python call or current executable.
    """
537
    if resourceutil.mainfrozen():
538
        if getattr(sys, 'frozen', None) == 'macosx_app':
539
            # Env variable set by py2app
540
            return [encoding.environ[b'EXECUTABLEPATH']]
541
542
543
544
        else:
            return [pycompat.sysexecutable]
    return _gethgcmd()

545

546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
def rundetached(args, condfn):
    """Execute the argument list in a detached process.

    condfn is a callable which is called repeatedly and should return
    True once the child process is known to have started successfully.
    At this point, the child process PID is returned. If the child
    process fails to start or finishes before condfn() evaluates to
    True, return -1.
    """
    # Windows case is easier because the child process is either
    # successfully starting and validating the condition or exiting
    # on failure. We just poll on its PID. On Unix, if the child
    # process fails to start, it will be left in a zombie state until
    # the parent wait on it, which we cannot do since we expect a long
    # running process on success. Instead we listen for SIGCHLD telling
    # us our child process terminated.
    terminated = set()
563

564
565
    def handler(signum, frame):
        terminated.add(os.wait())
566

567
568
569
570
571
572
573
    prevhandler = None
    SIGCHLD = getattr(signal, 'SIGCHLD', None)
    if SIGCHLD is not None:
        prevhandler = signal.signal(SIGCHLD, handler)
    try:
        pid = spawndetached(args)
        while not condfn():
574
            if (pid in terminated or not testpid(pid)) and not condfn():
575
576
577
578
579
580
                return -1
            time.sleep(0.1)
        return pid
    finally:
        if prevhandler is not None:
            signal.signal(signal.SIGCHLD, prevhandler)
581

582

583
@contextlib.contextmanager
584
def uninterruptible(warn):
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
    """Inhibit SIGINT handling on a region of code.

    Note that if this is called in a non-main thread, it turns into a no-op.

    Args:
      warn: A callable which takes no arguments, and returns True if the
            previous signal handling should be restored.
    """

    oldsiginthandler = [signal.getsignal(signal.SIGINT)]
    shouldbail = []

    def disabledsiginthandler(*args):
        if warn():
            signal.signal(signal.SIGINT, oldsiginthandler[0])
            del oldsiginthandler[0]
        shouldbail.append(True)

    try:
        try:
            signal.signal(signal.SIGINT, disabledsiginthandler)
        except ValueError:
            # wrong thread, oh well, we tried
            del oldsiginthandler[0]
        yield
    finally:
        if oldsiginthandler:
            signal.signal(signal.SIGINT, oldsiginthandler[0])
        if shouldbail:
            raise KeyboardInterrupt
615

616

617
618
619
620
621
if pycompat.iswindows:
    # no fork on Windows, but we can create a detached process
    # https://msdn.microsoft.com/en-us/library/windows/desktop/ms684863.aspx
    # No stdlib constant exists for this value
    DETACHED_PROCESS = 0x00000008
622
623
624
    # Following creation flags might create a console GUI window.
    # Using subprocess.CREATE_NEW_CONSOLE might helps.
    # See https://phab.mercurial-scm.org/D1701 for discussion
625
626
627
628
    _creationflags = (
        DETACHED_PROCESS
        | subprocess.CREATE_NEW_PROCESS_GROUP  # pytype: disable=module-attr
    )
629

630
    def runbgcommand(
631
632
633
634
635
636
637
        script,
        env,
        shell=False,
        stdout=None,
        stderr=None,
        ensurestart=True,
        record_wait=None,
638
    ):
639
640
641
        '''Spawn a command without waiting for it to finish.'''
        # we can't use close_fds *and* redirect stdin. I'm not sure that we
        # need to because the detached process has no console connection.
642
        p = subprocess.Popen(
643
            pycompat.rapply(tonativestr, script),
644
645
646
647
648
649
650
            shell=shell,
            env=tonativeenv(env),
            close_fds=True,
            creationflags=_creationflags,
            stdout=stdout,
            stderr=stderr,
        )
651
652
        if record_wait is not None:
            record_wait(p.wait)
653
654


655
else:
656

657
    def runbgcommand(
658
659
660
661
662
663
664
        cmd,
        env,
        shell=False,
        stdout=None,
        stderr=None,
        ensurestart=True,
        record_wait=None,
665
    ):
666
667
668
669
670
671
672
673
        '''Spawn a command without waiting for it to finish.


        When `record_wait` is not None, the spawned process will not be fully
        detached and the `record_wait` argument will be called with a the
        `Subprocess.wait` function for the spawned process.  This is mostly
        useful for developers that need to make sure the spawned process
        finished before a certain point. (eg: writing test)'''
674
675
676
677
678
        if pycompat.isdarwin:
            # avoid crash in CoreFoundation in case another thread
            # calls gui() while we're calling fork().
            gui()

679
680
        # double-fork to completely detach from the parent process
        # based on http://code.activestate.com/recipes/278731
681
682
683
684
        if record_wait is None:
            pid = os.fork()
            if pid:
                if not ensurestart:
685
686
687
688
689
690
691
                    # Even though we're not waiting on the child process,
                    # we still must call waitpid() on it at some point so
                    # it's not a zombie/defunct. This is especially relevant for
                    # chg since the parent process won't die anytime soon.
                    # We use a thread to make the overhead tiny.
                    def _do_wait():
                        os.waitpid(pid, 0)
692

693
694
695
                    t = threading.Thread(target=_do_wait)
                    t.daemon = True
                    t.start()
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
                    return
                # Parent process
                (_pid, status) = os.waitpid(pid, 0)
                if os.WIFEXITED(status):
                    returncode = os.WEXITSTATUS(status)
                else:
                    returncode = -(os.WTERMSIG(status))
                if returncode != 0:
                    # The child process's return code is 0 on success, an errno
                    # value on failure, or 255 if we don't have a valid errno
                    # value.
                    #
                    # (It would be slightly nicer to return the full exception info
                    # over a pipe as the subprocess module does.  For now it
                    # doesn't seem worth adding that complexity here, though.)
                    if returncode == 255:
                        returncode = errno.EINVAL
                    raise OSError(
                        returncode,
                        b'error running %r: %s'
                        % (cmd, os.strerror(returncode)),
                    )
718
                return
719
720
721

        returncode = 255
        try:
722
723
724
            if record_wait is None:
                # Start a new session
                os.setsid()
725

726
            stdin = open(os.devnull, b'r')
727
            if stdout is None:
728
                stdout = open(os.devnull, b'w')
729
            if stderr is None:
730
                stderr = open(os.devnull, b'w')
731
732
733

            # connect stdin to devnull to make sure the subprocess can't
            # muck up that stream for mercurial.
734
            p = subprocess.Popen(
735
736
737
738
739
740
741
742
                cmd,
                shell=shell,
                env=env,
                close_fds=True,
                stdin=stdin,
                stdout=stdout,
                stderr=stderr,
            )
743
744
            if record_wait is not None:
                record_wait(p.wait)
745
746
            returncode = 0
        except EnvironmentError as ex:
747
            returncode = ex.errno & 0xFF
748
749
750
751
752
753
754
755
756
            if returncode == 0:
                # This shouldn't happen, but just in case make sure the
                # return code is never 0 here.
                returncode = 255
        except Exception:
            returncode = 255
        finally:
            # mission accomplished, this child needs to exit and not
            # continue the hg process here.
757
758
            if record_wait is None:
                os._exit(returncode)