procutil.py 21.9 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
# procutil.py - utility for managing processes and executable environment
#
#  Copyright 2005 K. Thananchayan <thananck@yahoo.com>
#  Copyright 2005-2007 Matt Mackall <mpm@selenic.com>
#  Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.

from __future__ import absolute_import

12
import contextlib
13
import errno
14
15
16
17
18
import io
import os
import signal
import subprocess
import sys
19
import threading
20
21
22
import time

from ..i18n import _
23
24
25
26
from ..pycompat import (
    getattr,
    open,
)
27
28
29
30
31
32
33
34

from .. import (
    encoding,
    error,
    policy,
    pycompat,
)

35
36
37
# Import like this to keep import-checker happy
from ..utils import resourceutil

38
osutil = policy.importmod('osutil')
39

40
41
42
43
44
if pycompat.iswindows:
    from .. import windows as platform
else:
    from .. import posix as platform

45

46
47
48
49
50
51
def isatty(fp):
    try:
        return fp.isatty()
    except AttributeError:
        return False

52

53
54
55
class LineBufferedWrapper(object):
    def __init__(self, orig):
        self.orig = orig
56

57
58
    def __getattr__(self, attr):
        return getattr(self.orig, attr)
59

60
61
62
63
64
65
    def write(self, s):
        orig = self.orig
        res = orig.write(s)
        if s.endswith(b'\n'):
            orig.flush()
        return res
66

67
68

io.BufferedIOBase.register(LineBufferedWrapper)
69
70


71
72
73
74
75
76
77
78
79
80
81
82
def make_line_buffered(stream):
    if pycompat.ispy3 and not isinstance(stream, io.BufferedIOBase):
        # On Python 3, buffered streams can be expected to subclass
        # BufferedIOBase. This is definitively the case for the streams
        # initialized by the interpreter. For unbuffered streams, we don't need
        # to emulate line buffering.
        return stream
    if isinstance(stream, LineBufferedWrapper):
        return stream
    return LineBufferedWrapper(stream)


83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
class WriteAllWrapper(object):
    def __init__(self, orig):
        self.orig = orig

    def __getattr__(self, attr):
        return getattr(self.orig, attr)

    def write(self, s):
        write1 = self.orig.write
        m = memoryview(s)
        total_to_write = len(s)
        total_written = 0
        while total_written < total_to_write:
            total_written += write1(m[total_written:])
        return total_written


io.IOBase.register(WriteAllWrapper)


103
def _make_write_all(stream):
104
105
106
107
108
109
110
111
112
113
114
115
    assert pycompat.ispy3
    if isinstance(stream, WriteAllWrapper):
        return stream
    if isinstance(stream, io.BufferedIOBase):
        # The io.BufferedIOBase.write() contract guarantees that all data is
        # written.
        return stream
    # In general, the write() method of streams is free to write only part of
    # the data.
    return WriteAllWrapper(stream)


116
if pycompat.ispy3:
117
    # Python 3 implements its own I/O streams.
118
119
120
    # TODO: .buffer might not exist if std streams were replaced; we'll need
    # a silly wrapper to make a bytes stream backed by a unicode one.
    stdin = sys.stdin.buffer
121
    stdout = _make_write_all(sys.stdout.buffer)
122
123
124
    if isatty(stdout):
        # The standard library doesn't offer line-buffered binary streams.
        stdout = make_line_buffered(stdout)
125
    stderr = _make_write_all(sys.stderr.buffer)
126
else:
127
    # Python 2 uses the I/O streams provided by the C library.
128
129
    stdin = sys.stdin
    stdout = sys.stdout
130
131
132
133
134
135
136
137
138
139
140
    if isatty(stdout):
        if pycompat.iswindows:
            # Work around size limit when writing to console.
            stdout = platform.winstdout(stdout)
            # The Windows C runtime library doesn't support line buffering.
            stdout = make_line_buffered(stdout)
        else:
            # glibc determines buffering on first write to stdout - if we
            # replace a TTY destined stdout with a pipe destined stdout (e.g.
            # pager), we want line buffering.
            stdout = os.fdopen(stdout.fileno(), 'wb', 1)
Manuel Jacob's avatar
Manuel Jacob committed
141
    stderr = sys.stderr
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168


findexe = platform.findexe
_gethgcmd = platform.gethgcmd
getuser = platform.getuser
getpid = os.getpid
hidewindow = platform.hidewindow
readpipe = platform.readpipe
setbinary = platform.setbinary
setsignalhandler = platform.setsignalhandler
shellquote = platform.shellquote
shellsplit = platform.shellsplit
spawndetached = platform.spawndetached
sshargs = platform.sshargs
testpid = platform.testpid

try:
    setprocname = osutil.setprocname
except AttributeError:
    pass
try:
    unblocksignal = osutil.unblocksignal
except AttributeError:
    pass

closefds = pycompat.isposix

169

170
def explainexit(code):
171
    """return a message describing a subprocess status
172
173
    (codes from kill are negative - not os.system/wait encoding)"""
    if code >= 0:
174
175
        return _(b"exited with status %d") % code
    return _(b"killed by signal %d") % -code
176

177

178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
class _pfile(object):
    """File-like wrapper for a stream opened by subprocess.Popen()"""

    def __init__(self, proc, fp):
        self._proc = proc
        self._fp = fp

    def close(self):
        # unlike os.popen(), this returns an integer in subprocess coding
        self._fp.close()
        return self._proc.wait()

    def __iter__(self):
        return iter(self._fp)

    def __getattr__(self, attr):
        return getattr(self._fp, attr)

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, exc_tb):
        self.close()

202

203
204
def popen(cmd, mode=b'rb', bufsize=-1):
    if mode == b'rb':
205
        return _popenreader(cmd, bufsize)
206
    elif mode == b'wb':
207
        return _popenwriter(cmd, bufsize)
208
    raise error.ProgrammingError(b'unsupported mode: %r' % mode)
209

210

211
def _popenreader(cmd, bufsize):
212
    p = subprocess.Popen(
213
        tonativestr(cmd),
214
215
216
217
218
        shell=True,
        bufsize=bufsize,
        close_fds=closefds,
        stdout=subprocess.PIPE,
    )
219
220
    return _pfile(p, p.stdout)

221

222
def _popenwriter(cmd, bufsize):
223
    p = subprocess.Popen(
224
        tonativestr(cmd),
225
226
227
228
229
        shell=True,
        bufsize=bufsize,
        close_fds=closefds,
        stdin=subprocess.PIPE,
    )
230
231
    return _pfile(p, p.stdin)

232

233
def popen2(cmd, env=None):
234
235
236
    # Setting bufsize to -1 lets the system decide the buffer size.
    # The default for bufsize is 0, meaning unbuffered. This leads to
    # poor performance on Mac OS X: http://bugs.python.org/issue4194
237
238
239
240
241
242
243
244
245
    p = subprocess.Popen(
        tonativestr(cmd),
        shell=True,
        bufsize=-1,
        close_fds=closefds,
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        env=tonativeenv(env),
    )
246
247
    return p.stdin, p.stdout

248

249
250
def popen3(cmd, env=None):
    stdin, stdout, stderr, p = popen4(cmd, env)
251
252
    return stdin, stdout, stderr

253

254
def popen4(cmd, env=None, bufsize=-1):
255
256
257
258
259
260
261
262
263
264
    p = subprocess.Popen(
        tonativestr(cmd),
        shell=True,
        bufsize=bufsize,
        close_fds=closefds,
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        env=tonativeenv(env),
    )
265
266
    return p.stdin, p.stdout, p.stderr, p

267

268
269
def pipefilter(s, cmd):
    '''filter string S through command CMD, returning its output'''
270
271
272
273
274
275
276
    p = subprocess.Popen(
        tonativestr(cmd),
        shell=True,
        close_fds=closefds,
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
    )
277
278
279
    pout, perr = p.communicate(s)
    return pout

280

281
282
283
284
285
286
287
def tempfilter(s, cmd):
    '''filter string S through a pair of temporary files with CMD.
    CMD is used as a template to create the real command to be run,
    with the strings INFILE and OUTFILE replaced by the real names of
    the temporary files generated.'''
    inname, outname = None, None
    try:
288
        infd, inname = pycompat.mkstemp(prefix=b'hg-filter-in-')
289
        fp = os.fdopen(infd, 'wb')
290
291
        fp.write(s)
        fp.close()
292
        outfd, outname = pycompat.mkstemp(prefix=b'hg-filter-out-')
293
        os.close(outfd)
294
295
        cmd = cmd.replace(b'INFILE', inname)
        cmd = cmd.replace(b'OUTFILE', outname)
296
        code = system(cmd)
297
        if pycompat.sysplatform == b'OpenVMS' and code & 1:
298
299
            code = 0
        if code:
300
            raise error.Abort(
301
                _(b"command '%s' failed: %s") % (cmd, explainexit(code))
302
            )
303
        with open(outname, b'rb') as fp:
304
305
306
307
308
309
310
311
312
313
314
315
316
            return fp.read()
    finally:
        try:
            if inname:
                os.unlink(inname)
        except OSError:
            pass
        try:
            if outname:
                os.unlink(outname)
        except OSError:
            pass

317

318
_filtertable = {
319
320
    b'tempfile:': tempfilter,
    b'pipe:': pipefilter,
321
322
}

323

324
def filter(s, cmd):
325
326
    """filter a string through a command that transforms its input to its
    output"""
327
    for name, fn in pycompat.iteritems(_filtertable):
328
        if cmd.startswith(name):
329
            return fn(s, cmd[len(name) :].lstrip())
330
331
    return pipefilter(s, cmd)

332

333
334
_hgexecutable = None

335

336
337
338
339
340
341
def hgexecutable():
    """return location of the 'hg' executable.

    Defaults to $HG or 'hg' in the search path.
    """
    if _hgexecutable is None:
342
        hg = encoding.environ.get(b'HG')
343
        mainmod = sys.modules['__main__']
344
345
        if hg:
            _sethgexecutable(hg)
346
        elif resourceutil.mainfrozen():
347
            if getattr(sys, 'frozen', None) == 'macosx_app':
348
                # Env variable set by py2app
349
                _sethgexecutable(encoding.environ[b'EXECUTABLEPATH'])
350
351
            else:
                _sethgexecutable(pycompat.sysexecutable)
352
353
        elif (
            not pycompat.iswindows
354
            and os.path.basename(getattr(mainmod, '__file__', '')) == 'hg'
355
        ):
356
357
            _sethgexecutable(pycompat.fsencode(mainmod.__file__))
        else:
358
            _sethgexecutable(
359
                findexe(b'hg') or os.path.basename(pycompat.sysargv[0])
360
            )
361
362
    return _hgexecutable

363

364
365
366
367
368
def _sethgexecutable(path):
    """set location of the 'hg' executable"""
    global _hgexecutable
    _hgexecutable = path

369

370
371
372
373
374
def _testfileno(f, stdf):
    fileno = getattr(f, 'fileno', None)
    try:
        return fileno and fileno() == stdf.fileno()
    except io.UnsupportedOperation:
375
376
        return False  # fileno() raised UnsupportedOperation

377
378
379
380

def isstdin(f):
    return _testfileno(f, sys.__stdin__)

381

382
383
384
def isstdout(f):
    return _testfileno(f, sys.__stdout__)

385

386
def protectstdio(uin, uout):
387
388
389
390
    """Duplicate streams and redirect original if (uin, uout) are stdio

    If uin is stdin, it's redirected to /dev/null. If uout is stdout, it's
    redirected to stderr so the output is still readable.
391
392
393
394
395
396

    Returns (fin, fout) which point to the original (uin, uout) fds, but
    may be copy of (uin, uout). The returned streams can be considered
    "owned" in that print(), exec(), etc. never reach to them.
    """
    uout.flush()
397
    fin, fout = uin, uout
398
    if _testfileno(uin, stdin):
399
        newfd = os.dup(uin.fileno())
400
        nullfd = os.open(os.devnull, os.O_RDONLY)
401
        os.dup2(nullfd, uin.fileno())
402
        os.close(nullfd)
403
        fin = os.fdopen(newfd, 'rb')
404
    if _testfileno(uout, stdout):
405
        newfd = os.dup(uout.fileno())
406
        os.dup2(stderr.fileno(), uout.fileno())
407
        fout = os.fdopen(newfd, 'wb')
408
    return fin, fout
409

410

411
412
413
414
415
416
417
418
def restorestdio(uin, uout, fin, fout):
    """Restore (uin, uout) streams from possibly duplicated (fin, fout)"""
    uout.flush()
    for f, uif in [(fin, uin), (fout, uout)]:
        if f is not uif:
            os.dup2(f.fileno(), uif.fileno())
            f.close()

419

420
421
def shellenviron(environ=None):
    """return environ with optional override, useful for shelling out"""
422

423
    def py2shell(val):
424
        """convert python object into string that is useful to shell"""
425
        if val is None or val is False:
426
            return b'0'
427
        if val is True:
428
            return b'1'
429
        return pycompat.bytestr(val)
430

431
432
    env = dict(encoding.environ)
    if environ:
433
        env.update((k, py2shell(v)) for k, v in pycompat.iteritems(environ))
434
    env[b'HG'] = hgexecutable()
435
436
    return env

437

438
if pycompat.iswindows:
439

440
    def shelltonative(cmd, env):
441
442
443
        return platform.shelltocmdexe(  # pytype: disable=module-attr
            cmd, shellenviron(env)
        )
444
445

    tonativestr = encoding.strfromlocal
446
else:
447

448
449
450
    def shelltonative(cmd, env):
        return cmd

451
452
    tonativestr = pycompat.identity

453

454
455
456
457
458
def tonativeenv(env):
    '''convert the environment from bytes to strings suitable for Popen(), etc.
    '''
    return pycompat.rapply(tonativestr, env)

459

460
461
462
463
464
465
466
467
468
469
470
471
def system(cmd, environ=None, cwd=None, out=None):
    '''enhanced shell command execution.
    run with environment maybe modified, maybe in different dir.

    if out is specified, it is assumed to be a file-like object that has a
    write() method. stdout and stderr will be redirected to out.'''
    try:
        stdout.flush()
    except Exception:
        pass
    env = shellenviron(environ)
    if out is None or isstdout(out):
472
473
474
475
476
477
478
        rc = subprocess.call(
            tonativestr(cmd),
            shell=True,
            close_fds=closefds,
            env=tonativeenv(env),
            cwd=pycompat.rapply(tonativestr, cwd),
        )
479
    else:
480
481
482
483
484
485
486
487
488
        proc = subprocess.Popen(
            tonativestr(cmd),
            shell=True,
            close_fds=closefds,
            env=tonativeenv(env),
            cwd=pycompat.rapply(tonativestr, cwd),
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT,
        )
489
        for line in iter(proc.stdout.readline, b''):
490
491
492
            out.write(line)
        proc.wait()
        rc = proc.returncode
493
    if pycompat.sysplatform == b'OpenVMS' and rc & 1:
494
495
496
        rc = 0
    return rc

497

498
499
500
501
_is_gui = None


def _gui():
502
503
    '''Are we running in a GUI?'''
    if pycompat.isdarwin:
504
        if b'SSH_CONNECTION' in encoding.environ:
505
506
507
508
509
510
511
512
513
            # handle SSH access to a box where the user is logged in
            return False
        elif getattr(osutil, 'isgui', None):
            # check if a CoreGraphics session is available
            return osutil.isgui()
        else:
            # pure build; use a safe default
            return True
    else:
514
        return pycompat.iswindows or encoding.environ.get(b"DISPLAY")
515

516

517
518
519
520
521
522
523
def gui():
    global _is_gui
    if _is_gui is None:
        _is_gui = _gui()
    return _is_gui


524
525
526
527
528
529
530
def hgcmd():
    """Return the command used to execute current hg

    This is different from hgexecutable() because on Windows we want
    to avoid things opening new shell windows like batch files, so we
    get either the python call or current executable.
    """
531
    if resourceutil.mainfrozen():
532
        if getattr(sys, 'frozen', None) == 'macosx_app':
533
            # Env variable set by py2app
534
            return [encoding.environ[b'EXECUTABLEPATH']]
535
536
537
538
        else:
            return [pycompat.sysexecutable]
    return _gethgcmd()

539

540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
def rundetached(args, condfn):
    """Execute the argument list in a detached process.

    condfn is a callable which is called repeatedly and should return
    True once the child process is known to have started successfully.
    At this point, the child process PID is returned. If the child
    process fails to start or finishes before condfn() evaluates to
    True, return -1.
    """
    # Windows case is easier because the child process is either
    # successfully starting and validating the condition or exiting
    # on failure. We just poll on its PID. On Unix, if the child
    # process fails to start, it will be left in a zombie state until
    # the parent wait on it, which we cannot do since we expect a long
    # running process on success. Instead we listen for SIGCHLD telling
    # us our child process terminated.
    terminated = set()
557

558
559
    def handler(signum, frame):
        terminated.add(os.wait())
560

561
562
563
564
565
566
567
    prevhandler = None
    SIGCHLD = getattr(signal, 'SIGCHLD', None)
    if SIGCHLD is not None:
        prevhandler = signal.signal(SIGCHLD, handler)
    try:
        pid = spawndetached(args)
        while not condfn():
568
            if (pid in terminated or not testpid(pid)) and not condfn():
569
570
571
572
573
574
                return -1
            time.sleep(0.1)
        return pid
    finally:
        if prevhandler is not None:
            signal.signal(signal.SIGCHLD, prevhandler)
575

576

577
@contextlib.contextmanager
578
def uninterruptible(warn):
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
    """Inhibit SIGINT handling on a region of code.

    Note that if this is called in a non-main thread, it turns into a no-op.

    Args:
      warn: A callable which takes no arguments, and returns True if the
            previous signal handling should be restored.
    """

    oldsiginthandler = [signal.getsignal(signal.SIGINT)]
    shouldbail = []

    def disabledsiginthandler(*args):
        if warn():
            signal.signal(signal.SIGINT, oldsiginthandler[0])
            del oldsiginthandler[0]
        shouldbail.append(True)

    try:
        try:
            signal.signal(signal.SIGINT, disabledsiginthandler)
        except ValueError:
            # wrong thread, oh well, we tried
            del oldsiginthandler[0]
        yield
    finally:
        if oldsiginthandler:
            signal.signal(signal.SIGINT, oldsiginthandler[0])
        if shouldbail:
            raise KeyboardInterrupt
609

610

611
612
613
614
615
if pycompat.iswindows:
    # no fork on Windows, but we can create a detached process
    # https://msdn.microsoft.com/en-us/library/windows/desktop/ms684863.aspx
    # No stdlib constant exists for this value
    DETACHED_PROCESS = 0x00000008
616
617
618
    # Following creation flags might create a console GUI window.
    # Using subprocess.CREATE_NEW_CONSOLE might helps.
    # See https://phab.mercurial-scm.org/D1701 for discussion
619
620
621
622
    _creationflags = (
        DETACHED_PROCESS
        | subprocess.CREATE_NEW_PROCESS_GROUP  # pytype: disable=module-attr
    )
623

624
    def runbgcommand(
625
626
627
628
629
630
631
        script,
        env,
        shell=False,
        stdout=None,
        stderr=None,
        ensurestart=True,
        record_wait=None,
632
    ):
633
634
635
        '''Spawn a command without waiting for it to finish.'''
        # we can't use close_fds *and* redirect stdin. I'm not sure that we
        # need to because the detached process has no console connection.
636
        p = subprocess.Popen(
637
            tonativestr(script),
638
639
640
641
642
643
644
            shell=shell,
            env=tonativeenv(env),
            close_fds=True,
            creationflags=_creationflags,
            stdout=stdout,
            stderr=stderr,
        )
645
646
        if record_wait is not None:
            record_wait(p.wait)
647
648


649
else:
650

651
    def runbgcommand(
652
653
654
655
656
657
658
        cmd,
        env,
        shell=False,
        stdout=None,
        stderr=None,
        ensurestart=True,
        record_wait=None,
659
    ):
660
661
662
663
664
665
666
667
        '''Spawn a command without waiting for it to finish.


        When `record_wait` is not None, the spawned process will not be fully
        detached and the `record_wait` argument will be called with a the
        `Subprocess.wait` function for the spawned process.  This is mostly
        useful for developers that need to make sure the spawned process
        finished before a certain point. (eg: writing test)'''
668
669
670
671
672
        if pycompat.isdarwin:
            # avoid crash in CoreFoundation in case another thread
            # calls gui() while we're calling fork().
            gui()

673
674
        # double-fork to completely detach from the parent process
        # based on http://code.activestate.com/recipes/278731
675
676
677
678
        if record_wait is None:
            pid = os.fork()
            if pid:
                if not ensurestart:
679
680
681
682
683
684
685
                    # Even though we're not waiting on the child process,
                    # we still must call waitpid() on it at some point so
                    # it's not a zombie/defunct. This is especially relevant for
                    # chg since the parent process won't die anytime soon.
                    # We use a thread to make the overhead tiny.
                    def _do_wait():
                        os.waitpid(pid, 0)
686

687
688
689
                    t = threading.Thread(target=_do_wait)
                    t.daemon = True
                    t.start()
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
                    return
                # Parent process
                (_pid, status) = os.waitpid(pid, 0)
                if os.WIFEXITED(status):
                    returncode = os.WEXITSTATUS(status)
                else:
                    returncode = -(os.WTERMSIG(status))
                if returncode != 0:
                    # The child process's return code is 0 on success, an errno
                    # value on failure, or 255 if we don't have a valid errno
                    # value.
                    #
                    # (It would be slightly nicer to return the full exception info
                    # over a pipe as the subprocess module does.  For now it
                    # doesn't seem worth adding that complexity here, though.)
                    if returncode == 255:
                        returncode = errno.EINVAL
                    raise OSError(
                        returncode,
                        b'error running %r: %s'
                        % (cmd, os.strerror(returncode)),
                    )
712
                return
713
714
715

        returncode = 255
        try:
716
717
718
            if record_wait is None:
                # Start a new session
                os.setsid()
719

720
            stdin = open(os.devnull, b'r')
721
            if stdout is None:
722
                stdout = open(os.devnull, b'w')
723
            if stderr is None:
724
                stderr = open(os.devnull, b'w')
725
726
727

            # connect stdin to devnull to make sure the subprocess can't
            # muck up that stream for mercurial.
728
            p = subprocess.Popen(
729
730
731
732
733
734
735
736
                cmd,
                shell=shell,
                env=env,
                close_fds=True,
                stdin=stdin,
                stdout=stdout,
                stderr=stderr,
            )
737
738
            if record_wait is not None:
                record_wait(p.wait)
739
740
            returncode = 0
        except EnvironmentError as ex:
741
            returncode = ex.errno & 0xFF
742
743
744
745
746
747
748
749
750
            if returncode == 0:
                # This shouldn't happen, but just in case make sure the
                # return code is never 0 here.
                returncode = 255
        except Exception:
            returncode = 255
        finally:
            # mission accomplished, this child needs to exit and not
            # continue the hg process here.
751
752
            if record_wait is None:
                os._exit(returncode)