dirstatemap.py 23.8 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
# dirstatemap.py
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.

from __future__ import absolute_import

import errno

from .i18n import _

from . import (
    error,
    pathutil,
    policy,
    pycompat,
    txnutil,
    util,
)

21
22
from .dirstateutils import (
    docket as docketmod,
23
    v2,
24
25
)

26
27
28
29
30
parsers = policy.importmod('parsers')
rustmod = policy.importrust('dirstate')

propertycache = util.propertycache

31
32
33
34
if rustmod is None:
    DirstateItem = parsers.DirstateItem
else:
    DirstateItem = rustmod.DirstateItem
35

36
37
rangemask = 0x7FFFFFFF

38

39
40
41
42
43
44
class _dirstatemapcommon(object):
    """
    Methods that are identical for both implementations of the dirstatemap
    class, with and without Rust extensions enabled.
    """

45
46
47
48
49
    # please pytype

    _map = None
    copymap = None

50
51
52
53
54
55
56
57
58
59
    def __init__(self, ui, opener, root, nodeconstants, use_dirstate_v2):
        self._use_dirstate_v2 = use_dirstate_v2
        self._nodeconstants = nodeconstants
        self._ui = ui
        self._opener = opener
        self._root = root
        self._filename = b'dirstate'
        self._nodelen = 20  # Also update Rust code when changing this!
        self._parents = None
        self._dirtyparents = False
60
        self._docket = None
61
62
63
64

        # for consistent view between _pl() and _read() invocations
        self._pendingmode = None

65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
    def preload(self):
        """Loads the underlying data, if it's not already loaded"""
        self._map

    def get(self, key, default=None):
        return self._map.get(key, default)

    def __len__(self):
        return len(self._map)

    def __iter__(self):
        return iter(self._map)

    def __contains__(self, key):
        return key in self._map

    def __getitem__(self, item):
        return self._map[item]

84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
    ### sub-class utility method
    #
    # Use to allow for generic implementation of some method while still coping
    # with minor difference between implementation.

    def _dirs_incr(self, filename, old_entry=None):
        """incremente the dirstate counter if applicable

        This might be a no-op for some subclass who deal with directory
        tracking in a different way.
        """

    def _dirs_decr(self, filename, old_entry=None, remove_variant=False):
        """decremente the dirstate counter if applicable

        This might be a no-op for some subclass who deal with directory
        tracking in a different way.
        """

103
104
105
    def _refresh_entry(self, f, entry):
        """record updated state of an entry"""

106
107
108
109
110
111
    def _insert_entry(self, f, entry):
        """add a new dirstate entry (or replace an unrelated one)

        The fact it is actually new is the responsability of the caller
        """

112
113
114
115
116
117
118
    def _drop_entry(self, f):
        """remove any entry for file f

        This should also drop associated copy information

        The fact we actually need to drop it is the responsability of the caller"""

119
120
    ### method to manipulate the entries

121
122
123
124
125
126
    def set_possibly_dirty(self, filename):
        """record that the current state of the file on disk is unknown"""
        entry = self[filename]
        entry.set_possibly_dirty()
        self._refresh_entry(filename, entry)

127
128
129
130
131
132
133
134
135
    def set_clean(self, filename, mode, size, mtime):
        """mark a file as back to a clean state"""
        entry = self[filename]
        mtime = mtime & rangemask
        size = size & rangemask
        entry.set_clean(mode, size, mtime)
        self._refresh_entry(filename, entry)
        self.copymap.pop(filename, None)

136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
    def set_tracked(self, filename):
        new = False
        entry = self.get(filename)
        if entry is None:
            self._dirs_incr(filename)
            entry = DirstateItem(
                wc_tracked=True,
            )

            self._insert_entry(filename, entry)
            new = True
        elif not entry.tracked:
            self._dirs_incr(filename, entry)
            entry.set_tracked()
            self._refresh_entry(filename, entry)
            new = True
        else:
            # XXX This is probably overkill for more case, but we need this to
            # fully replace the `normallookup` call with `set_tracked` one.
            # Consider smoothing this in the future.
            entry.set_possibly_dirty()
            self._refresh_entry(filename, entry)
        return new

160
161
162
163
164
165
166
    def set_untracked(self, f):
        """Mark a file as no longer tracked in the dirstate map"""
        entry = self.get(f)
        if entry is None:
            return False
        else:
            self._dirs_decr(f, old_entry=entry, remove_variant=not entry.added)
167
            if not entry.p2_info:
168
169
170
171
172
                self.copymap.pop(f, None)
            entry.set_untracked()
            self._refresh_entry(f, entry)
            return True

173
174
175
176
177
    def reset_state(
        self,
        filename,
        wc_tracked=False,
        p1_tracked=False,
178
179
180
        p2_info=False,
        has_meaningful_mtime=True,
        has_meaningful_data=True,
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
        parentfiledata=None,
    ):
        """Set a entry to a given state, diregarding all previous state

        This is to be used by the part of the dirstate API dedicated to
        adjusting the dirstate after a update/merge.

        note: calling this might result to no entry existing at all if the
        dirstate map does not see any point at having one for this file
        anymore.
        """
        # copy information are now outdated
        # (maybe new information should be in directly passed to this function)
        self.copymap.pop(filename, None)

196
        if not (p1_tracked or p2_info or wc_tracked):
197
198
199
200
            old_entry = self._map.get(filename)
            self._drop_entry(filename)
            self._dirs_decr(filename, old_entry=old_entry)
            return
201

202
203
204
205
206
        old_entry = self._map.get(filename)
        self._dirs_incr(filename, old_entry)
        entry = DirstateItem(
            wc_tracked=wc_tracked,
            p1_tracked=p1_tracked,
207
208
            p2_info=p2_info,
            has_meaningful_mtime=has_meaningful_mtime,
209
210
211
212
            parentfiledata=parentfiledata,
        )
        self._insert_entry(filename, entry)

213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
    ### disk interaction

    def _opendirstatefile(self):
        fp, mode = txnutil.trypending(self._root, self._opener, self._filename)
        if self._pendingmode is not None and self._pendingmode != mode:
            fp.close()
            raise error.Abort(
                _(b'working directory state may be changed parallelly')
            )
        self._pendingmode = mode
        return fp

    def _readdirstatefile(self, size=-1):
        try:
            with self._opendirstatefile() as fp:
                return fp.read(size)
        except IOError as err:
            if err.errno != errno.ENOENT:
                raise
            # File doesn't exist, so the current state is empty
            return b''

    @property
    def docket(self):
        if not self._docket:
            if not self._use_dirstate_v2:
                raise error.ProgrammingError(
                    b'dirstate only has a docket in v2 format'
                )
            self._docket = docketmod.DirstateDocket.parse(
                self._readdirstatefile(), self._nodeconstants
            )
        return self._docket

    def write_v2_no_append(self, tr, st, meta, packed):
        old_docket = self.docket
        new_docket = docketmod.DirstateDocket.with_new_uuid(
            self.parents(), len(packed), meta
        )
        data_filename = new_docket.data_filename()
        if tr:
            tr.add(data_filename, 0)
        self._opener.write(data_filename, packed)
        # Write the new docket after the new data file has been
        # written. Because `st` was opened with `atomictemp=True`,
        # the actual `.hg/dirstate` file is only affected on close.
        st.write(new_docket.serialize())
        st.close()
        # Remove the old data file after the new docket pointing to
        # the new data file was written.
        if old_docket.uuid:
            data_filename = old_docket.data_filename()
            unlink = lambda _tr=None: self._opener.unlink(data_filename)
            if tr:
                category = b"dirstate-v2-clean-" + old_docket.uuid
                tr.addpostclose(category, unlink)
            else:
                unlink()
        self._docket = new_docket

    ### reading/setting parents

    def parents(self):
        if not self._parents:
            if self._use_dirstate_v2:
                self._parents = self.docket.parents
            else:
                read_len = self._nodelen * 2
                st = self._readdirstatefile(read_len)
                l = len(st)
                if l == read_len:
                    self._parents = (
                        st[: self._nodelen],
                        st[self._nodelen : 2 * self._nodelen],
                    )
                elif l == 0:
                    self._parents = (
                        self._nodeconstants.nullid,
                        self._nodeconstants.nullid,
                    )
                else:
                    raise error.Abort(
                        _(b'working directory state appears damaged!')
                    )

        return self._parents

300
301

class dirstatemap(_dirstatemapcommon):
302
303
304
305
306
307
308
309
310
311
312
313
314
    """Map encapsulating the dirstate's contents.

    The dirstate contains the following state:

    - `identity` is the identity of the dirstate file, which can be used to
      detect when changes have occurred to the dirstate file.

    - `parents` is a pair containing the parents of the working copy. The
      parents are updated by calling `setparents`.

    - the state map maps filenames to tuples of (state, mode, size, mtime),
      where state is a single character representing 'normal', 'added',
      'removed', or 'merged'. It is read by treating the dirstate as a
315
316
317
318
319
320
321
322
      dict.  File state is updated by calling various methods (see each
      documentation for details):

      - `reset_state`,
      - `set_tracked`
      - `set_untracked`
      - `set_clean`
      - `set_possibly_dirty`
323
324
325
326
327
328
329
330
331
332
333
334

    - `copymap` maps destination filenames to their source filename.

    The dirstate also provides the following views onto the state:

    - `filefoldmap` is a dict mapping normalized filenames to the denormalized
      form that they appear as in the dirstate.

    - `dirfoldmap` is a dict mapping normalized directory names to the
      denormalized form that they appear as in the dirstate.
    """

335
336
    ### Core data storage and access

337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
    @propertycache
    def _map(self):
        self._map = {}
        self.read()
        return self._map

    @propertycache
    def copymap(self):
        self.copymap = {}
        self._map
        return self.copymap

    def clear(self):
        self._map.clear()
        self.copymap.clear()
        self.setparents(self._nodeconstants.nullid, self._nodeconstants.nullid)
        util.clearcachedproperty(self, b"_dirs")
        util.clearcachedproperty(self, b"_alldirs")
        util.clearcachedproperty(self, b"filefoldmap")
        util.clearcachedproperty(self, b"dirfoldmap")

    def items(self):
        return pycompat.iteritems(self._map)

    # forward for python2,3 compat
    iteritems = items

364
365
    def debug_iter(self, all):
        """
366
367
        Return an iterator of (filename, state, mode, size, mtime) tuples

368
369
        `all` is unused when Rust is not enabled
        """
370
371
        for (filename, item) in self.items():
            yield (filename, item.state, item.mode, item.size, item.mtime)
372

373
374
375
    def keys(self):
        return self._map.keys()

376
377
378
379
380
381
382
383
384
    ### reading/setting parents

    def setparents(self, p1, p2, fold_p2=False):
        self._parents = (p1, p2)
        self._dirtyparents = True
        copies = {}
        if fold_p2:
            for f, s in pycompat.iteritems(self._map):
                # Discard "merged" markers when moving away from a merge state
385
                if s.p2_info:
386
387
388
389
390
391
392
393
394
395
396
397
398
399
                    source = self.copymap.pop(f, None)
                    if source:
                        copies[f] = source
                    s.drop_merge_data()
        return copies

    ### disk interaction

    def read(self):
        # ignore HG_PENDING because identity is used only for writing
        self.identity = util.filestat.frompath(
            self._opener.join(self._filename)
        )

400
401
402
403
404
405
406
        if self._use_dirstate_v2:
            if not self.docket.uuid:
                return
            st = self._opener.read(self.docket.data_filename())
        else:
            st = self._readdirstatefile()

407
408
409
        if not st:
            return

410
        # TODO: adjust this estimate for dirstate-v2
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
        if util.safehasattr(parsers, b'dict_new_presized'):
            # Make an estimate of the number of files in the dirstate based on
            # its size. This trades wasting some memory for avoiding costly
            # resizes. Each entry have a prefix of 17 bytes followed by one or
            # two path names. Studies on various large-scale real-world repositories
            # found 54 bytes a reasonable upper limit for the average path names.
            # Copy entries are ignored for the sake of this estimate.
            self._map = parsers.dict_new_presized(len(st) // 71)

        # Python's garbage collector triggers a GC each time a certain number
        # of container objects (the number being defined by
        # gc.get_threshold()) are allocated. parse_dirstate creates a tuple
        # for each file in the dirstate. The C version then immediately marks
        # them as not to be tracked by the collector. However, this has no
        # effect on when GCs are triggered, only on what objects the GC looks
        # into. This means that O(number of files) GCs are unavoidable.
        # Depending on when in the process's lifetime the dirstate is parsed,
        # this can get very expensive. As a workaround, disable GC while
        # parsing the dirstate.
        #
        # (we cannot decorate the function directly since it is in a C module)
432
433
434
435
436
437
438
439
        if self._use_dirstate_v2:
            p = self.docket.parents
            meta = self.docket.tree_metadata
            parse_dirstate = util.nogc(v2.parse_dirstate)
            parse_dirstate(self._map, self.copymap, st, meta)
        else:
            parse_dirstate = util.nogc(parsers.parse_dirstate)
            p = parse_dirstate(self._map, self.copymap, st)
440
441
442
443
444
445
446
447
        if not self._dirtyparents:
            self.setparents(*p)

        # Avoid excess attribute lookups by fast pathing certain checks
        self.__contains__ = self._map.__contains__
        self.__getitem__ = self._map.__getitem__
        self.get = self._map.get

448
449
450
451
452
453
454
455
456
457
    def write(self, tr, st, now):
        if self._use_dirstate_v2:
            packed, meta = v2.pack_dirstate(self._map, self.copymap, now)
            self.write_v2_no_append(tr, st, meta, packed)
        else:
            packed = parsers.pack_dirstate(
                self._map, self.copymap, self.parents(), now
            )
            st.write(packed)
            st.close()
458
459
460
461
462
463
464
465
466
467
        self._dirtyparents = False

    @propertycache
    def identity(self):
        self._map
        return self.identity

    ### code related to maintaining and accessing "extra" property
    # (e.g. "has_dir")

468
469
470
471
472
473
474
475
476
    def _dirs_incr(self, filename, old_entry=None):
        """incremente the dirstate counter if applicable"""
        if (
            old_entry is None or old_entry.removed
        ) and "_dirs" in self.__dict__:
            self._dirs.addpath(filename)
        if old_entry is None and "_alldirs" in self.__dict__:
            self._alldirs.addpath(filename)

477
    def _dirs_decr(self, filename, old_entry=None, remove_variant=False):
478
479
480
481
        """decremente the dirstate counter if applicable"""
        if old_entry is not None:
            if "_dirs" in self.__dict__ and not old_entry.removed:
                self._dirs.delpath(filename)
482
            if "_alldirs" in self.__dict__ and not remove_variant:
483
                self._alldirs.delpath(filename)
484
485
        elif remove_variant and "_alldirs" in self.__dict__:
            self._alldirs.addpath(filename)
486
487
488
489
        if "filefoldmap" in self.__dict__:
            normed = util.normcase(filename)
            self.filefoldmap.pop(normed, None)

490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
    @propertycache
    def filefoldmap(self):
        """Returns a dictionary mapping normalized case paths to their
        non-normalized versions.
        """
        try:
            makefilefoldmap = parsers.make_file_foldmap
        except AttributeError:
            pass
        else:
            return makefilefoldmap(
                self._map, util.normcasespec, util.normcasefallback
            )

        f = {}
        normcase = util.normcase
        for name, s in pycompat.iteritems(self._map):
            if not s.removed:
                f[normcase(name)] = name
        f[b'.'] = b'.'  # prevents useless util.fspath() invocation
        return f

    @propertycache
    def dirfoldmap(self):
        f = {}
        normcase = util.normcase
        for name in self._dirs:
            f[normcase(name)] = name
        return f

    def hastrackeddir(self, d):
        """
        Returns True if the dirstate contains a tracked (not removed) file
        in this directory.
        """
        return d in self._dirs

    def hasdir(self, d):
        """
        Returns True if the dirstate contains a file (tracked or removed)
        in this directory.
        """
        return d in self._alldirs

    @propertycache
    def _dirs(self):
        return pathutil.dirs(self._map, only_tracked=True)

    @propertycache
    def _alldirs(self):
        return pathutil.dirs(self._map)

    ### code related to manipulation of entries and copy-sources

544
545
546
547
    def _refresh_entry(self, f, entry):
        if not entry.any_tracked:
            self._map.pop(f, None)

548
549
550
    def _insert_entry(self, f, entry):
        self._map[f] = entry

551
552
    def _drop_entry(self, f):
        self._map.pop(f, None)
553
        self.copymap.pop(f, None)
554

555
556
557

if rustmod is not None:

558
    class dirstatemap(_dirstatemapcommon):
559

560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
        ### Core data storage and access

        @propertycache
        def _map(self):
            """
            Fills the Dirstatemap when called.
            """
            # ignore HG_PENDING because identity is used only for writing
            self.identity = util.filestat.frompath(
                self._opener.join(self._filename)
            )

            if self._use_dirstate_v2:
                if self.docket.uuid:
                    # TODO: use mmap when possible
                    data = self._opener.read(self.docket.data_filename())
                else:
                    data = b''
                self._map = rustmod.DirstateMap.new_v2(
                    data, self.docket.data_size, self.docket.tree_metadata
                )
                parents = self.docket.parents
582
            else:
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
                self._map, parents = rustmod.DirstateMap.new_v1(
                    self._readdirstatefile()
                )

            if parents and not self._dirtyparents:
                self.setparents(*parents)

            self.__contains__ = self._map.__contains__
            self.__getitem__ = self._map.__getitem__
            self.get = self._map.get
            return self._map

        @property
        def copymap(self):
            return self._map.copymap()

        def debug_iter(self, all):
            """
            Return an iterator of (filename, state, mode, size, mtime) tuples

            `all`: also include with `state == b' '` dirstate tree nodes that
            don't have an associated `DirstateItem`.

            """
            return self._map.debug_iter(all)

        def clear(self):
            self._map.clear()
            self.setparents(
                self._nodeconstants.nullid, self._nodeconstants.nullid
            )
            util.clearcachedproperty(self, b"_dirs")
            util.clearcachedproperty(self, b"_alldirs")
            util.clearcachedproperty(self, b"dirfoldmap")

        def items(self):
            return self._map.items()

        # forward for python2,3 compat
        iteritems = items

        def keys(self):
            return iter(self._map)

        ### reading/setting parents

        def setparents(self, p1, p2, fold_p2=False):
            self._parents = (p1, p2)
            self._dirtyparents = True
            copies = {}
            if fold_p2:
                # Collect into an intermediate list to avoid a `RuntimeError`
                # exception due to mutation during iteration.
                # TODO: move this the whole loop to Rust where `iter_mut`
                # enables in-place mutation of elements of a collection while
                # iterating it, without mutating the collection itself.
639
                files_with_p2_info = [
640
                    f for f, s in self._map.items() if s.p2_info
641
                ]
642
643
644
645
646
647
648
649
                rust_map = self._map
                for f in files_with_p2_info:
                    e = rust_map.get(f)
                    source = self.copymap.pop(f, None)
                    if source:
                        copies[f] = source
                    e.drop_merge_data()
                    rust_map.set_dirstate_item(f, e)
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
            return copies

        ### disk interaction

        @propertycache
        def identity(self):
            self._map
            return self.identity

        def write(self, tr, st, now):
            if not self._use_dirstate_v2:
                p1, p2 = self.parents()
                packed = self._map.write_v1(p1, p2, now)
                st.write(packed)
                st.close()
                self._dirtyparents = False
                return

            # We can only append to an existing data file if there is one
            can_append = self.docket.uuid is not None
            packed, meta, append = self._map.write_v2(now, can_append)
            if append:
                docket = self.docket
                data_filename = docket.data_filename()
                if tr:
                    tr.add(data_filename, docket.data_size)
                with self._opener(data_filename, b'r+b') as fp:
                    fp.seek(docket.data_size)
                    assert fp.tell() == docket.data_size
                    written = fp.write(packed)
                    if written is not None:  # py2 may return None
                        assert written == len(packed), (written, len(packed))
                docket.data_size += len(packed)
                docket.parents = self.parents()
                docket.tree_metadata = meta
                st.write(docket.serialize())
                st.close()
            else:
688
                self.write_v2_no_append(tr, st, meta, packed)
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
            # Reload from the newly-written file
            util.clearcachedproperty(self, b"_map")
            self._dirtyparents = False

        ### code related to maintaining and accessing "extra" property
        # (e.g. "has_dir")

        @propertycache
        def filefoldmap(self):
            """Returns a dictionary mapping normalized case paths to their
            non-normalized versions.
            """
            return self._map.filefoldmapasdict()

        def hastrackeddir(self, d):
            return self._map.hastrackeddir(d)

        def hasdir(self, d):
            return self._map.hasdir(d)

        @propertycache
        def dirfoldmap(self):
            f = {}
            normcase = util.normcase
            for name in self._map.tracked_dirs():
                f[normcase(name)] = name
            return f

        ### code related to manipulation of entries and copy-sources

719
720
721
722
723
724
        def _refresh_entry(self, f, entry):
            if not entry.any_tracked:
                self._map.drop_item_and_copy_source(f)
            else:
                self._map.addfile(f, entry)

725
726
727
        def _insert_entry(self, f, entry):
            self._map.addfile(f, entry)

728
729
730
        def _drop_entry(self, f):
            self._map.drop_item_and_copy_source(f)

731
732
733
        def __setitem__(self, key, value):
            assert isinstance(value, DirstateItem)
            self._map.set_dirstate_item(key, value)