diff --git a/.gitlab/merge_request_templates/Default.md b/.gitlab/merge_request_templates/Default.md new file mode 100644 index 0000000000000000000000000000000000000000..13dfad0f9f7a253ff292b3f232ee426fb221a02e_LmdpdGxhYi9tZXJnZV9yZXF1ZXN0X3RlbXBsYXRlcy9EZWZhdWx0Lm1k --- /dev/null +++ b/.gitlab/merge_request_templates/Default.md @@ -0,0 +1,18 @@ +--- +name: Official Review +about: Submit a series for review +--- + +/assign_reviewer @mercurial.review + +Welcome to the Mercurial Merge Request creation process: + +* Set a simple title for your MR, +* All important information should be contained in your changesets' content or description, +* You can add some workflow-relevant information here (eg: when this depends on another MR), +* If your changes are not ready for review yet, click `Start the title with Draft:` under the title. + +More details here: + +* https://www.mercurial-scm.org/wiki/ContributingChanges +* https://www.mercurial-scm.org/wiki/Heptapod diff --git a/contrib/check-py3-compat.py b/contrib/check-py3-compat.py index d3d3495a5749fdd9b9611dda1d0eee75aa79e891_Y29udHJpYi9jaGVjay1weTMtY29tcGF0LnB5..13dfad0f9f7a253ff292b3f232ee426fb221a02e_Y29udHJpYi9jaGVjay1weTMtY29tcGF0LnB5 100755 --- a/contrib/check-py3-compat.py +++ b/contrib/check-py3-compat.py @@ -76,7 +76,7 @@ # specified. When running as test-check-py3-compat.t, we technically # would import the correct paths, but it's cleaner to have both cases # use the same import logic. - sys.path.insert(0, '.') + sys.path.insert(0, os.getcwd()) for f in sys.argv[1:]: with warnings.catch_warnings(record=True) as warns: diff --git a/doc/check-seclevel.py b/doc/check-seclevel.py index d3d3495a5749fdd9b9611dda1d0eee75aa79e891_ZG9jL2NoZWNrLXNlY2xldmVsLnB5..13dfad0f9f7a253ff292b3f232ee426fb221a02e_ZG9jL2NoZWNrLXNlY2xldmVsLnB5 100755 --- a/doc/check-seclevel.py +++ b/doc/check-seclevel.py @@ -9,7 +9,7 @@ # import from the live mercurial repo os.environ['HGMODULEPOLICY'] = 'py' -sys.path.insert(0, "..") +sys.path.insert(0, os.path.abspath("..")) from mercurial import demandimport demandimport.enable() diff --git a/mercurial/worker.py b/mercurial/worker.py index d3d3495a5749fdd9b9611dda1d0eee75aa79e891_bWVyY3VyaWFsL3dvcmtlci5weQ==..13dfad0f9f7a253ff292b3f232ee426fb221a02e_bWVyY3VyaWFsL3dvcmtlci5weQ== 100644 --- a/mercurial/worker.py +++ b/mercurial/worker.py @@ -68,6 +68,52 @@ return threading.current_thread() == threading.main_thread() +class _blockingreader(object): + def __init__(self, wrapped): + self._wrapped = wrapped + + # Do NOT implement readinto() by making it delegate to + # _wrapped.readinto(), since that is unbuffered. The unpickler is fine + # with just read() and readline(), so we don't need to implement it. + + if (3, 8, 0) <= sys.version_info[:3] < (3, 8, 2): + + # This is required for python 3.8, prior to 3.8.2. See issue6444. + def readinto(self, b): + pos = 0 + size = len(b) + + while pos < size: + ret = self._wrapped.readinto(b[pos:]) + if not ret: + break + pos += ret + + return pos + + def readline(self): + return self._wrapped.readline() + + # issue multiple reads until size is fulfilled + def read(self, size=-1): + if size < 0: + return self._wrapped.readall() + + buf = bytearray(size) + view = memoryview(buf) + pos = 0 + + while pos < size: + ret = self._wrapped.readinto(view[pos:]) + if not ret: + break + pos += ret + + del view + del buf[pos:] + return bytes(buf) + + class _blockingreader: def __init__(self, wrapped): self._wrapped = wrapped diff --git a/rust/hg-core/src/repo.rs b/rust/hg-core/src/repo.rs index d3d3495a5749fdd9b9611dda1d0eee75aa79e891_cnVzdC9oZy1jb3JlL3NyYy9yZXBvLnJz..13dfad0f9f7a253ff292b3f232ee426fb221a02e_cnVzdC9oZy1jb3JlL3NyYy9yZXBvLnJz 100644 --- a/rust/hg-core/src/repo.rs +++ b/rust/hg-core/src/repo.rs @@ -456,8 +456,23 @@ let data_filename = format!("dirstate.{}", uuid); let data_filename = self.hg_vfs().join(data_filename); let mut options = std::fs::OpenOptions::new(); - if append { - options.append(true); - } else { - options.write(true).create_new(true); + options.write(true); + + // Why are we not using the O_APPEND flag when appending? + // + // - O_APPEND makes it trickier to deal with garbage at the end of + // the file, left by a previous uncommitted transaction. By + // starting the write at [old_data_size] we make sure we erase + // all such garbage. + // + // - O_APPEND requires to special-case 0-byte writes, whereas we + // don't need that. + // + // - Some OSes have bugs in implementation O_APPEND: + // revlog.py talks about a Solaris bug, but we also saw some ZFS + // bug: https://github.com/openzfs/zfs/pull/3124, + // https://github.com/openzfs/zfs/issues/13370 + // + if !append { + options.create_new(true); } @@ -463,6 +478,7 @@ } + let data_size = (|| { // TODO: loop and try another random ID if !append and this // returns `ErrorKind::AlreadyExists`? Collision chance of two // random IDs is one in 2**32 let mut file = options.open(&data_filename)?; @@ -464,18 +480,8 @@ let data_size = (|| { // TODO: loop and try another random ID if !append and this // returns `ErrorKind::AlreadyExists`? Collision chance of two // random IDs is one in 2**32 let mut file = options.open(&data_filename)?; - if data.is_empty() { - // If we're not appending anything, the data size is the - // same as in the previous docket. It is *not* the file - // length, since it could have garbage at the end. - // We don't have to worry about it when we do have data - // to append since we rewrite the root node in this case. - Ok(old_data_size as u64) - } else { - file.write_all(&data)?; - file.flush()?; - // TODO: use https://doc.rust-lang.org/std/io/trait.Seek.html#method.stream_position when we require Rust 1.51+ - file.seek(SeekFrom::Current(0)) + if append { + file.seek(SeekFrom::Start(old_data_size as u64))?; } @@ -481,4 +487,7 @@ } + file.write_all(&data)?; + file.flush()?; + file.seek(SeekFrom::Current(0)) })() .when_writing_file(&data_filename)?; diff --git a/rust/hg-core/src/revlog/revlog.rs b/rust/hg-core/src/revlog/revlog.rs index d3d3495a5749fdd9b9611dda1d0eee75aa79e891_cnVzdC9oZy1jb3JlL3NyYy9yZXZsb2cvcmV2bG9nLnJz..13dfad0f9f7a253ff292b3f232ee426fb221a02e_cnVzdC9oZy1jb3JlL3NyYy9yZXZsb2cvcmV2bG9nLnJz 100644 --- a/rust/hg-core/src/revlog/revlog.rs +++ b/rust/hg-core/src/revlog/revlog.rs @@ -32,6 +32,8 @@ | REVISION_FLAG_EXTSTORED | REVISION_FLAG_HASCOPIESINFO; +const NULL_REVLOG_ENTRY_FLAGS: u16 = 0; + #[derive(Debug, derive_more::From)] pub enum RevlogError { InvalidRevision, @@ -265,8 +267,23 @@ } } + pub fn make_null_entry(&self) -> RevlogEntry { + RevlogEntry { + revlog: self, + rev: NULL_REVISION, + bytes: b"", + compressed_len: 0, + uncompressed_len: 0, + base_rev_or_base_of_delta_chain: None, + p1: NULL_REVISION, + p2: NULL_REVISION, + flags: NULL_REVLOG_ENTRY_FLAGS, + hash: NULL_NODE, + } + } + /// Get an entry of the revlog. pub fn get_entry( &self, rev: Revision, ) -> Result<RevlogEntry, RevlogError> { @@ -268,8 +285,11 @@ /// Get an entry of the revlog. pub fn get_entry( &self, rev: Revision, ) -> Result<RevlogEntry, RevlogError> { + if rev == NULL_REVISION { + return Ok(self.make_null_entry()); + } let index_entry = self .index .get_entry(rev) diff --git a/tests/bundles/test-revlog-diff-relative-to-nullrev.sh b/tests/bundles/test-revlog-diff-relative-to-nullrev.sh new file mode 100755 index 0000000000000000000000000000000000000000..13dfad0f9f7a253ff292b3f232ee426fb221a02e_dGVzdHMvYnVuZGxlcy90ZXN0LXJldmxvZy1kaWZmLXJlbGF0aXZlLXRvLW51bGxyZXYuc2g= --- /dev/null +++ b/tests/bundles/test-revlog-diff-relative-to-nullrev.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# +# Make sure to patch mercurial to create the delta against nullrev +# if deltainfo is None: +#- deltainfo = self._fullsnapshotinfo(fh, revinfo, target_rev) +#+ deltainfo = self._builddeltainfo(revinfo, nullrev, fh) + +cd "`dirname \"$0\"`" +export HGRCPATH= +export HGMODULEPOLICY=py + +rm -rf nullrev-diff +../../hg init nullrev-diff --config format.revlog-compression=zlib +cd nullrev-diff +echo hi > a +../../../hg commit -Am root-B +../../../hg debugdeltachain a +rm -rf .hg/cache/ .hg/wcache/ +cd .. + +tar cf test-revlog-diff-relative-to-nullrev.tar nullrev-diff + +rm -rf nullrev-diff diff --git a/tests/bundles/test-revlog-diff-relative-to-nullrev.tar b/tests/bundles/test-revlog-diff-relative-to-nullrev.tar new file mode 100644 index 0000000000000000000000000000000000000000..28d0754dc9d18310896fae6d75a4c00634f07c59 GIT binary patch literal 30720 zc%1E=U5MON6vwmFg*HlC5p|{7M)BLun0u3(>{=^Z3+j{AClOrmPI5C7CzDL`QM-y1 z#g)GJAi`p?R4w=<qAe9cC|HZWNa=%H@dFhK3ieIJexSG}`!UXDnhnEd?sj+14`ydF zyE8YFbN}bubIzT0BggUCo|0)>*8M_nkYjX0l6#~Rx&3|~0jXHU1Y@kJf{YX$V?iQ4 zI-dnZK}dZ`5*pNRcu~l*{x0wLD-^fg{^eS=7jvL{`>Tqsf&F>VVgDfXd{*o%=-&Q> zD4>5XWZ2(lyCd6YK_O!#2G&$H+y1(evcIk%RgmODM$e(o?QeP^a}CdAqU9RYs4=n1 zT;@~9WKKxMcxpIabq^AQCiMe0dnnG#5plUs9Vmbd{hKtTg$04`{eSE&6!8B%Sj_&k zY!@;{p8rQkPy2sa0sqg3HNwE$qYcT?bWCzEJ;$wMW7j`&@!s*rHXr@v$p>CMx#RdB z`?B7tW<yL^X=LbMmK)TyEf$0Y_<&sfYoty8B+<VXhYQd@5B}GG5^nEH>;KpdA3y!~ z^S|Br*Jr11n$pe<z4hJX*X=LgdvN;+G$2T8uNt*J7~cL$?dFZYZ#i*l+u@J4A(gy` z>qmZ^oY-{m{ULYvYtqqY&uqRldi4bF&j@6aq5o!$#;bkL3ya7AbMrsak^gBb2L1B@ zNrr+HJX}GlMGTD^SW)B=OEZW$f_0sk6q6CHLabqk?W>W@_;1vxTV-*!zw`w_p8m1g zvHsVS`Y-680|Tx2UyW~-bo@Vg-Q7oaJ@ecb%DVGket%c=!yS{KTx@@N690Fse^t6h z7(M;c#Fo#G-8Sfc_0ALbY<lzcA@$CGe|}pTyKwD+{exp$&mPB@&ffmvMCH)d`r(Z` z*T3=9%f>g~PCfYPp`DLkn5_P^>(71X9-2D8rc!;hTB>;e>|eX?-7_y7_;v79^Og;~ zuS(yLq5rvJU@>Fl#(#t$|6eBN3q26s`;!d)N3Q9WE7Yh*P0Mzc*aynZ|K-&BAIq@= z0R8g-i$Z&@CyPSM^~++@p|&fSwjW=k_>;FyigN2o89c{IW)b}tHb#E_k30H5NC*A% zpw;M+#RWCr<gh&j1evc+7K{Hy6#;YgpYH!+nLz%}hYtG}TKSnJfB#Q9|I;)D`ahhQ zo|eq0(HNInc9%j=YQ~d-xuFyVwi`cDW0L2XlFyo+<k0c>LM-=b1BK9`|3Y{E&5~dL zN%wy=M4<n}g`h@#Rtl&E`}z8k4*fec2ulqX1hmS^;oc(reE#}ROV@u1mO=kKNS61d zhnM>-faRh?|E<DaapbT6+v`8tO#D|7=${99{oQ%o<i~tdIDR-lxt=D_BRced*&_ea z=uZDL^&drr`tKFRjNC#0tB%X-zt}}U*ZM~~g8uK71?Ycuk`e!X>Ke78#>mhA)B8U) z0{ve;n9QP)1J(R~B_sZ`z$k8v?)QIcNQd~(1z9w_M#B!F`rn^q#DCAL#}mHJ>HmLp zeE(m^D(IgJ3+&&cHGRJQ)BRsnCb0j93-FzN000000000000000000000000000000 M000000DzUnf3O+GNdN!< diff --git a/tests/run-tests.py b/tests/run-tests.py index d3d3495a5749fdd9b9611dda1d0eee75aa79e891_dGVzdHMvcnVuLXRlc3RzLnB5..13dfad0f9f7a253ff292b3f232ee426fb221a02e_dGVzdHMvcnVuLXRlc3RzLnB5 100755 --- a/tests/run-tests.py +++ b/tests/run-tests.py @@ -2505,6 +2505,7 @@ done = queue.Queue() running = 0 + channels_lock = threading.Lock() channels = [""] * self._jobs def job(test, result): @@ -2508,12 +2509,15 @@ channels = [""] * self._jobs def job(test, result): - for n, v in enumerate(channels): - if not v: - channel = n - break - else: - raise ValueError('Could not find output channel') - channels[channel] = "=" + test.name[5:].split(".")[0] + with channels_lock: + for n, v in enumerate(channels): + if not v: + channel = n + break + else: + raise ValueError('Could not find output channel') + channels[channel] = "=" + test.name[5:].split(".")[0] + + r = None try: test(result) @@ -2518,6 +2522,5 @@ try: test(result) - done.put(None) except KeyboardInterrupt: pass except: # re-raises @@ -2521,10 +2524,10 @@ except KeyboardInterrupt: pass except: # re-raises - done.put(('!', test, 'run-test raised an error, see traceback')) + r = ('!', test, 'run-test raised an error, see traceback') raise finally: try: channels[channel] = '' except IndexError: pass @@ -2525,9 +2528,10 @@ raise finally: try: channels[channel] = '' except IndexError: pass + done.put(r) def stat(): count = 0 diff --git a/tests/test-dirstate.t b/tests/test-dirstate.t index d3d3495a5749fdd9b9611dda1d0eee75aa79e891_dGVzdHMvdGVzdC1kaXJzdGF0ZS50..13dfad0f9f7a253ff292b3f232ee426fb221a02e_dGVzdHMvdGVzdC1kaXJzdGF0ZS50 100644 --- a/tests/test-dirstate.t +++ b/tests/test-dirstate.t @@ -131,6 +131,10 @@ > hg debugstate --docket | grep uuid | sed 's/.*uuid: \(.*\)/\1/' > } + $ find_dirstate_data_size () { + > hg debugstate --docket | grep 'size of dirstate data' | sed 's/.*size of dirstate data: \(.*\)/\1/' + > } + $ dirstate_uuid_has_not_changed () { > # Non-Rust always rewrites the whole dirstate > if [ $# -eq 1 ] || ([ -n "$HGMODULEPOLICY" ] && [ -z "${HGMODULEPOLICY##*rust*}" ]) || [ -n "$RHG_INSTALLED_AS_HG" ]; then @@ -161,5 +165,6 @@ Trigger an append with a small change - $ echo "modified" > dir2/f + $ current_data_size=$(find_dirstate_data_size) + $ rm dir2/f $ hg st @@ -165,6 +170,6 @@ $ hg st - M dir2/f + ! dir2/f $ dirstate_data_files | wc -l *1 (re) $ dirstate_uuid_has_not_changed not testing because using Python implementation (no-rust no-rhg !) @@ -167,7 +172,12 @@ $ dirstate_data_files | wc -l *1 (re) $ dirstate_uuid_has_not_changed not testing because using Python implementation (no-rust no-rhg !) + $ new_data_size=$(find_dirstate_data_size) + $ [ "$current_data_size" -eq "$new_data_size" ]; echo $? + 0 (no-rust no-rhg !) + 1 (rust !) + 1 (no-rust rhg !) Unused bytes counter is non-0 when appending $ touch file @@ -176,5 +186,4 @@ Trigger a rust/rhg run which updates the unused bytes value $ hg st - M dir2/f A file @@ -180,4 +189,5 @@ A file + ! dir2/f $ dirstate_data_files | wc -l *1 (re) $ dirstate_uuid_has_not_changed diff --git a/tests/test-revlog.t b/tests/test-revlog.t index d3d3495a5749fdd9b9611dda1d0eee75aa79e891_dGVzdHMvdGVzdC1yZXZsb2cudA==..13dfad0f9f7a253ff292b3f232ee426fb221a02e_dGVzdHMvdGVzdC1yZXZsb2cudA== 100644 --- a/tests/test-revlog.t +++ b/tests/test-revlog.t @@ -32,6 +32,7 @@ Test for CVE-2016-3630 + $ mkdir test2; cd test2 $ hg init >>> import codecs @@ -52,3 +53,31 @@ >>> rl = revlog.revlog(tvfs, target=(KIND_OTHER, b'test'), radix=b'a') >>> rl.revision(1) mpatchError(*'patch cannot be decoded'*) (glob) + + $ cd .. + + +Regression test for support for the old repos with strange diff encoding. +Apparently it used to be possible (maybe it's still possible, but we don't know how) +to create commits whose diffs are encoded relative to a nullrev. +This test checks that a repo with that encoding can still be read. + +This is what we did to produce the repo in test-revlog-diff-relative-to-nullrev.tar: + +- tweak the code in mercurial/revlogutils/deltas.py to produce such "trivial" deltas: +> if deltainfo is None: +> - deltainfo = self._fullsnapshotinfo(fh, revinfo, target_rev) +> + deltainfo = self._builddeltainfo(revinfo, nullrev, fh) +- hg init +- echo hi > a +- hg commit -Am_ +- remove some cache files + + $ tar --force-local -xf "$TESTDIR"/bundles/test-revlog-diff-relative-to-nullrev.tar + $ cd nullrev-diff + $ hg debugdeltachain a + rev chain# chainlen prev delta size rawsize chainsize ratio lindist extradist extraratio readsize largestblk rddensity srchunks + 0 1 2 -1 p1 15 3 15 5.00000 15 0 0.00000 15 15 1.00000 1 + $ hg cat --config rhg.cat=true -r 0 a + hi + $ cd ..