Skip to content
Snippets Groups Projects
Commit 985a98c6 authored by Yuya Nishihara's avatar Yuya Nishihara
Browse files

similar: use cheaper hash() function to test exact matches

We just need a hash table {fctx.data(): fctx} which doesn't keep fctx.data()
in memory. Let's simply use hash(fctx.data()) to put data out from memory,
and manage collided fctx objects by list.

This isn't significantly faster than using sha1, but is more correct as we
know SHA-1 collision attack is getting practical.

Benchmark with 50k added/removed files, on tmpfs:

  $ hg addremove --dry-run --time -q

  previous:   real 12.420 secs (user 11.120+0.000 sys 1.280+0.000)
  this patch: real 12.350 secs (user 11.210+0.000 sys 1.140+0.000)
parent 2efd9771
No related branches found
No related tags found
No related merge requests found
...@@ -7,8 +7,6 @@ ...@@ -7,8 +7,6 @@
from __future__ import absolute_import from __future__ import absolute_import
import hashlib
from .i18n import _ from .i18n import _
from . import ( from . import (
bdiff, bdiff,
...@@ -23,5 +21,6 @@ ...@@ -23,5 +21,6 @@
''' '''
numfiles = len(added) + len(removed) numfiles = len(added) + len(removed)
# Get hashes of removed files. # Build table of removed files: {hash(fctx.data()): [fctx, ...]}.
# We use hash() to discard fctx.data() from memory.
hashes = {} hashes = {}
...@@ -27,4 +26,4 @@ ...@@ -27,4 +26,4 @@
hashes = {} hashes = {}
for i, fctx in enumerate(reversed(removed)): for i, fctx in enumerate(removed):
repo.ui.progress(_('searching for exact renames'), i, total=numfiles, repo.ui.progress(_('searching for exact renames'), i, total=numfiles,
unit=_('files')) unit=_('files'))
...@@ -29,10 +28,13 @@ ...@@ -29,10 +28,13 @@
repo.ui.progress(_('searching for exact renames'), i, total=numfiles, repo.ui.progress(_('searching for exact renames'), i, total=numfiles,
unit=_('files')) unit=_('files'))
h = hashlib.sha1(fctx.data()).digest() h = hash(fctx.data())
hashes[h] = fctx if h not in hashes:
hashes[h] = [fctx]
else:
hashes[h].append(fctx)
# For each added file, see if it corresponds to a removed file. # For each added file, see if it corresponds to a removed file.
for i, fctx in enumerate(added): for i, fctx in enumerate(added):
repo.ui.progress(_('searching for exact renames'), i + len(removed), repo.ui.progress(_('searching for exact renames'), i + len(removed),
total=numfiles, unit=_('files')) total=numfiles, unit=_('files'))
adata = fctx.data() adata = fctx.data()
...@@ -33,12 +35,11 @@ ...@@ -33,12 +35,11 @@
# For each added file, see if it corresponds to a removed file. # For each added file, see if it corresponds to a removed file.
for i, fctx in enumerate(added): for i, fctx in enumerate(added):
repo.ui.progress(_('searching for exact renames'), i + len(removed), repo.ui.progress(_('searching for exact renames'), i + len(removed),
total=numfiles, unit=_('files')) total=numfiles, unit=_('files'))
adata = fctx.data() adata = fctx.data()
h = hashlib.sha1(adata).digest() h = hash(adata)
if h in hashes: for rfctx in hashes.get(h, []):
rfctx = hashes[h]
# compare between actual file contents for exact identity # compare between actual file contents for exact identity
if adata == rfctx.data(): if adata == rfctx.data():
yield (rfctx, fctx) yield (rfctx, fctx)
...@@ -42,6 +43,7 @@ ...@@ -42,6 +43,7 @@
# compare between actual file contents for exact identity # compare between actual file contents for exact identity
if adata == rfctx.data(): if adata == rfctx.data():
yield (rfctx, fctx) yield (rfctx, fctx)
break
# Done # Done
repo.ui.progress(_('searching for exact renames'), None) repo.ui.progress(_('searching for exact renames'), None)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment