Skip to content
Snippets Groups Projects
Commit 9b413478f261 authored by Matt Harbison's avatar Matt Harbison
Browse files

lfs: deduplicate oids in the transfer

Apparently, we can't rely on the server to deduplicate for us.

Sadly, the pointer object isn't hashable, so it can't be reduced by converting
it to a set.  In order to be hashable, it needs to be immutable.  I had a bunch
of code to change it to composition and forward the readonly dict methods to a
member dict.  But the pointer is updated via __setitem__() when creating the
pointer file.  So it didn't see worth adding all of that code to the class.
parent 264b90a060b7
No related branches found
No related tags found
No related merge requests found
...@@ -194,7 +194,7 @@ ...@@ -194,7 +194,7 @@
def writebatch(self, pointers, fromstore): def writebatch(self, pointers, fromstore):
"""Batch upload from local to remote blobstore.""" """Batch upload from local to remote blobstore."""
self._batch(pointers, fromstore, 'upload') self._batch(_deduplicate(pointers), fromstore, 'upload')
def readbatch(self, pointers, tostore): def readbatch(self, pointers, tostore):
"""Batch download from remote to local blostore.""" """Batch download from remote to local blostore."""
...@@ -198,7 +198,7 @@ ...@@ -198,7 +198,7 @@
def readbatch(self, pointers, tostore): def readbatch(self, pointers, tostore):
"""Batch download from remote to local blostore.""" """Batch download from remote to local blostore."""
self._batch(pointers, tostore, 'download') self._batch(_deduplicate(pointers), tostore, 'download')
def _batchrequest(self, pointers, action): def _batchrequest(self, pointers, action):
"""Get metadata about objects pointed by pointers for given action """Get metadata about objects pointed by pointers for given action
...@@ -399,9 +399,9 @@ ...@@ -399,9 +399,9 @@
self.vfs = lfsvfs(fullpath) self.vfs = lfsvfs(fullpath)
def writebatch(self, pointers, fromstore): def writebatch(self, pointers, fromstore):
for p in pointers: for p in _deduplicate(pointers):
content = fromstore.read(p.oid(), verify=True) content = fromstore.read(p.oid(), verify=True)
with self.vfs(p.oid(), 'wb', atomictemp=True) as fp: with self.vfs(p.oid(), 'wb', atomictemp=True) as fp:
fp.write(content) fp.write(content)
def readbatch(self, pointers, tostore): def readbatch(self, pointers, tostore):
...@@ -403,9 +403,9 @@ ...@@ -403,9 +403,9 @@
content = fromstore.read(p.oid(), verify=True) content = fromstore.read(p.oid(), verify=True)
with self.vfs(p.oid(), 'wb', atomictemp=True) as fp: with self.vfs(p.oid(), 'wb', atomictemp=True) as fp:
fp.write(content) fp.write(content)
def readbatch(self, pointers, tostore): def readbatch(self, pointers, tostore):
for p in pointers: for p in _deduplicate(pointers):
with self.vfs(p.oid(), 'rb') as fp: with self.vfs(p.oid(), 'rb') as fp:
tostore.download(p.oid(), fp) tostore.download(p.oid(), fp)
...@@ -444,6 +444,13 @@ ...@@ -444,6 +444,13 @@
None: _promptremote, None: _promptremote,
} }
def _deduplicate(pointers):
"""Remove any duplicate oids that exist in the list"""
reduced = util.sortdict()
for p in pointers:
reduced[p.oid()] = p
return reduced.values()
def _verify(oid, content): def _verify(oid, content):
realoid = hashlib.sha256(content).hexdigest() realoid = hashlib.sha256(content).hexdigest()
if realoid != oid: if realoid != oid:
......
...@@ -153,7 +153,7 @@ ...@@ -153,7 +153,7 @@
$ rm -rf .hg/store/lfs `hg config lfs.usercache` $ rm -rf .hg/store/lfs `hg config lfs.usercache`
$ hg archive -vr 1 ../archive $ hg archive -vr 1 ../archive
lfs: need to transfer 4 objects (63 bytes) lfs: need to transfer 3 objects (51 bytes)
lfs: downloading 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b (12 bytes) lfs: downloading 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b (12 bytes)
lfs: adding 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b to the usercache lfs: adding 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b to the usercache
lfs: processed: 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b lfs: processed: 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b
...@@ -157,8 +157,6 @@ ...@@ -157,8 +157,6 @@
lfs: downloading 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b (12 bytes) lfs: downloading 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b (12 bytes)
lfs: adding 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b to the usercache lfs: adding 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b to the usercache
lfs: processed: 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b lfs: processed: 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b
lfs: downloading 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b (12 bytes)
lfs: processed: 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b
lfs: downloading 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 (20 bytes) lfs: downloading 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 (20 bytes)
lfs: adding 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 to the usercache lfs: adding 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 to the usercache
lfs: processed: 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 lfs: processed: 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19
...@@ -181,7 +179,7 @@ ...@@ -181,7 +179,7 @@
$ rm -rf .hg/store/lfs `hg config lfs.usercache` $ rm -rf .hg/store/lfs `hg config lfs.usercache`
$ hg cat -vr 1 a b c $ hg cat -vr 1 a b c
lfs: need to transfer 3 objects (43 bytes) lfs: need to transfer 2 objects (31 bytes)
lfs: downloading 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b (12 bytes) lfs: downloading 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b (12 bytes)
lfs: adding 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b to the usercache lfs: adding 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b to the usercache
lfs: processed: 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b lfs: processed: 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b
...@@ -185,8 +183,6 @@ ...@@ -185,8 +183,6 @@
lfs: downloading 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b (12 bytes) lfs: downloading 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b (12 bytes)
lfs: adding 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b to the usercache lfs: adding 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b to the usercache
lfs: processed: 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b lfs: processed: 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b
lfs: downloading 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b (12 bytes)
lfs: processed: 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b
lfs: downloading d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 (19 bytes) lfs: downloading d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 (19 bytes)
lfs: adding d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 to the usercache lfs: adding d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 to the usercache
lfs: processed: d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 lfs: processed: d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998
...@@ -207,7 +203,7 @@ ...@@ -207,7 +203,7 @@
reverting b reverting b
reverting c reverting c
reverting d reverting d
lfs: need to transfer 4 objects (63 bytes) lfs: need to transfer 3 objects (51 bytes)
lfs: downloading 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b (12 bytes) lfs: downloading 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b (12 bytes)
lfs: adding 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b to the usercache lfs: adding 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b to the usercache
lfs: processed: 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b lfs: processed: 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b
...@@ -211,8 +207,6 @@ ...@@ -211,8 +207,6 @@
lfs: downloading 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b (12 bytes) lfs: downloading 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b (12 bytes)
lfs: adding 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b to the usercache lfs: adding 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b to the usercache
lfs: processed: 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b lfs: processed: 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b
lfs: downloading 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b (12 bytes)
lfs: processed: 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b
lfs: downloading 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 (20 bytes) lfs: downloading 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 (20 bytes)
lfs: adding 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 to the usercache lfs: adding 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 to the usercache
lfs: processed: 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 lfs: processed: 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment