Commit de9eb7a8 authored by Armin Rigo's avatar Armin Rigo
Browse files

hg merge keys_with_hash

Improve the performance of dict.update() and a bunch of methods from
sets, by reusing the hash value stored in one dict when inspecting or
changing another dict with that key.  This is done with the help of new
xxx_with_hash() functions in objectmodel that work for any RPython dict
or r_dict.
......@@ -3,7 +3,7 @@ maps keys to objects. If a specific key is changed a lot, a level of
indirection is introduced to make the version tag change less often.
"""
from rpython.rlib import jit, rerased
from rpython.rlib import jit, rerased, objectmodel
from pypy.interpreter.baseobjspace import W_Root
from pypy.objspace.std.dictmultiobject import (
......@@ -162,8 +162,8 @@ class ModuleDictStrategy(DictStrategy):
def getitervalues(self, w_dict):
return self.unerase(w_dict.dstorage).itervalues()
def getiteritems(self, w_dict):
return self.unerase(w_dict.dstorage).iteritems()
def getiteritems_with_hash(self, w_dict):
return objectmodel.iteritems_with_hash(self.unerase(w_dict.dstorage))
wrapkey = _wrapkey
......
......@@ -511,7 +511,7 @@ class DictStrategy(object):
def getitervalues(self, w_dict):
raise NotImplementedError
def getiteritems(self, w_dict):
def getiteritems_with_hash(self, w_dict):
raise NotImplementedError
has_iterreversed = False
......@@ -634,7 +634,7 @@ class EmptyDictStrategy(DictStrategy):
def getitervalues(self, w_dict):
return iter([])
def getiteritems(self, w_dict):
def getiteritems_with_hash(self, w_dict):
return iter([])
def getiterreversed(self, w_dict):
......@@ -751,11 +751,11 @@ def create_iterator_classes(dictimpl):
class IterClassItems(BaseItemIterator):
def __init__(self, space, strategy, impl):
self.iterator = strategy.getiteritems(impl)
self.iterator = strategy.getiteritems_with_hash(impl)
BaseIteratorImplementation.__init__(self, space, strategy, impl)
def next_item_entry(self):
for key, value in self.iterator:
for key, value, keyhash in self.iterator:
return (wrapkey(self.space, key),
wrapvalue(self.space, value))
else:
......@@ -793,10 +793,10 @@ def create_iterator_classes(dictimpl):
# the logic is to call prepare_dict_update() after the first setitem():
# it gives the w_updatedict a chance to switch its strategy.
if 1: # (preserve indentation)
iteritems = self.getiteritems(w_dict)
iteritemsh = self.getiteritems_with_hash(w_dict)
if not same_strategy(self, w_updatedict):
# Different strategy. Try to copy one item of w_dict
for key, value in iteritems:
for key, value, keyhash in iteritemsh:
w_key = wrapkey(self.space, key)
w_value = wrapvalue(self.space, value)
w_updatedict.setitem(w_key, w_value)
......@@ -807,7 +807,7 @@ def create_iterator_classes(dictimpl):
w_updatedict.strategy.prepare_update(w_updatedict, count)
# If the strategy is still different, continue the slow way
if not same_strategy(self, w_updatedict):
for key, value in iteritems:
for key, value, keyhash in iteritemsh:
w_key = wrapkey(self.space, key)
w_value = wrapvalue(self.space, value)
w_updatedict.setitem(w_key, w_value)
......@@ -820,8 +820,8 @@ def create_iterator_classes(dictimpl):
# wrapping/unwrapping the key.
assert setitem_untyped is not None
dstorage = w_updatedict.dstorage
for key, value in iteritems:
setitem_untyped(self, dstorage, key, value)
for key, value, keyhash in iteritemsh:
setitem_untyped(self, dstorage, key, value, keyhash)
def same_strategy(self, w_otherdict):
return (setitem_untyped is not None and
......@@ -945,8 +945,8 @@ class AbstractTypedStrategy(object):
def getitervalues(self, w_dict):
return self.unerase(w_dict.dstorage).itervalues()
def getiteritems(self, w_dict):
return self.unerase(w_dict.dstorage).iteritems()
def getiteritems_with_hash(self, w_dict):
return objectmodel.iteritems_with_hash(self.unerase(w_dict.dstorage))
def getiterreversed(self, w_dict):
return objectmodel.reversed_dict(self.unerase(w_dict.dstorage))
......@@ -955,8 +955,9 @@ class AbstractTypedStrategy(object):
objectmodel.prepare_dict_update(self.unerase(w_dict.dstorage),
num_extra)
def setitem_untyped(self, dstorage, key, w_value):
self.unerase(dstorage)[key] = w_value
def setitem_untyped(self, dstorage, key, w_value, keyhash):
d = self.unerase(dstorage)
objectmodel.setitem_with_hash(d, key, keyhash, w_value)
class ObjectDictStrategy(AbstractTypedStrategy, DictStrategy):
......
from rpython.rlib import rerased
from rpython.rlib.objectmodel import iteritems_with_hash
from pypy.interpreter.error import OperationError, oefmt
from pypy.objspace.std.dictmultiobject import (
......@@ -103,8 +104,8 @@ class DictProxyStrategy(DictStrategy):
return self.unerase(w_dict.dstorage).dict_w.iterkeys()
def getitervalues(self, w_dict):
return self.unerase(w_dict.dstorage).dict_w.itervalues()
def getiteritems(self, w_dict):
return self.unerase(w_dict.dstorage).dict_w.iteritems()
def getiteritems_with_hash(self, w_dict):
return iteritems_with_hash(self.unerase(w_dict.dstorage).dict_w)
def wrapkey(space, key):
return space.wrap(key)
def wrapvalue(space, value):
......
......@@ -3,7 +3,7 @@
Based on two lists containing unwrapped key value pairs.
"""
from rpython.rlib import jit, rerased
from rpython.rlib import jit, rerased, objectmodel
from pypy.objspace.std.dictmultiobject import (
BytesDictStrategy, DictStrategy, EmptyDictStrategy, ObjectDictStrategy,
......@@ -165,13 +165,14 @@ class KwargsDictStrategy(DictStrategy):
def getitervalues(self, w_dict):
return iter(self.unerase(w_dict.dstorage)[1])
def getiteritems(self, w_dict):
return Zip(*self.unerase(w_dict.dstorage))
def getiteritems_with_hash(self, w_dict):
keys, values_w = self.unerase(w_dict.dstorage)
return ZipItemsWithHash(keys, values_w)
wrapkey = _wrapkey
class Zip(object):
class ZipItemsWithHash(object):
def __init__(self, list1, list2):
assert len(list1) == len(list2)
self.list1 = list1
......@@ -186,6 +187,7 @@ class Zip(object):
if i >= len(self.list1):
raise StopIteration
self.i = i + 1
return (self.list1[i], self.list2[i])
key = self.list1[i]
return (key, self.list2[i], objectmodel.compute_hash(key))
create_iterator_classes(KwargsDictStrategy)
......@@ -8,6 +8,8 @@ from pypy.objspace.std.intobject import W_IntObject
from pypy.objspace.std.unicodeobject import W_UnicodeObject
from rpython.rlib.objectmodel import r_dict
from rpython.rlib.objectmodel import iterkeys_with_hash, contains_with_hash
from rpython.rlib.objectmodel import setitem_with_hash, delitem_with_hash
from rpython.rlib.rarithmetic import intmask, r_uint
from rpython.rlib import rerased, jit
......@@ -961,12 +963,12 @@ class AbstractUnwrappedSetStrategy(object):
return self.erase(result_dict)
def _difference_unwrapped(self, w_set, w_other):
iterator = self.unerase(w_set.sstorage).iterkeys()
self_dict = self.unerase(w_set.sstorage)
other_dict = self.unerase(w_other.sstorage)
result_dict = self.get_empty_dict()
for key in iterator:
if key not in other_dict:
result_dict[key] = None
for key, keyhash in iterkeys_with_hash(self_dict):
if not contains_with_hash(other_dict, key, keyhash):
setitem_with_hash(result_dict, key, keyhash, None)
return self.erase(result_dict)
def _difference_base(self, w_set, w_other):
......@@ -989,10 +991,10 @@ class AbstractUnwrappedSetStrategy(object):
if w_set.sstorage is w_other.sstorage:
my_dict.clear()
return
iterator = self.unerase(w_other.sstorage).iterkeys()
for key in iterator:
other_dict = self.unerase(w_other.sstorage)
for key, keyhash in iterkeys_with_hash(other_dict):
try:
del my_dict[key]
delitem_with_hash(my_dict, key, keyhash)
except KeyError:
pass
......@@ -1020,12 +1022,12 @@ class AbstractUnwrappedSetStrategy(object):
d_new = self.get_empty_dict()
d_this = self.unerase(w_set.sstorage)
d_other = self.unerase(w_other.sstorage)
for key in d_other.keys():
if not key in d_this:
d_new[key] = None
for key in d_this.keys():
if not key in d_other:
d_new[key] = None
for key, keyhash in iterkeys_with_hash(d_other):
if not contains_with_hash(d_this, key, keyhash):
setitem_with_hash(d_new, key, keyhash, None)
for key, keyhash in iterkeys_with_hash(d_this):
if not contains_with_hash(d_other, key, keyhash):
setitem_with_hash(d_new, key, keyhash, None)
storage = self.erase(d_new)
return storage
......@@ -1105,9 +1107,9 @@ class AbstractUnwrappedSetStrategy(object):
result = self.get_empty_dict()
d_this = self.unerase(w_set.sstorage)
d_other = self.unerase(w_other.sstorage)
for key in d_this:
if key in d_other:
result[key] = None
for key, keyhash in iterkeys_with_hash(d_this):
if contains_with_hash(d_other, key, keyhash):
setitem_with_hash(result, key, keyhash, None)
return self.erase(result)
def intersect(self, w_set, w_other):
......@@ -1125,9 +1127,10 @@ class AbstractUnwrappedSetStrategy(object):
w_set.sstorage = storage
def _issubset_unwrapped(self, w_set, w_other):
d_set = self.unerase(w_set.sstorage)
d_other = self.unerase(w_other.sstorage)
for item in self.unerase(w_set.sstorage):
if not item in d_other:
for key, keyhash in iterkeys_with_hash(d_set):
if not contains_with_hash(d_other, key, keyhash):
return False
return True
......@@ -1152,8 +1155,8 @@ class AbstractUnwrappedSetStrategy(object):
def _isdisjoint_unwrapped(self, w_set, w_other):
d_set = self.unerase(w_set.sstorage)
d_other = self.unerase(w_other.sstorage)
for key in d_set:
if key in d_other:
for key, keyhash in iterkeys_with_hash(d_set):
if contains_with_hash(d_other, key, keyhash):
return False
return True
......
......@@ -520,26 +520,32 @@ class __extend__(pairtype(SomeDict, SomeDict)):
return dic1.__class__(dic1.dictdef.union(dic2.dictdef))
class __extend__(pairtype(SomeDict, SomeObject)):
def _dict_can_only_throw_keyerror(s_dct, *ignore):
if s_dct.dictdef.dictkey.custom_eq_hash:
return None # r_dict: can throw anything
return [KeyError]
def _dict_can_only_throw_nothing(s_dct, *ignore):
if s_dct.dictdef.dictkey.custom_eq_hash:
return None # r_dict: can throw anything
return [] # else: no possible exception
def _can_only_throw(dic1, *ignore):
if dic1.dictdef.dictkey.custom_eq_hash:
return None
return [KeyError]
class __extend__(pairtype(SomeDict, SomeObject)):
def getitem((dic1, obj2)):
dic1.dictdef.generalize_key(obj2)
return dic1.dictdef.read_value()
getitem.can_only_throw = _can_only_throw
getitem.can_only_throw = _dict_can_only_throw_keyerror
def setitem((dic1, obj2), s_value):
dic1.dictdef.generalize_key(obj2)
dic1.dictdef.generalize_value(s_value)
setitem.can_only_throw = _can_only_throw
setitem.can_only_throw = _dict_can_only_throw_nothing
def delitem((dic1, obj2)):
dic1.dictdef.generalize_key(obj2)
delitem.can_only_throw = _can_only_throw
delitem.can_only_throw = _dict_can_only_throw_keyerror
class __extend__(pairtype(SomeTuple, SomeInteger)):
......
......@@ -4,6 +4,7 @@ Unary operations on SomeValues.
from __future__ import absolute_import
from rpython.tool.pairtype import pair
from rpython.flowspace.operation import op
from rpython.flowspace.model import const, Constant
from rpython.flowspace.argument import CallSpec
......@@ -11,11 +12,13 @@ from rpython.annotator.model import (SomeObject, SomeInteger, SomeBool,
SomeString, SomeChar, SomeList, SomeDict, SomeTuple, SomeImpossibleValue,
SomeUnicodeCodePoint, SomeInstance, SomeBuiltin, SomeBuiltinMethod,
SomeFloat, SomeIterator, SomePBC, SomeNone, SomeType, s_ImpossibleValue,
s_Bool, s_None, unionof, add_knowntypedata,
s_Bool, s_None, s_Int, unionof, add_knowntypedata,
HarmlesslyBlocked, SomeWeakRef, SomeUnicodeString, SomeByteArray)
from rpython.annotator.bookkeeper import getbookkeeper, immutablevalue
from rpython.annotator import builtin
from rpython.annotator.binaryop import _clone ## XXX where to put this?
from rpython.annotator.binaryop import _dict_can_only_throw_keyerror
from rpython.annotator.binaryop import _dict_can_only_throw_nothing
from rpython.annotator.model import AnnotatorError
from rpython.annotator.argument import simple_args, complex_args
......@@ -364,20 +367,19 @@ def check_negative_slice(s_start, s_stop, error="slicing"):
raise AnnotatorError("%s: not proven to have non-negative stop" % error)
def _can_only_throw(s_dct, *ignore):
if s_dct.dictdef.dictkey.custom_eq_hash:
return None # r_dict: can throw anything
return [] # else: no possible exception
@op.contains.register(SomeDict)
def contains_SomeDict(annotator, dct, element):
annotator.annotation(dct).dictdef.generalize_key(annotator.annotation(element))
if annotator.annotation(dct)._is_empty():
def dict_contains(s_dct, s_element):
s_dct.dictdef.generalize_key(s_element)
if s_dct._is_empty():
s_bool = SomeBool()
s_bool.const = False
return s_bool
return s_Bool
contains_SomeDict.can_only_throw = _can_only_throw
@op.contains.register(SomeDict)
def contains_SomeDict(annotator, dct, element):
return dict_contains(annotator.annotation(dct),
annotator.annotation(element))
contains_SomeDict.can_only_throw = _dict_can_only_throw_nothing
class __extend__(SomeDict):
......@@ -401,16 +403,22 @@ class __extend__(SomeDict):
return self.dictdef.read_key()
elif variant == 'values':
return self.dictdef.read_value()
elif variant == 'items':
elif variant == 'items' or variant == 'items_with_hash':
s_key = self.dictdef.read_key()
s_value = self.dictdef.read_value()
if (isinstance(s_key, SomeImpossibleValue) or
isinstance(s_value, SomeImpossibleValue)):
return s_ImpossibleValue
else:
elif variant == 'items':
return SomeTuple((s_key, s_value))
else:
raise ValueError
elif variant == 'items_with_hash':
return SomeTuple((s_key, s_value, s_Int))
elif variant == 'keys_with_hash':
s_key = self.dictdef.read_key()
if isinstance(s_key, SomeImpossibleValue):
return s_ImpossibleValue
return SomeTuple((s_key, s_Int))
raise ValueError(variant)
def method_get(self, key, dfl):
self.dictdef.generalize_key(key)
......@@ -448,6 +456,12 @@ class __extend__(SomeDict):
def method_iteritems(self):
return SomeIterator(self, 'items')
def method_iterkeys_with_hash(self):
return SomeIterator(self, 'keys_with_hash')
def method_iteritems_with_hash(self):
return SomeIterator(self, 'items_with_hash')
def method_clear(self):
pass
......@@ -460,6 +474,22 @@ class __extend__(SomeDict):
self.dictdef.generalize_value(s_dfl)
return self.dictdef.read_value()
def method_contains_with_hash(self, s_key, s_hash):
return dict_contains(self, s_key)
method_contains_with_hash.can_only_throw = _dict_can_only_throw_nothing
def method_setitem_with_hash(self, s_key, s_hash, s_value):
pair(self, s_key).setitem(s_value)
method_setitem_with_hash.can_only_throw = _dict_can_only_throw_nothing
def method_getitem_with_hash(self, s_key, s_hash):
return pair(self, s_key).getitem()
method_getitem_with_hash.can_only_throw = _dict_can_only_throw_keyerror
def method_delitem_with_hash(self, s_key, s_hash):
pair(self, s_key).delitem()
method_delitem_with_hash.can_only_throw = _dict_can_only_throw_keyerror
@op.contains.register(SomeString)
@op.contains.register(SomeUnicodeString)
def contains_String(annotator, string, char):
......
......@@ -788,6 +788,71 @@ def reversed_dict(d):
d = d.keys()
return reversed(d)
def _expected_hash(d, key):
if isinstance(d, r_dict):
return d.key_hash(key)
else:
return compute_hash(key)
def _iterkeys_with_hash_untranslated(d):
for k in d:
yield (k, _expected_hash(d, k))
@specialize.call_location()
def iterkeys_with_hash(d):
"""Iterates (key, hash) pairs without recomputing the hash."""
if not we_are_translated():
return _iterkeys_with_hash_untranslated(d)
return d.iterkeys_with_hash()
def _iteritems_with_hash_untranslated(d):
for k, v in d.iteritems():
yield (k, v, _expected_hash(d, k))
@specialize.call_location()
def iteritems_with_hash(d):
"""Iterates (key, value, keyhash) triples without recomputing the hash."""
if not we_are_translated():
return _iteritems_with_hash_untranslated(d)
return d.iteritems_with_hash()
@specialize.call_location()
def contains_with_hash(d, key, h):
"""Same as 'key in d'. The extra argument is the hash. Use this only
if you got the hash just now from some other ..._with_hash() function."""
if not we_are_translated():
assert _expected_hash(d, key) == h
return key in d
return d.contains_with_hash(key, h)
@specialize.call_location()
def setitem_with_hash(d, key, h, value):
"""Same as 'd[key] = value'. The extra argument is the hash. Use this only
if you got the hash just now from some other ..._with_hash() function."""
if not we_are_translated():
assert _expected_hash(d, key) == h
d[key] = value
return
d.setitem_with_hash(key, h, value)
@specialize.call_location()
def getitem_with_hash(d, key, h):
"""Same as 'd[key]'. The extra argument is the hash. Use this only
if you got the hash just now from some other ..._with_hash() function."""
if not we_are_translated():
assert _expected_hash(d, key) == h
return d[key]
return d.getitem_with_hash(key, h)
@specialize.call_location()
def delitem_with_hash(d, key, h):
"""Same as 'del d[key]'. The extra argument is the hash. Use this only
if you got the hash just now from some other ..._with_hash() function."""
if not we_are_translated():
assert _expected_hash(d, key) == h
del d[key]
return
d.delitem_with_hash(key, h)
# ____________________________________________________________
......
......@@ -592,6 +592,97 @@ def test_resizelist_hint_len():
r = interpret(f, [29])
assert r == 1
def test_iterkeys_with_hash():
def f(i):
d = {i+.0: 5, i+.5: 6}
total = 0
for k, h in iterkeys_with_hash(d):
total += k * h
total -= (i + 0.0) * compute_hash(i + 0.0)
total -= (i + 0.5) * compute_hash(i + 0.5)
return total
assert f(29) == 0.0
r = interpret(f, [29])
assert r == 0.0
def test_iteritems_with_hash():
def f(i):
d = {i+.0: 5, i+.5: 6}
total = 0
for k, v, h in iteritems_with_hash(d):
total += k * h * v
total -= (i + 0.0) * compute_hash(i + 0.0) * 5
total -= (i + 0.5) * compute_hash(i + 0.5) * 6
return total
assert f(29) == 0.0
r = interpret(f, [29])
assert r == 0.0
def test_contains_with_hash():
def f(i):
d = {i+.5: 5}
assert contains_with_hash(d, i+.5, compute_hash(i+.5))
assert not contains_with_hash(d, i+.3, compute_hash(i+.3))
return 0
f(29)
interpret(f, [29])
def test_setitem_with_hash():
def f(i):
d = {}
setitem_with_hash(d, i+.5, compute_hash(i+.5), 42)
setitem_with_hash(d, i+.6, compute_hash(i+.6), -612)
return d[i+.5]
assert f(29) == 42
res = interpret(f, [27])
assert res == 42
def test_getitem_with_hash():
def f(i):
d = {i+.5: 42, i+.6: -612}
return getitem_with_hash(d, i+.5, compute_hash(i+.5))
assert f(29) == 42
res = interpret(f, [27])
assert res == 42
def test_delitem_with_hash():
def f(i):
d = {i+.5: 42, i+.6: -612}
delitem_with_hash(d, i+.5, compute_hash(i+.5))
try:
delitem_with_hash(d, i+.5, compute_hash(i+.5))
except KeyError:
pass
else:
raise AssertionError
return 0
f(29)
interpret(f, [27])
def test_rdict_with_hash():
def f(i):
d = r_dict(strange_key_eq, strange_key_hash)
h = strange_key_hash("abc")
assert h == strange_key_hash("aXX") and strange_key_eq("abc", "aXX")
setitem_with_hash(d, "abc", h, i)
assert getitem_with_hash(d, "aXX", h) == i
try:
getitem_with_hash(d, "bYY", strange_key_hash("bYY"))
except KeyError:
pass
else:
raise AssertionError
return 0
assert f(29) == 0
interpret(f, [27])
def test_import_from_mixin():
class M: # old-style
def f(self): pass
......
......@@ -335,6 +335,14 @@ class OrderedDictRepr(AbstractDictRepr):
hop.exception_cannot_occur()
return DictIteratorRepr(self, "items").newiter(hop)
def rtype_method_iterkeys_with_hash(self, hop):
hop.exception_cannot_occur()
return DictIteratorRepr(self, "keys_with_hash").newiter(hop)
def rtype_method_iteritems_with_hash(self, hop):
hop.exception_cannot_occur()