pypy/doc/whatsnew-head.rst

@@ -20,3 +20,10 @@ Fixes incorrect error text for ``int('1', base=1000)``
 .. branch: heptapod
 
 adapt contributing documentation to heptapod
+
+
+.. branch: pypy-jitdriver-greenkeys
+
+Improve code generation for generators (and generator expressions in
+particular) when passing them to builtin functions that consume iterators, such
+as ``sum``, ``map``, ``max``, etc.

pypy/interpreter/baseobjspace.py

@@ -22,12 +22,12 @@ from pypy.interpreter.miscutils import ThreadLocals, make_weak_value_dictionary
 
 __all__ = ['ObjSpace', 'OperationError', 'W_Root']
 
-def get_printable_location(tp):
-    return "unpackiterable: %s" % (tp, )
+def get_printable_location(greenkey):
+    return "unpackiterable [%s]" % (greenkey.iterator_greenkey_printable(), )
 
 unpackiterable_driver = jit.JitDriver(name='unpackiterable',
-                                      greens=['tp'],
-                                      reds=['items', 'w_iterator'],
+                                      greens=['greenkey'],
+                                      reds='auto',
                                       get_printable_location=get_printable_location)
 
 
@@ -373,6 +373,15 @@ class W_Root(object):
             return lst[:]
         return None
 
+    def iterator_greenkey(self, space):
+        """ Return something that can be used as a green key in jit drivers
+        that iterate over self. by default, it's just the type of self, but
+        custom iterators should override it. """
+        return space.type(self)
+
+    def iterator_greenkey_printable(self):
+        return "?"
+
 
 class InternalSpaceCache(Cache):
     """A generic cache for an object space.  Arbitrary information can
@@ -966,11 +975,9 @@ class ObjSpace(object):
         except MemoryError:
             items = [] # it might have lied
 
-        tp = self.type(w_iterator)
+        greenkey = self.iterator_greenkey(w_iterator)
         while True:
-            unpackiterable_driver.jit_merge_point(tp=tp,
-                                                  w_iterator=w_iterator,
-                                                  items=items)
+            unpackiterable_driver.jit_merge_point(greenkey=greenkey)
             try:
                 w_item = self.next(w_iterator)
             except OperationError as e:
@@ -2025,6 +2032,12 @@ class ObjSpace(object):
         finally:
             self.sys.track_resources = flag
 
+    def iterator_greenkey(self, w_iterable):
+        """ Return something that can be used as a green key in jit drivers
+        that iterate over self. by default, it's just the type of self, but
+        custom iterators should override it. """
+        return w_iterable.iterator_greenkey(self)
+
 
 class AppExecCache(SpaceCache):
     @not_rpython

pypy/interpreter/generator.py

@@ -231,6 +231,9 @@ return next yielded value or raise StopIteration."""
         self.frame = None
         rgc.may_ignore_finalizer(self)
 
+    def iterator_greenkey(self, space):
+        return self.pycode
+
 
 def get_printable_location_genentry(bytecode):
     return '%s <generator>' % (bytecode.get_repr(),)

pypy/interpreter/pycode.py

@@ -425,6 +425,9 @@ class PyCode(eval.Code):
         return "<code object %s, file '%s', line %d>" % (
             self.co_name, self.co_filename, self.co_firstlineno)
 
+    def iterator_greenkey_printable(self):
+        return self.get_repr()
+
     def __repr__(self):
         return self.get_repr()
 

pypy/interpreter/pyopcode.py

@@ -147,10 +147,12 @@ class __extend__(pyframe.PyFrame):
         while True:
             self.last_instr = intmask(next_instr)
             if jit.we_are_jitted():
-                ec.bytecode_only_trace(self)
+                if self.debugdata:
+                    ec.bytecode_only_trace(self)
+                    next_instr = r_uint(self.last_instr)
             else:
                 ec.bytecode_trace(self)
-            next_instr = r_uint(self.last_instr)
+                next_instr = r_uint(self.last_instr)
             opcode = ord(co_code[next_instr])
             next_instr += 1
 

pypy/module/__builtin__/functional.py

@@ -131,10 +131,21 @@ def range_with_longs(space, w_start, w_stop, w_step):
         v = v.add(step)
     return space.newlist(res_w)
 
+def get_printable_location(has_key, has_item, greenkey):
+    return "min [has_key=%s, has_item=%s, %s]" % (
+            has_key, has_item, greenkey.iterator_greenkey_printable())
+
 min_jitdriver = jit.JitDriver(name='min',
-        greens=['has_key', 'has_item', 'w_type'], reds='auto')
+        greens=['has_key', 'has_item', 'greenkey'], reds='auto',
+        get_printable_location=get_printable_location)
+
+def get_printable_location(has_key, has_item, greenkey):
+    return "min [has_key=%s, has_item=%s, %s]" % (
+            has_key, has_item, greenkey.iterator_greenkey_printable())
+
 max_jitdriver = jit.JitDriver(name='max',
-        greens=['has_key', 'has_item', 'w_type'], reds='auto')
+        greens=['has_key', 'has_item', 'greenkey'], reds='auto',
+        get_printable_location=get_printable_location)
 
 @specialize.arg(3)
 def min_max_sequence(space, w_sequence, w_key, implementation_of):
@@ -145,14 +156,14 @@ def min_max_sequence(space, w_sequence, w_key, implementation_of):
         compare = space.lt
         jitdriver = min_jitdriver
     w_iter = space.iter(w_sequence)
-    w_type = space.type(w_iter)
+    greenkey = space.iterator_greenkey(w_iter)
     has_key = w_key is not None
     has_item = False
     w_max_item = None
     w_max_val = None
     while True:
         jitdriver.jit_merge_point(has_key=has_key, has_item=has_item,
-                                  w_type=w_type)
+                                  greenkey=greenkey)
         try:
             w_item = space.next(w_iter)
         except OperationError as e:

pypy/module/array/interp_array.py

@@ -72,7 +72,8 @@ def compare_arrays(space, arr1, arr2, comp_op):
     if comp_op == NE and arr1.len != arr2.len:
         return space.w_True
     lgt = min(arr1.len, arr2.len)
-    for i in range(lgt):
+    i = 0
+    while i < lgt:
         arr_eq_driver.jit_merge_point(comp_func=comp_op)
         w_elem1 = arr1.w_getitem(space, i, integer_instead_of_char=True)
         w_elem2 = arr2.w_getitem(space, i, integer_instead_of_char=True)
@@ -102,6 +103,7 @@ def compare_arrays(space, arr1, arr2, comp_op):
                 return space.w_False
             elif not space.is_true(space.eq(w_elem1, w_elem2)):
                 return space.w_True
+        i += 1
     # we have some leftovers
     if comp_op == EQ:
         return space.w_True
@@ -128,7 +130,8 @@ def index_count_array(arr, w_val, count=False):
     tp_item = space.type(w_val)
     arrclass = arr.__class__
     cnt = 0
-    for i in range(arr.len):
+    i = 0
+    while i < arr.len:
         index_count_jd.jit_merge_point(
             tp_item=tp_item, count=count,
             arrclass=arrclass)
@@ -138,6 +141,7 @@ def index_count_array(arr, w_val, count=False):
                 cnt += 1
             else:
                 return i
+        i += 1
     if count:
         return cnt
     return -1

pypy/module/itertools/interp_itertools.py

@@ -322,10 +322,12 @@ W_IFilterFalse.typedef = TypeDef(
     """)
 
 
+def get_printable_location(greenkey):
+    return "islice_ignore_items [%s]" % (greenkey.iterator_greenkey_printable(), )
 islice_ignore_items_driver = jit.JitDriver(name='islice_ignore_items',
-                                           greens=['tp'],
-                                           reds=['num', 'w_islice',
-                                                 'w_iterator'])
+                                           greens=['greenkey'],
+                                           reds='auto',
+                                           get_printable_location=get_printable_location)
 
 class W_ISlice(W_Root):
     def __init__(self, space, w_iterable, w_startstop, args_w):
@@ -418,12 +420,9 @@ class W_ISlice(W_Root):
         if w_iterator is None:
             raise OperationError(self.space.w_StopIteration, self.space.w_None)
 
-        tp = self.space.type(w_iterator)
+        greenkey = self.space.iterator_greenkey(w_iterator)
         while True:
-            islice_ignore_items_driver.jit_merge_point(tp=tp,
-                                                       num=num,
-                                                       w_islice=self,
-                                                       w_iterator=w_iterator)
+            islice_ignore_items_driver.jit_merge_point(greenkey=greenkey)
             try:
                 self.space.next(w_iterator)
             except OperationError as e:

pypy/module/pypyjit/test_pypy_c/test_generators.py

@@ -61,3 +61,13 @@ class TestGenerators(BaseTestPyPyC):
             i2 = int_sub_ovf(i1, 42)
             guard_no_overflow(descr=...)
             """)
+
+    def test_nonstd_jitdriver_distinguishes_generators(self):
+        def main():
+            # test the "contains" jitdriver, but the others are the same
+            res = (9999 in (i for i in range(20000)))
+            res += (9999 in (i for i in range(20000)))
+            return res
+        log = self.run(main, [])
+        assert len(log.loops) == 2  # as opposed to one loop, one bridge
+

pypy/objspace/descroperation.py

@@ -124,8 +124,14 @@ class Object(object):
     def descr__init__(space, w_obj, __args__):
         pass
 
+def get_printable_location(itergreenkey, w_itemtype):
+    return "DescrOperation.contains [%s, %s]" % (
+            itergreenkey.iterator_greenkey_printable(),
+            w_itemtype.getname(w_itemtype.space))
+
 contains_jitdriver = jit.JitDriver(name='contains',
-        greens=['w_type'], reds='auto')
+        greens=['itergreenkey', 'w_itemtype'], reds='auto',
+        get_printable_location=get_printable_location)
 
 class DescrOperation(object):
     # This is meant to be a *mixin*.
@@ -406,9 +412,10 @@ class DescrOperation(object):
 
     def _contains(space, w_container, w_item):
         w_iter = space.iter(w_container)
-        w_type = space.type(w_iter)
+        itergreenkey = space.iterator_greenkey(w_iter)
+        w_itemtype = space.type(w_item)
         while 1:
-            contains_jitdriver.jit_merge_point(w_type=w_type)
+            contains_jitdriver.jit_merge_point(itergreenkey=itergreenkey, w_itemtype=w_itemtype)
             try:
                 w_next = space.next(w_iter)
             except OperationError as e:

pypy/objspace/std/bytearrayobject.py

@@ -561,14 +561,14 @@ def makebytearraydata_w(space, w_source):
         return list(buf.as_str())
     return _from_byte_sequence(space, w_source)
 
-def _get_printable_location(w_type):
-    return ('bytearray_from_byte_sequence [w_type=%s]' %
-            w_type.getname(w_type.space))
+def _get_printable_location(greenkey):
+    return ('bytearray_from_byte_sequence [%s]' %
+            greenkey.iterator_greenkey_printable())
 
 _byteseq_jitdriver = jit.JitDriver(
     name='bytearray_from_byte_sequence',
-    greens=['w_type'],
-    reds=['w_iter', 'data'],
+    greens=['greenkey'],
+    reds='auto',
     get_printable_location=_get_printable_location)
 
 def _from_byte_sequence(space, w_source):
@@ -586,11 +586,9 @@ def _from_byte_sequence(space, w_source):
     return data
 
 def _from_byte_sequence_loop(space, w_iter, data):
-    w_type = space.type(w_iter)
+    greenkey = space.iterator_greenkey(w_iter)
     while True:
-        _byteseq_jitdriver.jit_merge_point(w_type=w_type,
-                                           w_iter=w_iter,
-                                           data=data)
+        _byteseq_jitdriver.jit_merge_point(greenkey=greenkey)
         try:
             w_item = space.next(w_iter)
         except OperationError as e:

pypy/objspace/std/listobject.py

@@ -131,26 +131,26 @@ def get_strategy_from_list_objects(space, list_w, sizehint):
     return space.fromcache(ObjectListStrategy)
 
 
-def _get_printable_location(w_type):
-    return ('list__do_extend_from_iterable [w_type=%s]' %
-            w_type.getname(w_type.space))
+def _get_printable_location(strategy, greenkey):
+    return 'list__do_extend_from_iterable [%s, %s]' % (
+        strategy,
+        greenkey.iterator_greenkey_printable())
 
 
 _do_extend_jitdriver = jit.JitDriver(
     name='list__do_extend_from_iterable',
-    greens=['w_type'],
-    reds=['i', 'w_iterator', 'w_list'],
+    greens=['strategy', 'greenkey'],
+    reds='auto',
     get_printable_location=_get_printable_location)
 
 def _do_extend_from_iterable(space, w_list, w_iterable):
     w_iterator = space.iter(w_iterable)
-    w_type = space.type(w_iterator)
+    greenkey = space.iterator_greenkey(w_iterator)
     i = 0
     while True:
-        _do_extend_jitdriver.jit_merge_point(w_type=w_type,
-                                             i=i,
-                                             w_iterator=w_iterator,
-                                             w_list=w_list)
+        _do_extend_jitdriver.jit_merge_point(
+                greenkey=greenkey,
+                strategy = w_list.strategy)
         try:
             w_list.append(space.next(w_iterator))
         except OperationError as e:
@@ -749,7 +749,9 @@ class W_ListObject(W_Root):
         if mucked:
             raise oefmt(space.w_ValueError, "list modified during sort")
 
-find_jmp = jit.JitDriver(greens = ['tp'], reds = 'auto', name = 'list.find')
+def get_printable_location(strategy, tp):
+    return "list.find [%s, %s]" % (strategy, tp.getname(tp.space), )
+find_jmp = jit.JitDriver(greens = ['strategy', 'tp'], reds = 'auto', name = 'list.find', get_printable_location=get_printable_location)
 
 class ListStrategy(object):
 
@@ -777,7 +779,7 @@ class ListStrategy(object):
         # needs to be safe against eq_w mutating stuff
         tp = space.type(w_item)
         while i < stop and i < w_list.length():
-            find_jmp.jit_merge_point(tp=tp)
+            find_jmp.jit_merge_point(tp=tp, strategy=self)
             if space.eq_w(w_item, w_list.getitem(i)):
                 return i
             i += 1

pypy/objspace/std/setobject.py

@@ -1681,7 +1681,7 @@ def _pick_correct_strategy_unroll(space, w_set, w_iterable):
 
 
 def get_printable_location(tp, strategy):
-    return "create_set: %s %s" % (tp, strategy)
+    return "create_set: %s %s" % (tp.iterator_greenkey_printable(), strategy)
 
 create_set_driver = jit.JitDriver(name='create_set',
                                   greens=['tp', 'strategy'],
@@ -1692,7 +1692,7 @@ def _create_from_iterable(space, w_set, w_iterable):
     w_set.strategy = strategy = space.fromcache(EmptySetStrategy)
     w_set.sstorage = strategy.get_empty_storage()
 
-    tp = space.type(w_iterable)
+    tp = space.iterator_greenkey(w_iterable)
 
     w_iter = space.iter(w_iterable)
     while True:

pypy/objspace/std/tupleobject.py

@@ -27,13 +27,22 @@ def _unroll_condition_cmp(self, space, other):
             jit.loop_unrolling_heuristic(other, other.length(), UNROLL_CUTOFF))
 
 
-contains_jmp = jit.JitDriver(greens = ['tp'], reds = 'auto',
-                             name = 'tuple.contains')
+def get_printable_location(tp):
+    return "tuple.contains [%s]" % (tp.getname(tp.space), )
+
+contains_driver = jit.JitDriver(greens = ['tp'], reds = 'auto',
+                             name = 'tuple.contains',
+                             get_printable_location=get_printable_location)
+
+def get_printable_location(w_type):
+    return "tuple.hash [%s]" % (w_type.getname(w_type.space), )
 
 hash_driver = jit.JitDriver(
     name='tuple.hash',
     greens=['w_type'],
-    reds='auto')
+    reds='auto',
+    get_printable_location=get_printable_location
+    )
 
 class W_AbstractTupleObject(W_Root):
     __slots__ = ()
@@ -159,10 +168,14 @@ class W_AbstractTupleObject(W_Root):
 
     def _descr_contains_jmp(self, space, w_obj):
         tp = space.type(w_obj)
-        for w_item in self.tolist():
-            contains_jmp.jit_merge_point(tp=tp)
+        list_w = self.tolist()
+        i = 0
+        while i < len(list_w):
+            w_item = list_w[i]
+            contains_driver.jit_merge_point(tp=tp)
             if space.eq_w(w_obj, w_item):
                 return space.w_True
+            i += 1
         return space.w_False
 
     def descr_add(self, space, w_other):
@@ -307,12 +320,16 @@ class W_TupleObject(W_AbstractTupleObject):
         x = 0x345678
         z = len(self.wrappeditems)
         w_type = space.type(self.wrappeditems[0])
-        for w_item in self.wrappeditems:
+        wrappeditems = self.wrappeditems
+        i = 0
+        while i < len(wrappeditems):
             hash_driver.jit_merge_point(w_type=w_type)
+            w_item = wrappeditems[i]
             y = space.hash_w(w_item)
             x = (x ^ y) * mult
             z -= 1
             mult += 82520 + z + z
+            i += 1
         x += 97531
         return space.newint(intmask(x))
 

pypy/objspace/std/typeobject.py

@@ -711,6 +711,9 @@ class W_TypeObject(W_Root):
         else:
             return space.newtext("<%s '%s'>" % (kind, self.name))
 
+    def iterator_greenkey_printable(self):
+        return self.name
+
     def descr_getattribute(self, space, w_name):
         name = space.text_w(w_name)
         w_descr = space.lookup(self, name)

rpython/jit/metainterp/warmspot.py

@@ -425,9 +425,9 @@ class WarmRunnerDesc(object):
         graph = copygraph(graph)
         [jmpp] = find_jit_merge_points([graph])
         graph.startblock = support.split_before_jit_merge_point(*jmpp)
-        # XXX this is incredibly obscure, but this is sometiems necessary
+        # XXX this is incredibly obscure, but this is sometimes necessary
         #     so we don't explode in checkgraph. for reasons unknown this
-        #     is not contanied within simplify_graph
+        #     is not contained within simplify_graph
         removenoops.remove_same_as(graph)
         # a crash in the following checkgraph() means that you forgot
         # to list some variable in greens=[] or reds=[] in JitDriver,