Commit 082a8fc0 authored by Matti Picus's avatar Matti Picus
Browse files

add pinned object count to gc collection stats reported in the on_gc_collect hook

--HG--
branch : gc-hooks2
parent 672be4b02ed7
Pipeline #59422 passed with stage
in 15 minutes and 29 seconds
......@@ -13,8 +13,8 @@ Incminimark first allocates objects in so called *nursery* - place for young
objects, where allocation is very cheap, being just a pointer bump. The nursery
size is a very crucial variable - depending on your workload (one or many
processes) and cache sizes you might want to experiment with it via
*PYPY_GC_NURSERY* environment variable. When the nursery is full, there is
performed a minor collection. Freed objects are no longer referencable and
*PYPY_GC_NURSERY* environment variable. When the nursery is full, a minor
collection is performed. Freed objects are no longer referencable and
just die, just by not being referenced any more; on the other hand, objects
found to still be alive must survive and are copied from the nursery
to the old generation. Either to arenas, which are collections
......@@ -260,6 +260,9 @@ The attributes for ``GcCollectStats`` in the ``on_gc_collect`` hook are:
Total number of bytes used by raw-malloced objects, before and after the
major collection.
``pinned_objects``
the number of pinned objects.
Note that ``GcCollectStats`` does **not** have a ``duration`` field. This is
because all the GC work is done inside ``gc-collect-step``:
``gc-collect-done`` is used only to give additional stats, but doesn't do any
......
......@@ -57,7 +57,7 @@ class LowLevelGcHooks(GcHooks):
def on_gc_collect(self, num_major_collects,
arenas_count_before, arenas_count_after,
arenas_bytes, rawmalloc_bytes_before,
rawmalloc_bytes_after):
rawmalloc_bytes_after, pinned_objects):
action = self.w_hooks.gc_collect
action.count += 1
action.num_major_collects = num_major_collects
......@@ -66,6 +66,7 @@ class LowLevelGcHooks(GcHooks):
action.arenas_bytes = arenas_bytes
action.rawmalloc_bytes_before = rawmalloc_bytes_before
action.rawmalloc_bytes_after = rawmalloc_bytes_after
action.pinned_objects = pinned_objects
action.fire()
......@@ -218,6 +219,7 @@ class GcCollectHookAction(NoRecursiveAction):
arenas_bytes = 0
rawmalloc_bytes_before = 0
rawmalloc_bytes_after = 0
pinned_objects = 0
def __init__(self, space):
NoRecursiveAction.__init__(self, space)
......@@ -239,6 +241,7 @@ class GcCollectHookAction(NoRecursiveAction):
self.arenas_bytes = NonConstant(r_uint(42))
self.rawmalloc_bytes_before = NonConstant(r_uint(42))
self.rawmalloc_bytes_after = NonConstant(r_uint(42))
self.pinned_objects = NonConstant(-42)
self.fire()
def _do_perform(self, ec, frame):
......@@ -248,7 +251,9 @@ class GcCollectHookAction(NoRecursiveAction):
self.arenas_count_after,
self.arenas_bytes,
self.rawmalloc_bytes_before,
self.rawmalloc_bytes_after)
self.rawmalloc_bytes_after,
self.pinned_objects,
)
self.reset()
self.space.call_function(self.w_callable, w_stats)
......@@ -298,7 +303,7 @@ class W_GcCollectStats(W_Root):
def __init__(self, count, num_major_collects,
arenas_count_before, arenas_count_after,
arenas_bytes, rawmalloc_bytes_before,
rawmalloc_bytes_after):
rawmalloc_bytes_after, pinned_objects):
self.count = count
self.num_major_collects = num_major_collects
self.arenas_count_before = arenas_count_before
......@@ -306,6 +311,7 @@ class W_GcCollectStats(W_Root):
self.arenas_bytes = arenas_bytes
self.rawmalloc_bytes_before = rawmalloc_bytes_before
self.rawmalloc_bytes_after = rawmalloc_bytes_after
self.pinned_objects = pinned_objects
# just a shortcut to make the typedefs shorter
......@@ -379,5 +385,7 @@ W_GcCollectStats.typedef = TypeDef(
"arenas_count_after",
"arenas_bytes",
"rawmalloc_bytes_before",
"rawmalloc_bytes_after"))
"rawmalloc_bytes_after",
"pinned_objects",
))
)
......@@ -20,9 +20,9 @@ class AppTestGcHooks(object):
def fire_gc_collect_step(space, duration, oldstate, newstate):
gchooks.fire_gc_collect_step(duration, oldstate, newstate)
@unwrap_spec(ObjSpace, int, int, int, r_uint, r_uint, r_uint)
def fire_gc_collect(space, a, b, c, d, e, f):
gchooks.fire_gc_collect(a, b, c, d, e, f)
@unwrap_spec(ObjSpace, int, int, int, r_uint, r_uint, r_uint, r_uint)
def fire_gc_collect(space, a, b, c, d, e, f, g):
gchooks.fire_gc_collect(a, b, c, d, e, f, g)
@unwrap_spec(ObjSpace)
def fire_many(space):
......@@ -31,7 +31,7 @@ class AppTestGcHooks(object):
gchooks.fire_gc_collect_step(5.0, 0, 0)
gchooks.fire_gc_collect_step(15.0, 0, 0)
gchooks.fire_gc_collect_step(22.0, 0, 0)
gchooks.fire_gc_collect(1, 2, 3, 4, 5, 6)
gchooks.fire_gc_collect(1, 2, 3, 4, 5, 6, 7)
cls.w_fire_gc_minor = space.wrap(interp2app(fire_gc_minor))
cls.w_fire_gc_collect_step = space.wrap(interp2app(fire_gc_collect_step))
......@@ -103,20 +103,21 @@ class AppTestGcHooks(object):
stats.arenas_count_after,
stats.arenas_bytes,
stats.rawmalloc_bytes_before,
stats.rawmalloc_bytes_after))
stats.rawmalloc_bytes_after,
stats.pinned_objects))
gc.hooks.on_gc_collect = on_gc_collect
self.fire_gc_collect(1, 2, 3, 4, 5, 6)
self.fire_gc_collect(7, 8, 9, 10, 11, 12)
self.fire_gc_collect(1, 2, 3, 4, 5, 6, 20)
self.fire_gc_collect(7, 8, 9, 10, 11, 12, 21)
assert lst == [
(1, 1, 2, 3, 4, 5, 6),
(1, 7, 8, 9, 10, 11, 12),
(1, 1, 2, 3, 4, 5, 6, 20),
(1, 7, 8, 9, 10, 11, 12, 21),
]
#
gc.hooks.on_gc_collect = None
self.fire_gc_collect(42, 42, 42, 42, 42, 42) # won't fire
self.fire_gc_collect(42, 42, 42, 42, 42, 42, 43) # won't fire
assert lst == [
(1, 1, 2, 3, 4, 5, 6),
(1, 7, 8, 9, 10, 11, 12),
(1, 1, 2, 3, 4, 5, 6, 20),
(1, 7, 8, 9, 10, 11, 12, 21),
]
def test_consts(self):
......
......@@ -40,12 +40,12 @@ class GcHooks(object):
def on_gc_collect(self, num_major_collects,
arenas_count_before, arenas_count_after,
arenas_bytes, rawmalloc_bytes_before,
rawmalloc_bytes_after):
rawmalloc_bytes_after, pinned_after):
"""
Called after a major collection is fully done
"""
# the fire_* methods are meant to be called from the GC are should NOT be
# the fire_* methods are meant to be called from the GC and should NOT be
# overridden
@rgc.no_collect
......@@ -62,9 +62,9 @@ class GcHooks(object):
def fire_gc_collect(self, num_major_collects,
arenas_count_before, arenas_count_after,
arenas_bytes, rawmalloc_bytes_before,
rawmalloc_bytes_after):
rawmalloc_bytes_after, pinned_objects):
if self.is_gc_collect_enabled():
self.on_gc_collect(num_major_collects,
arenas_count_before, arenas_count_after,
arenas_bytes, rawmalloc_bytes_before,
rawmalloc_bytes_after)
rawmalloc_bytes_after, pinned_objects)
......@@ -2523,7 +2523,8 @@ class IncrementalMiniMarkGC(MovingGCBase):
arenas_count_after=self.ac.arenas_count,
arenas_bytes=self.ac.total_memory_used,
rawmalloc_bytes_before=self.stat_rawmalloced_total_size,
rawmalloc_bytes_after=self.rawmalloced_total_size)
rawmalloc_bytes_after=self.rawmalloced_total_size,
pinned_objects = self.pinned_objects_in_nursery)
#
# Max heap size: gives an upper bound on the threshold. If we
# already have at least this much allocated, raise MemoryError.
......
......@@ -42,14 +42,16 @@ class MyGcHooks(GcHooks):
def on_gc_collect(self, num_major_collects,
arenas_count_before, arenas_count_after,
arenas_bytes, rawmalloc_bytes_before,
rawmalloc_bytes_after):
rawmalloc_bytes_after, pinned_objects):
self.collects.append({
'num_major_collects': num_major_collects,
'arenas_count_before': arenas_count_before,
'arenas_count_after': arenas_count_after,
'arenas_bytes': arenas_bytes,
'rawmalloc_bytes_before': rawmalloc_bytes_before,
'rawmalloc_bytes_after': rawmalloc_bytes_after})
'rawmalloc_bytes_after': rawmalloc_bytes_after,
'pinned_objects': pinned_objects,
})
class TestIncMiniMarkHooks(BaseDirectGCTest):
......@@ -99,7 +101,9 @@ class TestIncMiniMarkHooks(BaseDirectGCTest):
'arenas_count_after': 0,
'arenas_bytes': 0,
'rawmalloc_bytes_after': 0,
'rawmalloc_bytes_before': 0}
'rawmalloc_bytes_before': 0,
'pinned_objects': 0,
}
]
assert len(self.gc.hooks.durations) == 4 # 4 steps
for d in self.gc.hooks.durations:
......@@ -114,7 +118,9 @@ class TestIncMiniMarkHooks(BaseDirectGCTest):
'arenas_count_after': 1,
'arenas_bytes': self.size_of_S,
'rawmalloc_bytes_after': 0,
'rawmalloc_bytes_before': 0}
'rawmalloc_bytes_before': 0,
'pinned_objects': 0,
}
]
def test_hook_disabled(self):
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment