Skip to content
GitLab
Menu
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
PyPy
pypy
Commits
5f18636263b6
Commit
082a8fc0
authored
Dec 14, 2022
by
Matti Picus
Browse files
add pinned object count to gc collection stats reported in the on_gc_collect hook
--HG-- branch : gc-hooks2
parent
672be4b02ed7
Pipeline
#59422
passed with stage
in 15 minutes and 29 seconds
Changes
6
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
pypy/doc/gc_info.rst
View file @
5f186362
...
...
@@ -13,8 +13,8 @@ Incminimark first allocates objects in so called *nursery* - place for young
objects, where allocation is very cheap, being just a pointer bump. The nursery
size is a very crucial variable - depending on your workload (one or many
processes) and cache sizes you might want to experiment with it via
*PYPY_GC_NURSERY* environment variable. When the nursery is full,
there is
performed a minor collection
. Freed objects are no longer referencable and
*PYPY_GC_NURSERY* environment variable. When the nursery is full,
a minor
collection is performed
. Freed objects are no longer referencable and
just die, just by not being referenced any more; on the other hand, objects
found to still be alive must survive and are copied from the nursery
to the old generation. Either to arenas, which are collections
...
...
@@ -260,6 +260,9 @@ The attributes for ``GcCollectStats`` in the ``on_gc_collect`` hook are:
Total number of bytes used by raw-malloced objects, before and after the
major collection.
``pinned_objects``
the number of pinned objects.
Note that ``GcCollectStats`` does **not** have a ``duration`` field. This is
because all the GC work is done inside ``gc-collect-step``:
``gc-collect-done`` is used only to give additional stats, but doesn't do any
...
...
pypy/module/gc/hook.py
View file @
5f186362
...
...
@@ -57,7 +57,7 @@ class LowLevelGcHooks(GcHooks):
def
on_gc_collect
(
self
,
num_major_collects
,
arenas_count_before
,
arenas_count_after
,
arenas_bytes
,
rawmalloc_bytes_before
,
rawmalloc_bytes_after
):
rawmalloc_bytes_after
,
pinned_objects
):
action
=
self
.
w_hooks
.
gc_collect
action
.
count
+=
1
action
.
num_major_collects
=
num_major_collects
...
...
@@ -66,6 +66,7 @@ class LowLevelGcHooks(GcHooks):
action
.
arenas_bytes
=
arenas_bytes
action
.
rawmalloc_bytes_before
=
rawmalloc_bytes_before
action
.
rawmalloc_bytes_after
=
rawmalloc_bytes_after
action
.
pinned_objects
=
pinned_objects
action
.
fire
()
...
...
@@ -218,6 +219,7 @@ class GcCollectHookAction(NoRecursiveAction):
arenas_bytes
=
0
rawmalloc_bytes_before
=
0
rawmalloc_bytes_after
=
0
pinned_objects
=
0
def
__init__
(
self
,
space
):
NoRecursiveAction
.
__init__
(
self
,
space
)
...
...
@@ -239,6 +241,7 @@ class GcCollectHookAction(NoRecursiveAction):
self
.
arenas_bytes
=
NonConstant
(
r_uint
(
42
))
self
.
rawmalloc_bytes_before
=
NonConstant
(
r_uint
(
42
))
self
.
rawmalloc_bytes_after
=
NonConstant
(
r_uint
(
42
))
self
.
pinned_objects
=
NonConstant
(
-
42
)
self
.
fire
()
def
_do_perform
(
self
,
ec
,
frame
):
...
...
@@ -248,7 +251,9 @@ class GcCollectHookAction(NoRecursiveAction):
self
.
arenas_count_after
,
self
.
arenas_bytes
,
self
.
rawmalloc_bytes_before
,
self
.
rawmalloc_bytes_after
)
self
.
rawmalloc_bytes_after
,
self
.
pinned_objects
,
)
self
.
reset
()
self
.
space
.
call_function
(
self
.
w_callable
,
w_stats
)
...
...
@@ -298,7 +303,7 @@ class W_GcCollectStats(W_Root):
def
__init__
(
self
,
count
,
num_major_collects
,
arenas_count_before
,
arenas_count_after
,
arenas_bytes
,
rawmalloc_bytes_before
,
rawmalloc_bytes_after
):
rawmalloc_bytes_after
,
pinned_objects
):
self
.
count
=
count
self
.
num_major_collects
=
num_major_collects
self
.
arenas_count_before
=
arenas_count_before
...
...
@@ -306,6 +311,7 @@ class W_GcCollectStats(W_Root):
self
.
arenas_bytes
=
arenas_bytes
self
.
rawmalloc_bytes_before
=
rawmalloc_bytes_before
self
.
rawmalloc_bytes_after
=
rawmalloc_bytes_after
self
.
pinned_objects
=
pinned_objects
# just a shortcut to make the typedefs shorter
...
...
@@ -379,5 +385,7 @@ W_GcCollectStats.typedef = TypeDef(
"arenas_count_after"
,
"arenas_bytes"
,
"rawmalloc_bytes_before"
,
"rawmalloc_bytes_after"
))
"rawmalloc_bytes_after"
,
"pinned_objects"
,
))
)
pypy/module/gc/test/test_hook.py
View file @
5f186362
...
...
@@ -20,9 +20,9 @@ class AppTestGcHooks(object):
def
fire_gc_collect_step
(
space
,
duration
,
oldstate
,
newstate
):
gchooks
.
fire_gc_collect_step
(
duration
,
oldstate
,
newstate
)
@
unwrap_spec
(
ObjSpace
,
int
,
int
,
int
,
r_uint
,
r_uint
,
r_uint
)
def
fire_gc_collect
(
space
,
a
,
b
,
c
,
d
,
e
,
f
):
gchooks
.
fire_gc_collect
(
a
,
b
,
c
,
d
,
e
,
f
)
@
unwrap_spec
(
ObjSpace
,
int
,
int
,
int
,
r_uint
,
r_uint
,
r_uint
,
r_uint
)
def
fire_gc_collect
(
space
,
a
,
b
,
c
,
d
,
e
,
f
,
g
):
gchooks
.
fire_gc_collect
(
a
,
b
,
c
,
d
,
e
,
f
,
g
)
@
unwrap_spec
(
ObjSpace
)
def
fire_many
(
space
):
...
...
@@ -31,7 +31,7 @@ class AppTestGcHooks(object):
gchooks
.
fire_gc_collect_step
(
5.0
,
0
,
0
)
gchooks
.
fire_gc_collect_step
(
15.0
,
0
,
0
)
gchooks
.
fire_gc_collect_step
(
22.0
,
0
,
0
)
gchooks
.
fire_gc_collect
(
1
,
2
,
3
,
4
,
5
,
6
)
gchooks
.
fire_gc_collect
(
1
,
2
,
3
,
4
,
5
,
6
,
7
)
cls
.
w_fire_gc_minor
=
space
.
wrap
(
interp2app
(
fire_gc_minor
))
cls
.
w_fire_gc_collect_step
=
space
.
wrap
(
interp2app
(
fire_gc_collect_step
))
...
...
@@ -103,20 +103,21 @@ class AppTestGcHooks(object):
stats
.
arenas_count_after
,
stats
.
arenas_bytes
,
stats
.
rawmalloc_bytes_before
,
stats
.
rawmalloc_bytes_after
))
stats
.
rawmalloc_bytes_after
,
stats
.
pinned_objects
))
gc
.
hooks
.
on_gc_collect
=
on_gc_collect
self
.
fire_gc_collect
(
1
,
2
,
3
,
4
,
5
,
6
)
self
.
fire_gc_collect
(
7
,
8
,
9
,
10
,
11
,
12
)
self
.
fire_gc_collect
(
1
,
2
,
3
,
4
,
5
,
6
,
20
)
self
.
fire_gc_collect
(
7
,
8
,
9
,
10
,
11
,
12
,
21
)
assert
lst
==
[
(
1
,
1
,
2
,
3
,
4
,
5
,
6
),
(
1
,
7
,
8
,
9
,
10
,
11
,
12
),
(
1
,
1
,
2
,
3
,
4
,
5
,
6
,
20
),
(
1
,
7
,
8
,
9
,
10
,
11
,
12
,
21
),
]
#
gc
.
hooks
.
on_gc_collect
=
None
self
.
fire_gc_collect
(
42
,
42
,
42
,
42
,
42
,
42
)
# won't fire
self
.
fire_gc_collect
(
42
,
42
,
42
,
42
,
42
,
42
,
43
)
# won't fire
assert
lst
==
[
(
1
,
1
,
2
,
3
,
4
,
5
,
6
),
(
1
,
7
,
8
,
9
,
10
,
11
,
12
),
(
1
,
1
,
2
,
3
,
4
,
5
,
6
,
20
),
(
1
,
7
,
8
,
9
,
10
,
11
,
12
,
21
),
]
def
test_consts
(
self
):
...
...
rpython/memory/gc/hook.py
View file @
5f186362
...
...
@@ -40,12 +40,12 @@ class GcHooks(object):
def
on_gc_collect
(
self
,
num_major_collects
,
arenas_count_before
,
arenas_count_after
,
arenas_bytes
,
rawmalloc_bytes_before
,
rawmalloc_bytes_after
):
rawmalloc_bytes_after
,
pinned_after
):
"""
Called after a major collection is fully done
"""
# the fire_* methods are meant to be called from the GC a
re
should NOT be
# the fire_* methods are meant to be called from the GC a
nd
should NOT be
# overridden
@
rgc
.
no_collect
...
...
@@ -62,9 +62,9 @@ class GcHooks(object):
def
fire_gc_collect
(
self
,
num_major_collects
,
arenas_count_before
,
arenas_count_after
,
arenas_bytes
,
rawmalloc_bytes_before
,
rawmalloc_bytes_after
):
rawmalloc_bytes_after
,
pinned_objects
):
if
self
.
is_gc_collect_enabled
():
self
.
on_gc_collect
(
num_major_collects
,
arenas_count_before
,
arenas_count_after
,
arenas_bytes
,
rawmalloc_bytes_before
,
rawmalloc_bytes_after
)
rawmalloc_bytes_after
,
pinned_objects
)
rpython/memory/gc/incminimark.py
View file @
5f186362
...
...
@@ -2523,7 +2523,8 @@ class IncrementalMiniMarkGC(MovingGCBase):
arenas_count_after
=
self
.
ac
.
arenas_count
,
arenas_bytes
=
self
.
ac
.
total_memory_used
,
rawmalloc_bytes_before
=
self
.
stat_rawmalloced_total_size
,
rawmalloc_bytes_after
=
self
.
rawmalloced_total_size
)
rawmalloc_bytes_after
=
self
.
rawmalloced_total_size
,
pinned_objects
=
self
.
pinned_objects_in_nursery
)
#
# Max heap size: gives an upper bound on the threshold. If we
# already have at least this much allocated, raise MemoryError.
...
...
rpython/memory/gc/test/test_hook.py
View file @
5f186362
...
...
@@ -42,14 +42,16 @@ class MyGcHooks(GcHooks):
def
on_gc_collect
(
self
,
num_major_collects
,
arenas_count_before
,
arenas_count_after
,
arenas_bytes
,
rawmalloc_bytes_before
,
rawmalloc_bytes_after
):
rawmalloc_bytes_after
,
pinned_objects
):
self
.
collects
.
append
({
'num_major_collects'
:
num_major_collects
,
'arenas_count_before'
:
arenas_count_before
,
'arenas_count_after'
:
arenas_count_after
,
'arenas_bytes'
:
arenas_bytes
,
'rawmalloc_bytes_before'
:
rawmalloc_bytes_before
,
'rawmalloc_bytes_after'
:
rawmalloc_bytes_after
})
'rawmalloc_bytes_after'
:
rawmalloc_bytes_after
,
'pinned_objects'
:
pinned_objects
,
})
class
TestIncMiniMarkHooks
(
BaseDirectGCTest
):
...
...
@@ -99,7 +101,9 @@ class TestIncMiniMarkHooks(BaseDirectGCTest):
'arenas_count_after'
:
0
,
'arenas_bytes'
:
0
,
'rawmalloc_bytes_after'
:
0
,
'rawmalloc_bytes_before'
:
0
}
'rawmalloc_bytes_before'
:
0
,
'pinned_objects'
:
0
,
}
]
assert
len
(
self
.
gc
.
hooks
.
durations
)
==
4
# 4 steps
for
d
in
self
.
gc
.
hooks
.
durations
:
...
...
@@ -114,7 +118,9 @@ class TestIncMiniMarkHooks(BaseDirectGCTest):
'arenas_count_after'
:
1
,
'arenas_bytes'
:
self
.
size_of_S
,
'rawmalloc_bytes_after'
:
0
,
'rawmalloc_bytes_before'
:
0
}
'rawmalloc_bytes_before'
:
0
,
'pinned_objects'
:
0
,
}
]
def
test_hook_disabled
(
self
):
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment