path: root/mm/memcontrol.c
diff options
authorRoman Gushchin <>2021-02-24 12:03:11 -0800
committerLinus Torvalds <>2021-02-24 13:38:29 -0800
commit2e9bd483159939ed2c0704b914294653c8341d25 (patch)
tree878ce98090f4ad550e476a5e80c746f62c44fda8 /mm/memcontrol.c
parentcad8320b4b395702e49578580c70026c8271ea88 (diff)
mm: memcg/slab: pre-allocate obj_cgroups for slab caches with SLAB_ACCOUNT
In general it's unknown in advance if a slab page will contain accounted objects or not. In order to avoid memory waste, an obj_cgroup vector is allocated dynamically when a need to account of a new object arises. Such approach is memory efficient, but requires an expensive cmpxchg() to set up the memcg/objcgs pointer, because an allocation can race with a different allocation on another cpu. But in some common cases it's known for sure that a slab page will contain accounted objects: if the page belongs to a slab cache with a SLAB_ACCOUNT flag set. It includes such popular objects like vm_area_struct, anon_vma, task_struct, etc. In such cases we can pre-allocate the objcgs vector and simple assign it to the page without any atomic operations, because at this early stage the page is not visible to anyone else. A very simplistic benchmark (allocating 10000000 64-bytes objects in a row) shows ~15% win. In the real life it seems that most workloads are not very sensitive to the speed of (accounted) slab allocations. [ open-code set_page_objcgs() and add some comments, by Johannes] Link: [ fix it for mm-slub-call-account_slab_page-after-slab-page-initialization-fix.patch] Link: Signed-off-by: Roman Gushchin <> Acked-by: Johannes Weiner <> Reviewed-by: Shakeel Butt <> Cc: Michal Hocko <> Cc: Christoph Lameter <> Signed-off-by: Andrew Morton <> Signed-off-by: Linus Torvalds <>
Diffstat (limited to 'mm/memcontrol.c')
1 files changed, 19 insertions, 4 deletions
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 0b9bd354e97e..60ce452e42e6 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -2935,9 +2935,10 @@ static void commit_charge(struct page *page, struct mem_cgroup *memcg)
int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
- gfp_t gfp)
+ gfp_t gfp, bool new_page)
unsigned int objects = objs_per_slab_page(s, page);
+ unsigned long memcg_data;
void *vec;
vec = kcalloc_node(objects, sizeof(struct obj_cgroup *), gfp,
@@ -2945,11 +2946,25 @@ int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
if (!vec)
return -ENOMEM;
- if (!set_page_objcgs(page, vec))
+ memcg_data = (unsigned long) vec | MEMCG_DATA_OBJCGS;
+ if (new_page) {
+ /*
+ * If the slab page is brand new and nobody can yet access
+ * it's memcg_data, no synchronization is required and
+ * memcg_data can be simply assigned.
+ */
+ page->memcg_data = memcg_data;
+ } else if (cmpxchg(&page->memcg_data, 0, memcg_data)) {
+ /*
+ * If the slab page is already in use, somebody can allocate
+ * and assign obj_cgroups in parallel. In this case the existing
+ * objcg vector should be reused.
+ */
- else
- kmemleak_not_leak(vec);
+ return 0;
+ }
+ kmemleak_not_leak(vec);
return 0;