path: root/mm/kasan
diff options
authorAndrey Konovalov <>2021-02-25 17:19:55 -0800
committerLinus Torvalds <>2021-02-26 09:41:02 -0800
commit928501344fc645f80390afc12708c81b3595745d (patch)
tree98e66961054a59acab4663ee4249eaa33f7c1535 /mm/kasan
parentd3a61f745e0d089a2484740283a434deb6dd4eb5 (diff)
kasan, mm: don't save alloc stacks twice
Patch series "kasan: optimizations and fixes for HW_TAGS", v4. This patchset makes the HW_TAGS mode more efficient, mostly by reworking poisoning approaches and simplifying/inlining some internal helpers. With this change, the overhead of HW_TAGS annotations excluding setting and checking memory tags is ~3%. The performance impact caused by tags will be unknown until we have hardware that supports MTE. As a side-effect, this patchset speeds up generic KASAN by ~15%. This patch (of 13): Currently KASAN saves allocation stacks in both kasan_slab_alloc() and kasan_kmalloc() annotations. This patch changes KASAN to save allocation stacks for slab objects from kmalloc caches in kasan_kmalloc() only, and stacks for other slab objects in kasan_slab_alloc() only. This change requires ____kasan_kmalloc() knowing whether the object belongs to a kmalloc cache. This is implemented by adding a flag field to the kasan_info structure. That flag is only set for kmalloc caches via a new kasan_cache_create_kmalloc() annotation. Link: Link: Signed-off-by: Andrey Konovalov <> Reviewed-by: Marco Elver <> Cc: Catalin Marinas <> Cc: Vincenzo Frascino <> Cc: Dmitry Vyukov <> Cc: Alexander Potapenko <> Cc: Will Deacon <> Cc: Andrey Ryabinin <> Cc: Peter Collingbourne <> Cc: Evgenii Stepanov <> Cc: Branislav Rankov <> Cc: Kevin Brodsky <> Signed-off-by: Andrew Morton <> Signed-off-by: Linus Torvalds <>
Diffstat (limited to 'mm/kasan')
1 files changed, 14 insertions, 4 deletions
diff --git a/mm/kasan/common.c b/mm/kasan/common.c
index af1768c4fee5..d8d83ca56fe2 100644
--- a/mm/kasan/common.c
+++ b/mm/kasan/common.c
@@ -210,6 +210,11 @@ void __kasan_cache_create(struct kmem_cache *cache, unsigned int *size,
*size = optimal_size;
+void __kasan_cache_create_kmalloc(struct kmem_cache *cache)
+ cache->kasan_info.is_kmalloc = true;
size_t __kasan_metadata_size(struct kmem_cache *cache)
if (!kasan_stack_collection_enabled())
@@ -394,17 +399,22 @@ void __kasan_slab_free_mempool(void *ptr, unsigned long ip)
-static void set_alloc_info(struct kmem_cache *cache, void *object, gfp_t flags)
+static void set_alloc_info(struct kmem_cache *cache, void *object,
+ gfp_t flags, bool is_kmalloc)
struct kasan_alloc_meta *alloc_meta;
+ /* Don't save alloc info for kmalloc caches in kasan_slab_alloc(). */
+ if (cache->kasan_info.is_kmalloc && !is_kmalloc)
+ return;
alloc_meta = kasan_get_alloc_meta(cache, object);
if (alloc_meta)
kasan_set_track(&alloc_meta->alloc_track, flags);
static void *____kasan_kmalloc(struct kmem_cache *cache, const void *object,
- size_t size, gfp_t flags, bool keep_tag)
+ size_t size, gfp_t flags, bool is_kmalloc)
unsigned long redzone_start;
unsigned long redzone_end;
@@ -423,7 +433,7 @@ static void *____kasan_kmalloc(struct kmem_cache *cache, const void *object,
redzone_end = round_up((unsigned long)object + cache->object_size,
- tag = assign_tag(cache, object, false, keep_tag);
+ tag = assign_tag(cache, object, false, is_kmalloc);
/* Tag is ignored in set_tag without CONFIG_KASAN_SW/HW_TAGS */
kasan_unpoison(set_tag(object, tag), size);
@@ -431,7 +441,7 @@ static void *____kasan_kmalloc(struct kmem_cache *cache, const void *object,
if (kasan_stack_collection_enabled())
- set_alloc_info(cache, (void *)object, flags);
+ set_alloc_info(cache, (void *)object, flags, is_kmalloc);
return set_tag(object, tag);