path: root/mm/util.c
diff options
authorFeng Tang <>2020-08-06 23:23:15 -0700
committerLinus Torvalds <>2020-08-07 11:33:26 -0700
commit56f3547bfa4d361148aa748ccb86073bc57f5e6c (patch)
tree4cef4d923a82d85d56f2cb00cd7dbe40cd2c43e5 /mm/util.c
parent0a4954a850b0c4d0a5d18b1a55d6e5a653e362b5 (diff)
mm: adjust vm_committed_as_batch according to vm overcommit policy
When checking a performance change for will-it-scale scalability mmap test [1], we found very high lock contention for spinlock of percpu counter 'vm_committed_as': 94.14% 0.35% [kernel.kallsyms] [k] _raw_spin_lock_irqsave 48.21% _raw_spin_lock_irqsave;percpu_counter_add_batch;__vm_enough_memory;mmap_region;do_mmap; 45.91% _raw_spin_lock_irqsave;percpu_counter_add_batch;__do_munmap; Actually this heavy lock contention is not always necessary. The 'vm_committed_as' needs to be very precise when the strict OVERCOMMIT_NEVER policy is set, which requires a rather small batch number for the percpu counter. So keep 'batch' number unchanged for strict OVERCOMMIT_NEVER policy, and lift it to 64X for OVERCOMMIT_ALWAYS and OVERCOMMIT_GUESS policies. Also add a sysctl handler to adjust it when the policy is reconfigured. Benchmark with the same testcase in [1] shows 53% improvement on a 8C/16T desktop, and 2097%(20X) on a 4S/72C/144T server. We tested with test platforms in 0day (server, desktop and laptop), and 80%+ platforms shows improvements with that test. And whether it shows improvements depends on if the test mmap size is bigger than the batch number computed. And if the lift is 16X, 1/3 of the platforms will show improvements, though it should help the mmap/unmap usage generally, as Michal Hocko mentioned: : I believe that there are non-synthetic worklaods which would benefit from : a larger batch. E.g. large in memory databases which do large mmaps : during startups from multiple threads. [1] Signed-off-by: Feng Tang <> Signed-off-by: Andrew Morton <> Acked-by: Michal Hocko <> Cc: Matthew Wilcox (Oracle) <> Cc: Johannes Weiner <> Cc: Mel Gorman <> Cc: Qian Cai <> Cc: Kees Cook <> Cc: Andi Kleen <> Cc: Tim Chen <> Cc: Dave Hansen <> Cc: Huang Ying <> Cc: Christoph Lameter <> Cc: Dennis Zhou <> Cc: Haiyang Zhang <> Cc: kernel test robot <> Cc: "K. Y. Srinivasan" <> Cc: Tejun Heo <> Link: Link: Link: Signed-off-by: Linus Torvalds <>
Diffstat (limited to 'mm/util.c')
1 files changed, 41 insertions, 0 deletions
diff --git a/mm/util.c b/mm/util.c
index 1c9d097d7e48..8d6280c05238 100644
--- a/mm/util.c
+++ b/mm/util.c
@@ -746,6 +746,47 @@ int overcommit_ratio_handler(struct ctl_table *table, int write, void *buffer,
return ret;
+static void sync_overcommit_as(struct work_struct *dummy)
+ percpu_counter_sync(&vm_committed_as);
+int overcommit_policy_handler(struct ctl_table *table, int write, void *buffer,
+ size_t *lenp, loff_t *ppos)
+ struct ctl_table t;
+ int new_policy;
+ int ret;
+ /*
+ * The deviation of sync_overcommit_as could be big with loose policy
+ * like OVERCOMMIT_ALWAYS/OVERCOMMIT_GUESS. When changing policy to
+ * strict OVERCOMMIT_NEVER, we need to reduce the deviation to comply
+ * with the strict "NEVER", and to avoid possible race condtion (even
+ * though user usually won't too frequently do the switching to policy
+ * OVERCOMMIT_NEVER), the switch is done in the following order:
+ * 1. changing the batch
+ * 2. sync percpu count on each CPU
+ * 3. switch the policy
+ */
+ if (write) {
+ t = *table;
+ = &new_policy;
+ ret = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
+ if (ret)
+ return ret;
+ mm_compute_batch(new_policy);
+ if (new_policy == OVERCOMMIT_NEVER)
+ schedule_on_each_cpu(sync_overcommit_as);
+ sysctl_overcommit_memory = new_policy;
+ } else {
+ ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
+ }
+ return ret;
int overcommit_kbytes_handler(struct ctl_table *table, int write, void *buffer,
size_t *lenp, loff_t *ppos)