diff options
| -rw-r--r-- | include/asm-generic/vdso/vsyscall.h | 16 | ||||
| -rw-r--r-- | include/linux/time_namespace.h | 1 | ||||
| -rw-r--r-- | include/linux/vdso_datastore.h | 10 | ||||
| -rw-r--r-- | include/vdso/datapage.h | 41 | ||||
| -rw-r--r-- | kernel/time/vsyscall.c | 8 | ||||
| -rw-r--r-- | lib/Makefile | 2 | ||||
| -rw-r--r-- | lib/vdso/Kconfig | 5 | ||||
| -rw-r--r-- | lib/vdso/Makefile | 3 | ||||
| -rw-r--r-- | lib/vdso/datastore.c | 103 | ||||
| -rw-r--r-- | lib/vdso/gettimeofday.c | 25 |
10 files changed, 195 insertions, 19 deletions
diff --git a/include/asm-generic/vdso/vsyscall.h b/include/asm-generic/vdso/vsyscall.h index 01dafd604188..ac5b93b91993 100644 --- a/include/asm-generic/vdso/vsyscall.h +++ b/include/asm-generic/vdso/vsyscall.h @@ -4,12 +4,28 @@ #ifndef __ASSEMBLY__ +#ifdef CONFIG_GENERIC_VDSO_DATA_STORE + +#ifndef __arch_get_vdso_u_time_data +static __always_inline const struct vdso_time_data *__arch_get_vdso_u_time_data(void) +{ + return vdso_u_time_data; +} +#endif + +#else /* !CONFIG_GENERIC_VDSO_DATA_STORE */ + #ifndef __arch_get_k_vdso_data static __always_inline struct vdso_data *__arch_get_k_vdso_data(void) { return NULL; } #endif /* __arch_get_k_vdso_data */ +#define vdso_k_time_data __arch_get_k_vdso_data() + +#define __arch_get_vdso_u_time_data __arch_get_vdso_data + +#endif /* CONFIG_GENERIC_VDSO_DATA_STORE */ #ifndef __arch_update_vsyscall static __always_inline void __arch_update_vsyscall(struct vdso_data *vdata) diff --git a/include/linux/time_namespace.h b/include/linux/time_namespace.h index 876e31b4461d..4b81db223f54 100644 --- a/include/linux/time_namespace.h +++ b/include/linux/time_namespace.h @@ -8,6 +8,7 @@ #include <linux/ns_common.h> #include <linux/err.h> #include <linux/time64.h> +#include <vdso/datapage.h> struct user_namespace; extern struct user_namespace init_user_ns; diff --git a/include/linux/vdso_datastore.h b/include/linux/vdso_datastore.h new file mode 100644 index 000000000000..a91fa24b06e0 --- /dev/null +++ b/include/linux/vdso_datastore.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_VDSO_DATASTORE_H +#define _LINUX_VDSO_DATASTORE_H + +#include <linux/mm_types.h> + +extern const struct vm_special_mapping vdso_vvar_mapping; +struct vm_area_struct *vdso_install_vvar_mapping(struct mm_struct *mm, unsigned long addr); + +#endif /* _LINUX_VDSO_DATASTORE_H */ diff --git a/include/vdso/datapage.h b/include/vdso/datapage.h index d967baa0cd0c..b3d8087488ff 100644 --- a/include/vdso/datapage.h +++ b/include/vdso/datapage.h @@ -45,11 +45,11 @@ struct arch_vdso_time_data {}; * * There is one vdso_timestamp object in vvar for each vDSO-accelerated * clock_id. For high-resolution clocks, this encodes the time - * corresponding to vdso_data.cycle_last. For coarse clocks this encodes + * corresponding to vdso_time_data.cycle_last. For coarse clocks this encodes * the actual time. * * To be noticed that for highres clocks nsec is left-shifted by - * vdso_data.cs[x].shift. + * vdso_time_data[x].shift. */ struct vdso_timestamp { u64 sec; @@ -57,7 +57,7 @@ struct vdso_timestamp { }; /** - * struct vdso_data - vdso datapage representation + * struct vdso_time_data - vdso datapage representation * @seq: timebase sequence counter * @clock_mode: clock mode * @cycle_last: timebase at clocksource init @@ -74,7 +74,7 @@ struct vdso_timestamp { * @arch_data: architecture specific data (optional, defaults * to an empty struct) * - * vdso_data will be accessed by 64 bit and compat code at the same time + * vdso_time_data will be accessed by 64 bit and compat code at the same time * so we should be careful before modifying this structure. * * The ordering of the struct members is optimized to have fast access to the @@ -92,7 +92,7 @@ struct vdso_timestamp { * For clocks which are not affected by time namespace adjustment the * offset must be zero. */ -struct vdso_data { +struct vdso_time_data { u32 seq; s32 clock_mode; @@ -117,6 +117,8 @@ struct vdso_data { struct arch_vdso_time_data arch_data; }; +#define vdso_data vdso_time_data + /** * struct vdso_rng_data - vdso RNG state information * @generation: counter representing the number of RNG reseeds @@ -136,18 +138,34 @@ struct vdso_rng_data { * With the hidden visibility, the compiler simply generates a PC-relative * relocation, and this is what we need. */ -extern struct vdso_data _vdso_data[CS_BASES] __attribute__((visibility("hidden"))); -extern struct vdso_data _timens_data[CS_BASES] __attribute__((visibility("hidden"))); +#ifndef CONFIG_GENERIC_VDSO_DATA_STORE +extern struct vdso_time_data _vdso_data[CS_BASES] __attribute__((visibility("hidden"))); +extern struct vdso_time_data _timens_data[CS_BASES] __attribute__((visibility("hidden"))); extern struct vdso_rng_data _vdso_rng_data __attribute__((visibility("hidden"))); +#else +extern struct vdso_time_data vdso_u_time_data[CS_BASES] __attribute__((visibility("hidden"))); + +extern struct vdso_time_data *vdso_k_time_data; +#endif /** * union vdso_data_store - Generic vDSO data page */ union vdso_data_store { - struct vdso_data data[CS_BASES]; + struct vdso_time_data data[CS_BASES]; u8 page[1U << CONFIG_PAGE_SHIFT]; }; +#ifdef CONFIG_GENERIC_VDSO_DATA_STORE + +enum vdso_pages { + VDSO_TIME_PAGE_OFFSET, + VDSO_TIMENS_PAGE_OFFSET, + VDSO_NR_PAGES +}; + +#endif /* CONFIG_GENERIC_VDSO_DATA_STORE */ + /* * The generic vDSO implementation requires that gettimeofday.h * provides: @@ -164,6 +182,13 @@ union vdso_data_store { #include <asm/vdso/gettimeofday.h> #endif /* ENABLE_COMPAT_VDSO */ +#else /* !__ASSEMBLY__ */ + +#define VDSO_VVAR_SYMS \ + PROVIDE(vdso_u_data = . - __VDSO_PAGES * PAGE_SIZE); \ + PROVIDE(vdso_u_time_data = vdso_u_data); \ + + #endif /* !__ASSEMBLY__ */ #endif /* __VDSO_DATAPAGE_H */ diff --git a/kernel/time/vsyscall.c b/kernel/time/vsyscall.c index 05d383143165..09c1e39a6dd8 100644 --- a/kernel/time/vsyscall.c +++ b/kernel/time/vsyscall.c @@ -77,7 +77,7 @@ static inline void update_vdso_data(struct vdso_data *vdata, void update_vsyscall(struct timekeeper *tk) { - struct vdso_data *vdata = __arch_get_k_vdso_data(); + struct vdso_data *vdata = vdso_k_time_data; struct vdso_timestamp *vdso_ts; s32 clock_mode; u64 nsec; @@ -128,7 +128,7 @@ void update_vsyscall(struct timekeeper *tk) void update_vsyscall_tz(void) { - struct vdso_data *vdata = __arch_get_k_vdso_data(); + struct vdso_data *vdata = vdso_k_time_data; vdata[CS_HRES_COARSE].tz_minuteswest = sys_tz.tz_minuteswest; vdata[CS_HRES_COARSE].tz_dsttime = sys_tz.tz_dsttime; @@ -150,7 +150,7 @@ void update_vsyscall_tz(void) */ unsigned long vdso_update_begin(void) { - struct vdso_data *vdata = __arch_get_k_vdso_data(); + struct vdso_data *vdata = vdso_k_time_data; unsigned long flags = timekeeper_lock_irqsave(); vdso_write_begin(vdata); @@ -167,7 +167,7 @@ unsigned long vdso_update_begin(void) */ void vdso_update_end(unsigned long flags) { - struct vdso_data *vdata = __arch_get_k_vdso_data(); + struct vdso_data *vdata = vdso_k_time_data; vdso_write_end(vdata); __arch_sync_vdso_data(vdata); diff --git a/lib/Makefile b/lib/Makefile index d5cfc7afbbb8..fcdee83deb5c 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -132,7 +132,7 @@ endif obj-$(CONFIG_DEBUG_INFO_REDUCED) += debug_info.o CFLAGS_debug_info.o += $(call cc-option, -femit-struct-debug-detailed=any) -obj-y += math/ crypto/ +obj-y += math/ crypto/ vdso/ obj-$(CONFIG_GENERIC_IOMAP) += iomap.o obj-$(CONFIG_HAS_IOMEM) += iomap_copy.o devres.o diff --git a/lib/vdso/Kconfig b/lib/vdso/Kconfig index 82fe827af542..45df764b49ad 100644 --- a/lib/vdso/Kconfig +++ b/lib/vdso/Kconfig @@ -43,3 +43,8 @@ config VDSO_GETRANDOM bool help Selected by architectures that support vDSO getrandom(). + +config GENERIC_VDSO_DATA_STORE + bool + help + Selected by architectures that use the generic vDSO data store. diff --git a/lib/vdso/Makefile b/lib/vdso/Makefile new file mode 100644 index 000000000000..aedd40aaa950 --- /dev/null +++ b/lib/vdso/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only + +obj-$(CONFIG_GENERIC_VDSO_DATA_STORE) += datastore.o diff --git a/lib/vdso/datastore.c b/lib/vdso/datastore.c new file mode 100644 index 000000000000..d3181768424c --- /dev/null +++ b/lib/vdso/datastore.c @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include <linux/linkage.h> +#include <linux/mmap_lock.h> +#include <linux/mm.h> +#include <linux/time_namespace.h> +#include <linux/types.h> +#include <linux/vdso_datastore.h> +#include <vdso/datapage.h> + +/* + * The vDSO data page. + */ +#ifdef CONFIG_HAVE_GENERIC_VDSO +static union vdso_data_store vdso_time_data_store __page_aligned_data; +struct vdso_time_data *vdso_k_time_data = vdso_time_data_store.data; +static_assert(sizeof(vdso_time_data_store) == PAGE_SIZE); +#endif /* CONFIG_HAVE_GENERIC_VDSO */ + +static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, + struct vm_area_struct *vma, struct vm_fault *vmf) +{ + struct page *timens_page = find_timens_vvar_page(vma); + unsigned long addr, pfn; + vm_fault_t err; + + switch (vmf->pgoff) { + case VDSO_TIME_PAGE_OFFSET: + if (!IS_ENABLED(CONFIG_HAVE_GENERIC_VDSO)) + return VM_FAULT_SIGBUS; + pfn = __phys_to_pfn(__pa_symbol(vdso_k_time_data)); + if (timens_page) { + /* + * Fault in VVAR page too, since it will be accessed + * to get clock data anyway. + */ + addr = vmf->address + VDSO_TIMENS_PAGE_OFFSET * PAGE_SIZE; + err = vmf_insert_pfn(vma, addr, pfn); + if (unlikely(err & VM_FAULT_ERROR)) + return err; + pfn = page_to_pfn(timens_page); + } + break; + case VDSO_TIMENS_PAGE_OFFSET: + /* + * If a task belongs to a time namespace then a namespace + * specific VVAR is mapped with the VVAR_DATA_PAGE_OFFSET and + * the real VVAR page is mapped with the VVAR_TIMENS_PAGE_OFFSET + * offset. + * See also the comment near timens_setup_vdso_data(). + */ + if (!IS_ENABLED(CONFIG_TIME_NS) || !timens_page) + return VM_FAULT_SIGBUS; + pfn = __phys_to_pfn(__pa_symbol(vdso_k_time_data)); + break; + default: + return VM_FAULT_SIGBUS; + } + + return vmf_insert_pfn(vma, vmf->address, pfn); +} + +const struct vm_special_mapping vdso_vvar_mapping = { + .name = "[vvar]", + .fault = vvar_fault, +}; + +struct vm_area_struct *vdso_install_vvar_mapping(struct mm_struct *mm, unsigned long addr) +{ + return _install_special_mapping(mm, addr, VDSO_NR_PAGES * PAGE_SIZE, + VM_READ | VM_MAYREAD | VM_IO | VM_DONTDUMP | VM_PFNMAP, + &vdso_vvar_mapping); +} + +#ifdef CONFIG_TIME_NS +/* + * The vvar page layout depends on whether a task belongs to the root or + * non-root time namespace. Whenever a task changes its namespace, the VVAR + * page tables are cleared and then they will be re-faulted with a + * corresponding layout. + * See also the comment near timens_setup_vdso_data() for details. + */ +int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) +{ + struct mm_struct *mm = task->mm; + struct vm_area_struct *vma; + VMA_ITERATOR(vmi, mm, 0); + + mmap_read_lock(mm); + for_each_vma(vmi, vma) { + if (vma_is_special_mapping(vma, &vdso_vvar_mapping)) + zap_vma_pages(vma); + } + mmap_read_unlock(mm); + + return 0; +} + +struct vdso_time_data *arch_get_vdso_data(void *vvar_page) +{ + return (struct vdso_time_data *)vvar_page; +} +#endif diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c index c01eaafd8041..20c5b8752fcc 100644 --- a/lib/vdso/gettimeofday.c +++ b/lib/vdso/gettimeofday.c @@ -5,6 +5,9 @@ #include <vdso/datapage.h> #include <vdso/helpers.h> +/* Bring in default accessors */ +#include <vdso/vsyscall.h> + #ifndef vdso_calc_ns #ifdef VDSO_DELTA_NOMASK @@ -69,6 +72,16 @@ static inline bool vdso_cycles_ok(u64 cycles) #endif #ifdef CONFIG_TIME_NS + +#ifdef CONFIG_GENERIC_VDSO_DATA_STORE +static __always_inline +const struct vdso_time_data *__arch_get_vdso_u_timens_data(const struct vdso_time_data *vd) +{ + return (void *)vd + PAGE_SIZE; +} +#define __arch_get_timens_vdso_data(vd) __arch_get_vdso_u_timens_data(vd) +#endif /* CONFIG_GENERIC_VDSO_DATA_STORE */ + static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, struct __kernel_timespec *ts) { @@ -282,7 +295,7 @@ __cvdso_clock_gettime_data(const struct vdso_data *vd, clockid_t clock, static __maybe_unused int __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) { - return __cvdso_clock_gettime_data(__arch_get_vdso_data(), clock, ts); + return __cvdso_clock_gettime_data(__arch_get_vdso_u_time_data(), clock, ts); } #ifdef BUILD_VDSO32 @@ -308,7 +321,7 @@ __cvdso_clock_gettime32_data(const struct vdso_data *vd, clockid_t clock, static __maybe_unused int __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) { - return __cvdso_clock_gettime32_data(__arch_get_vdso_data(), clock, res); + return __cvdso_clock_gettime32_data(__arch_get_vdso_u_time_data(), clock, res); } #endif /* BUILD_VDSO32 */ @@ -342,7 +355,7 @@ __cvdso_gettimeofday_data(const struct vdso_data *vd, static __maybe_unused int __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) { - return __cvdso_gettimeofday_data(__arch_get_vdso_data(), tv, tz); + return __cvdso_gettimeofday_data(__arch_get_vdso_u_time_data(), tv, tz); } #ifdef VDSO_HAS_TIME @@ -365,7 +378,7 @@ __cvdso_time_data(const struct vdso_data *vd, __kernel_old_time_t *time) static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time) { - return __cvdso_time_data(__arch_get_vdso_data(), time); + return __cvdso_time_data(__arch_get_vdso_u_time_data(), time); } #endif /* VDSO_HAS_TIME */ @@ -425,7 +438,7 @@ int __cvdso_clock_getres_data(const struct vdso_data *vd, clockid_t clock, static __maybe_unused int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) { - return __cvdso_clock_getres_data(__arch_get_vdso_data(), clock, res); + return __cvdso_clock_getres_data(__arch_get_vdso_u_time_data(), clock, res); } #ifdef BUILD_VDSO32 @@ -451,7 +464,7 @@ __cvdso_clock_getres_time32_data(const struct vdso_data *vd, clockid_t clock, static __maybe_unused int __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) { - return __cvdso_clock_getres_time32_data(__arch_get_vdso_data(), + return __cvdso_clock_getres_time32_data(__arch_get_vdso_u_time_data(), clock, res); } #endif /* BUILD_VDSO32 */ |