diff options
Diffstat (limited to 'rust')
60 files changed, 4555 insertions, 10689 deletions
diff --git a/rust/.gitignore b/rust/.gitignore index 21552992b401..d3829ffab80b 100644 --- a/rust/.gitignore +++ b/rust/.gitignore @@ -2,6 +2,8 @@ bindings_generated.rs bindings_helpers_generated.rs +doctests_kernel_generated.rs +doctests_kernel_generated_kunit.c uapi_generated.rs exports_*_generated.h doc/ diff --git a/rust/Makefile b/rust/Makefile index 4124bfa01798..f70d5e244fee 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -1,5 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 +# Where to place rustdoc generated documentation +rustdoc_output := $(objtree)/Documentation/output/rust/rustdoc + obj-$(CONFIG_RUST) += core.o compiler_builtins.o always-$(CONFIG_RUST) += exports_core_generated.h @@ -27,11 +30,17 @@ endif obj-$(CONFIG_RUST) += exports.o +always-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated.rs +always-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated_kunit.c + +obj-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated.o +obj-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated_kunit.o + # Avoids running `$(RUSTC)` for the sysroot when it may not be available. ifdef CONFIG_RUST # `$(rust_flags)` is passed in case the user added `--sysroot`. -rustc_sysroot := $(shell $(RUSTC) $(rust_flags) --print sysroot) +rustc_sysroot := $(shell MAKEFLAGS= $(RUSTC) $(rust_flags) --print sysroot) rustc_host_target := $(shell $(RUSTC) --version --verbose | grep -F 'host: ' | cut -d' ' -f2) RUST_LIB_SRC ?= $(rustc_sysroot)/lib/rustlib/src/rust/library @@ -39,9 +48,11 @@ ifeq ($(quiet),silent_) cargo_quiet=-q rust_test_quiet=-q rustdoc_test_quiet=--test-args -q +rustdoc_test_kernel_quiet=>/dev/null else ifeq ($(quiet),quiet_) rust_test_quiet=-q rustdoc_test_quiet=--test-args -q +rustdoc_test_kernel_quiet=>/dev/null else cargo_quiet=--verbose endif @@ -50,23 +61,18 @@ core-cfgs = \ --cfg no_fp_fmt_parse alloc-cfgs = \ - --cfg no_borrow \ - --cfg no_fmt \ --cfg no_global_oom_handling \ - --cfg no_macros \ --cfg no_rc \ - --cfg no_str \ - --cfg no_string \ - --cfg no_sync \ - --cfg no_thin + --cfg no_sync quiet_cmd_rustdoc = RUSTDOC $(if $(rustdoc_host),H, ) $< cmd_rustdoc = \ OBJTREE=$(abspath $(objtree)) \ $(RUSTDOC) $(if $(rustdoc_host),$(rust_common_flags),$(rust_flags)) \ $(rustc_target_flags) -L$(objtree)/$(obj) \ - --output $(objtree)/$(obj)/doc \ + --output $(rustdoc_output) \ --crate-name $(subst rustdoc-,,$@) \ + $(if $(rustdoc_host),,--sysroot=/dev/null) \ @$(objtree)/include/generated/rustc_cfg $< # The `html_logo_url` and `html_favicon_url` forms of the `doc` attribute @@ -82,28 +88,28 @@ quiet_cmd_rustdoc = RUSTDOC $(if $(rustdoc_host),H, ) $< # and then retouch the generated files. rustdoc: rustdoc-core rustdoc-macros rustdoc-compiler_builtins \ rustdoc-alloc rustdoc-kernel - $(Q)cp $(srctree)/Documentation/images/logo.svg $(objtree)/$(obj)/doc - $(Q)cp $(srctree)/Documentation/images/COPYING-logo $(objtree)/$(obj)/doc - $(Q)find $(objtree)/$(obj)/doc -name '*.html' -type f -print0 | xargs -0 sed -Ei \ - -e 's:rust-logo\.svg:logo.svg:g' \ - -e 's:rust-logo\.png:logo.svg:g' \ - -e 's:favicon\.svg:logo.svg:g' \ - -e 's:<link rel="alternate icon" type="image/png" href="[./]*favicon-(16x16|32x32)\.png">::g' - $(Q)echo '.logo-container > img { object-fit: contain; }' \ - >> $(objtree)/$(obj)/doc/rustdoc.css + $(Q)cp $(srctree)/Documentation/images/logo.svg $(rustdoc_output)/static.files/ + $(Q)cp $(srctree)/Documentation/images/COPYING-logo $(rustdoc_output)/static.files/ + $(Q)find $(rustdoc_output) -name '*.html' -type f -print0 | xargs -0 sed -Ei \ + -e 's:rust-logo-[0-9a-f]+\.svg:logo.svg:g' \ + -e 's:favicon-[0-9a-f]+\.svg:logo.svg:g' \ + -e 's:<link rel="alternate icon" type="image/png" href="[/.]+/static\.files/favicon-(16x16|32x32)-[0-9a-f]+\.png">::g' \ + -e 's:<a href="srctree/([^"]+)">:<a href="$(realpath $(srctree))/\1">:g' + $(Q)for f in $(rustdoc_output)/static.files/rustdoc-*.css; do \ + echo ".logo-container > img { object-fit: contain; }" >> $$f; done rustdoc-macros: private rustdoc_host = yes rustdoc-macros: private rustc_target_flags = --crate-type proc-macro \ --extern proc_macro rustdoc-macros: $(src)/macros/lib.rs FORCE - $(call if_changed,rustdoc) + +$(call if_changed,rustdoc) rustdoc-core: private rustc_target_flags = $(core-cfgs) rustdoc-core: $(RUST_LIB_SRC)/core/src/lib.rs FORCE - $(call if_changed,rustdoc) + +$(call if_changed,rustdoc) rustdoc-compiler_builtins: $(src)/compiler_builtins.rs rustdoc-core FORCE - $(call if_changed,rustdoc) + +$(call if_changed,rustdoc) # We need to allow `rustdoc::broken_intra_doc_links` because some # `no_global_oom_handling` functions refer to non-`no_global_oom_handling` @@ -111,8 +117,8 @@ rustdoc-compiler_builtins: $(src)/compiler_builtins.rs rustdoc-core FORCE # due to things that are "configured out" vs. entirely non-existing ones. rustdoc-alloc: private rustc_target_flags = $(alloc-cfgs) \ -Arustdoc::broken_intra_doc_links -rustdoc-alloc: $(src)/alloc/lib.rs rustdoc-core rustdoc-compiler_builtins FORCE - $(call if_changed,rustdoc) +rustdoc-alloc: $(RUST_LIB_SRC)/alloc/src/lib.rs rustdoc-core rustdoc-compiler_builtins FORCE + +$(call if_changed,rustdoc) rustdoc-kernel: private rustc_target_flags = --extern alloc \ --extern build_error --extern macros=$(objtree)/$(obj)/libmacros.so \ @@ -120,7 +126,7 @@ rustdoc-kernel: private rustc_target_flags = --extern alloc \ rustdoc-kernel: $(src)/kernel/lib.rs rustdoc-core rustdoc-macros \ rustdoc-compiler_builtins rustdoc-alloc $(obj)/libmacros.so \ $(obj)/bindings.o FORCE - $(call if_changed,rustdoc) + +$(call if_changed,rustdoc) quiet_cmd_rustc_test_library = RUSTC TL $< cmd_rustc_test_library = \ @@ -134,18 +140,18 @@ quiet_cmd_rustc_test_library = RUSTC TL $< --crate-name $(subst rusttest-,,$(subst rusttestlib-,,$@)) $< rusttestlib-build_error: $(src)/build_error.rs rusttest-prepare FORCE - $(call if_changed,rustc_test_library) + +$(call if_changed,rustc_test_library) rusttestlib-macros: private rustc_target_flags = --extern proc_macro rusttestlib-macros: private rustc_test_library_proc = yes rusttestlib-macros: $(src)/macros/lib.rs rusttest-prepare FORCE - $(call if_changed,rustc_test_library) + +$(call if_changed,rustc_test_library) rusttestlib-bindings: $(src)/bindings/lib.rs rusttest-prepare FORCE - $(call if_changed,rustc_test_library) + +$(call if_changed,rustc_test_library) rusttestlib-uapi: $(src)/uapi/lib.rs rusttest-prepare FORCE - $(call if_changed,rustc_test_library) + +$(call if_changed,rustc_test_library) quiet_cmd_rustdoc_test = RUSTDOC T $< cmd_rustdoc_test = \ @@ -154,9 +160,30 @@ quiet_cmd_rustdoc_test = RUSTDOC T $< @$(objtree)/include/generated/rustc_cfg \ $(rustc_target_flags) $(rustdoc_test_target_flags) \ --sysroot $(objtree)/$(obj)/test/sysroot $(rustdoc_test_quiet) \ - -L$(objtree)/$(obj)/test --output $(objtree)/$(obj)/doc \ + -L$(objtree)/$(obj)/test --output $(rustdoc_output) \ --crate-name $(subst rusttest-,,$@) $< +quiet_cmd_rustdoc_test_kernel = RUSTDOC TK $< + cmd_rustdoc_test_kernel = \ + rm -rf $(objtree)/$(obj)/test/doctests/kernel; \ + mkdir -p $(objtree)/$(obj)/test/doctests/kernel; \ + OBJTREE=$(abspath $(objtree)) \ + $(RUSTDOC) --test $(rust_flags) \ + -L$(objtree)/$(obj) --extern alloc --extern kernel \ + --extern build_error --extern macros \ + --extern bindings --extern uapi \ + --no-run --crate-name kernel -Zunstable-options \ + --sysroot=/dev/null \ + --test-builder $(objtree)/scripts/rustdoc_test_builder \ + $< $(rustdoc_test_kernel_quiet); \ + $(objtree)/scripts/rustdoc_test_gen + +%/doctests_kernel_generated.rs %/doctests_kernel_generated_kunit.c: \ + $(src)/kernel/lib.rs $(obj)/kernel.o \ + $(objtree)/scripts/rustdoc_test_builder \ + $(objtree)/scripts/rustdoc_test_gen FORCE + +$(call if_changed,rustdoc_test_kernel) + # We cannot use `-Zpanic-abort-tests` because some tests are dynamic, # so for the moment we skip `-Cpanic=abort`. quiet_cmd_rustc_test = RUSTC T $< @@ -185,8 +212,6 @@ rusttest: rusttest-macros rusttest-kernel # - `cargo` only considers the use case of building the standard library # to use it in a given package. Thus we need to create a dummy package # and pick the generated libraries from there. -# - Since we only keep a subset of upstream `alloc` in-tree, we need -# to recreate it on the fly by putting our sources on top. # - The usual ways of modifying the dependency graph in `cargo` do not seem # to apply for the `-Zbuild-std` steps, thus we have to mislead it # by modifying the sources in the sysroot. @@ -205,8 +230,6 @@ quiet_cmd_rustsysroot = RUSTSYSROOT rm -rf $(objtree)/$(obj)/test; \ mkdir -p $(objtree)/$(obj)/test; \ cp -a $(rustc_sysroot) $(objtree)/$(obj)/test/sysroot; \ - cp -r $(srctree)/$(src)/alloc/* \ - $(objtree)/$(obj)/test/sysroot/lib/rustlib/src/rust/library/alloc/src; \ echo '\#!/bin/sh' > $(objtree)/$(obj)/test/rustc_sysroot; \ echo "$(RUSTC) --sysroot=$(abspath $(objtree)/$(obj)/test/sysroot) \"\$$@\"" \ >> $(objtree)/$(obj)/test/rustc_sysroot; \ @@ -220,21 +243,21 @@ quiet_cmd_rustsysroot = RUSTSYSROOT $(objtree)/$(obj)/test/sysroot/lib/rustlib/$(rustc_host_target)/lib rusttest-prepare: FORCE - $(call if_changed,rustsysroot) + +$(call if_changed,rustsysroot) rusttest-macros: private rustc_target_flags = --extern proc_macro rusttest-macros: private rustdoc_test_target_flags = --crate-type proc-macro rusttest-macros: $(src)/macros/lib.rs rusttest-prepare FORCE - $(call if_changed,rustc_test) - $(call if_changed,rustdoc_test) + +$(call if_changed,rustc_test) + +$(call if_changed,rustdoc_test) rusttest-kernel: private rustc_target_flags = --extern alloc \ --extern build_error --extern macros --extern bindings --extern uapi rusttest-kernel: $(src)/kernel/lib.rs rusttest-prepare \ rusttestlib-build_error rusttestlib-macros rusttestlib-bindings \ rusttestlib-uapi FORCE - $(call if_changed,rustc_test) - $(call if_changed,rustc_test_library) + +$(call if_changed,rustc_test) + +$(call if_changed,rustc_test_library) ifdef CONFIG_CC_IS_CLANG bindgen_c_flags = $(c_flags) @@ -258,10 +281,12 @@ bindgen_skip_c_flags := -mno-fp-ret-in-387 -mpreferred-stack-boundary=% \ -fno-reorder-blocks -fno-allow-store-data-races -fasan-shadow-offset=% \ -fzero-call-used-regs=% -fno-stack-clash-protection \ -fno-inline-functions-called-once -fsanitize=bounds-strict \ + -fstrict-flex-arrays=% \ --param=% --param asan-% # Derived from `scripts/Makefile.clang`. BINDGEN_TARGET_x86 := x86_64-linux-gnu +BINDGEN_TARGET_arm64 := aarch64-linux-gnu BINDGEN_TARGET := $(BINDGEN_TARGET_$(SRCARCH)) # All warnings are inhibited since GCC builds are very experimental, @@ -300,17 +325,19 @@ quiet_cmd_bindgen = BINDGEN $@ $(BINDGEN) $< $(bindgen_target_flags) \ --use-core --with-derive-default --ctypes-prefix core::ffi --no-layout-tests \ --no-debug '.*' \ - --size_t-is-usize -o $@ -- $(bindgen_c_flags_final) -DMODULE \ + -o $@ -- $(bindgen_c_flags_final) -DMODULE \ $(bindgen_target_cflags) $(bindgen_target_extra) $(obj)/bindings/bindings_generated.rs: private bindgen_target_flags = \ - $(shell grep -v '^#\|^$$' $(srctree)/$(src)/bindgen_parameters) + $(shell grep -Ev '^#|^$$' $(src)/bindgen_parameters) +$(obj)/bindings/bindings_generated.rs: private bindgen_target_extra = ; \ + sed -Ei 's/pub const RUST_CONST_HELPER_([a-zA-Z0-9_]*)/pub const \1/g' $@ $(obj)/bindings/bindings_generated.rs: $(src)/bindings/bindings_helper.h \ $(src)/bindgen_parameters FORCE $(call if_changed_dep,bindgen) $(obj)/uapi/uapi_generated.rs: private bindgen_target_flags = \ - $(shell grep -v '^#\|^$$' $(srctree)/$(src)/bindgen_parameters) + $(shell grep -Ev '^#|^$$' $(src)/bindgen_parameters) $(obj)/uapi/uapi_generated.rs: $(src)/uapi/uapi_helper.h \ $(src)/bindgen_parameters FORCE $(call if_changed_dep,bindgen) @@ -320,8 +347,8 @@ $(obj)/uapi/uapi_generated.rs: $(src)/uapi/uapi_helper.h \ # given it is `libclang`; but for consistency, future Clang changes and/or # a potential future GCC backend for `bindgen`, we disable it too. $(obj)/bindings/bindings_helpers_generated.rs: private bindgen_target_flags = \ - --blacklist-type '.*' --whitelist-var '' \ - --whitelist-function 'rust_helper_.*' + --blocklist-type '.*' --allowlist-var '' \ + --allowlist-function 'rust_helper_.*' $(obj)/bindings/bindings_helpers_generated.rs: private bindgen_target_cflags = \ -I$(objtree)/$(obj) -Wno-missing-prototypes -Wno-missing-declarations $(obj)/bindings/bindings_helpers_generated.rs: private bindgen_target_extra = ; \ @@ -332,9 +359,7 @@ $(obj)/bindings/bindings_helpers_generated.rs: $(src)/helpers.c FORCE quiet_cmd_exports = EXPORTS $@ cmd_exports = \ $(NM) -p --defined-only $< \ - | grep -E ' (T|R|D) ' | cut -d ' ' -f 3 \ - | xargs -Isymbol \ - echo 'EXPORT_SYMBOL_RUST_GPL(symbol);' > $@ + | awk '/ (T|R|D) / {printf "EXPORT_SYMBOL_RUST_GPL(%s);\n",$$3}' > $@ $(obj)/exports_core_generated.h: $(obj)/core.o FORCE $(call if_changed,exports) @@ -351,6 +376,8 @@ $(obj)/exports_kernel_generated.h: $(obj)/kernel.o FORCE quiet_cmd_rustc_procmacro = $(RUSTC_OR_CLIPPY_QUIET) P $@ cmd_rustc_procmacro = \ $(RUSTC_OR_CLIPPY) $(rust_common_flags) \ + -Clinker-flavor=gcc -Clinker=$(HOSTCC) \ + -Clink-args='$(call escsq,$(KBUILD_HOSTLDFLAGS))' \ --emit=dep-info=$(depfile) --emit=link=$@ --extern proc_macro \ --crate-type proc-macro \ --crate-name $(patsubst lib%.so,%,$(notdir $@)) $< @@ -359,7 +386,7 @@ quiet_cmd_rustc_procmacro = $(RUSTC_OR_CLIPPY_QUIET) P $@ # Therefore, to get `libmacros.so` automatically recompiled when the compiler # version changes, we add `core.o` as a dependency (even if it is not needed). $(obj)/libmacros.so: $(src)/macros/lib.rs $(obj)/core.o FORCE - $(call if_changed_dep,rustc_procmacro) + +$(call if_changed_dep,rustc_procmacro) quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L $@ cmd_rustc_library = \ @@ -370,15 +397,19 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L --emit=metadata=$(dir $@)$(patsubst %.o,lib%.rmeta,$(notdir $@)) \ --crate-type rlib -L$(objtree)/$(obj) \ --crate-name $(patsubst %.o,%,$(notdir $@)) $< \ + --sysroot=/dev/null \ $(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@) rust-analyzer: - $(Q)$(srctree)/scripts/generate_rust_analyzer.py $(srctree) $(objtree) \ - $(RUST_LIB_SRC) > $(objtree)/rust-project.json + $(Q)$(srctree)/scripts/generate_rust_analyzer.py \ + --cfgs='core=$(core-cfgs)' --cfgs='alloc=$(alloc-cfgs)' \ + $(realpath $(srctree)) $(realpath $(objtree)) \ + $(RUST_LIB_SRC) $(KBUILD_EXTMOD) > \ + $(if $(KBUILD_EXTMOD),$(extmod_prefix),$(objtree))/rust-project.json redirect-intrinsics = \ - __eqsf2 __gesf2 __lesf2 __nesf2 __unordsf2 \ - __unorddf2 \ + __addsf3 __eqsf2 __gesf2 __lesf2 __ltsf2 __mulsf3 __nesf2 __unordsf2 \ + __adddf3 __ledf2 __ltdf2 __muldf3 __unorddf2 \ __muloti4 __multi3 \ __udivmodti4 __udivti3 __umodti3 @@ -393,37 +424,40 @@ $(obj)/core.o: private skip_clippy = 1 $(obj)/core.o: private skip_flags = -Dunreachable_pub $(obj)/core.o: private rustc_objcopy = $(foreach sym,$(redirect-intrinsics),--redefine-sym $(sym)=__rust$(sym)) $(obj)/core.o: private rustc_target_flags = $(core-cfgs) -$(obj)/core.o: $(RUST_LIB_SRC)/core/src/lib.rs scripts/target.json FORCE - $(call if_changed_dep,rustc_library) +$(obj)/core.o: $(RUST_LIB_SRC)/core/src/lib.rs FORCE + +$(call if_changed_dep,rustc_library) +ifdef CONFIG_X86_64 +$(obj)/core.o: scripts/target.json +endif $(obj)/compiler_builtins.o: private rustc_objcopy = -w -W '__*' $(obj)/compiler_builtins.o: $(src)/compiler_builtins.rs $(obj)/core.o FORCE - $(call if_changed_dep,rustc_library) + +$(call if_changed_dep,rustc_library) $(obj)/alloc.o: private skip_clippy = 1 $(obj)/alloc.o: private skip_flags = -Dunreachable_pub $(obj)/alloc.o: private rustc_target_flags = $(alloc-cfgs) -$(obj)/alloc.o: $(src)/alloc/lib.rs $(obj)/compiler_builtins.o FORCE - $(call if_changed_dep,rustc_library) +$(obj)/alloc.o: $(RUST_LIB_SRC)/alloc/src/lib.rs $(obj)/compiler_builtins.o FORCE + +$(call if_changed_dep,rustc_library) $(obj)/build_error.o: $(src)/build_error.rs $(obj)/compiler_builtins.o FORCE - $(call if_changed_dep,rustc_library) + +$(call if_changed_dep,rustc_library) $(obj)/bindings.o: $(src)/bindings/lib.rs \ $(obj)/compiler_builtins.o \ $(obj)/bindings/bindings_generated.rs \ $(obj)/bindings/bindings_helpers_generated.rs FORCE - $(call if_changed_dep,rustc_library) + +$(call if_changed_dep,rustc_library) $(obj)/uapi.o: $(src)/uapi/lib.rs \ $(obj)/compiler_builtins.o \ $(obj)/uapi/uapi_generated.rs FORCE - $(call if_changed_dep,rustc_library) + +$(call if_changed_dep,rustc_library) $(obj)/kernel.o: private rustc_target_flags = --extern alloc \ --extern build_error --extern macros --extern bindings --extern uapi $(obj)/kernel.o: $(src)/kernel/lib.rs $(obj)/alloc.o $(obj)/build_error.o \ $(obj)/libmacros.so $(obj)/bindings.o $(obj)/uapi.o FORCE - $(call if_changed_dep,rustc_library) + +$(call if_changed_dep,rustc_library) endif # CONFIG_RUST diff --git a/rust/alloc/README.md b/rust/alloc/README.md deleted file mode 100644 index eb6f22e94ebf..000000000000 --- a/rust/alloc/README.md +++ /dev/null @@ -1,36 +0,0 @@ -# `alloc` - -These source files come from the Rust standard library, hosted in -the <https://github.com/rust-lang/rust> repository, licensed under -"Apache-2.0 OR MIT" and adapted for kernel use. For copyright details, -see <https://github.com/rust-lang/rust/blob/master/COPYRIGHT>. - -Please note that these files should be kept as close as possible to -upstream. In general, only additions should be performed (e.g. new -methods). Eventually, changes should make it into upstream so that, -at some point, this fork can be dropped from the kernel tree. - -The Rust upstream version on top of which these files are based matches -the output of `scripts/min-tool-version.sh rustc`. - - -## Rationale - -On one hand, kernel folks wanted to keep `alloc` in-tree to have more -freedom in both workflow and actual features if actually needed -(e.g. receiver types if we ended up using them), which is reasonable. - -On the other hand, Rust folks wanted to keep `alloc` as close as -upstream as possible and avoid as much divergence as possible, which -is also reasonable. - -We agreed on a middle-ground: we would keep a subset of `alloc` -in-tree that would be as small and as close as possible to upstream. -Then, upstream can start adding the functions that we add to `alloc` -etc., until we reach a point where the kernel already knows exactly -what it needs in `alloc` and all the new methods are merged into -upstream, so that we can drop `alloc` from the kernel tree and go back -to using the upstream one. - -By doing this, the kernel can go a bit faster now, and Rust can -slowly incorporate and discuss the changes as needed. diff --git a/rust/alloc/alloc.rs b/rust/alloc/alloc.rs deleted file mode 100644 index acf22d45e6f2..000000000000 --- a/rust/alloc/alloc.rs +++ /dev/null @@ -1,445 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 OR MIT - -//! Memory allocation APIs - -#![stable(feature = "alloc_module", since = "1.28.0")] - -#[cfg(not(test))] -use core::intrinsics; -use core::intrinsics::{min_align_of_val, size_of_val}; - -use core::ptr::Unique; -#[cfg(not(test))] -use core::ptr::{self, NonNull}; - -#[stable(feature = "alloc_module", since = "1.28.0")] -#[doc(inline)] -pub use core::alloc::*; - -use core::marker::Destruct; - -#[cfg(test)] -mod tests; - -extern "Rust" { - // These are the magic symbols to call the global allocator. rustc generates - // them to call `__rg_alloc` etc. if there is a `#[global_allocator]` attribute - // (the code expanding that attribute macro generates those functions), or to call - // the default implementations in std (`__rdl_alloc` etc. in `library/std/src/alloc.rs`) - // otherwise. - // The rustc fork of LLVM 14 and earlier also special-cases these function names to be able to optimize them - // like `malloc`, `realloc`, and `free`, respectively. - #[rustc_allocator] - #[rustc_nounwind] - fn __rust_alloc(size: usize, align: usize) -> *mut u8; - #[rustc_deallocator] - #[rustc_nounwind] - fn __rust_dealloc(ptr: *mut u8, size: usize, align: usize); - #[rustc_reallocator] - #[rustc_nounwind] - fn __rust_realloc(ptr: *mut u8, old_size: usize, align: usize, new_size: usize) -> *mut u8; - #[rustc_allocator_zeroed] - #[rustc_nounwind] - fn __rust_alloc_zeroed(size: usize, align: usize) -> *mut u8; -} - -/// The global memory allocator. -/// -/// This type implements the [`Allocator`] trait by forwarding calls -/// to the allocator registered with the `#[global_allocator]` attribute -/// if there is one, or the `std` crate’s default. -/// -/// Note: while this type is unstable, the functionality it provides can be -/// accessed through the [free functions in `alloc`](self#functions). -#[unstable(feature = "allocator_api", issue = "32838")] -#[derive(Copy, Clone, Default, Debug)] -#[cfg(not(test))] -pub struct Global; - -#[cfg(test)] -pub use std::alloc::Global; - -/// Allocate memory with the global allocator. -/// -/// This function forwards calls to the [`GlobalAlloc::alloc`] method -/// of the allocator registered with the `#[global_allocator]` attribute -/// if there is one, or the `std` crate’s default. -/// -/// This function is expected to be deprecated in favor of the `alloc` method -/// of the [`Global`] type when it and the [`Allocator`] trait become stable. -/// -/// # Safety -/// -/// See [`GlobalAlloc::alloc`]. -/// -/// # Examples -/// -/// ``` -/// use std::alloc::{alloc, dealloc, handle_alloc_error, Layout}; -/// -/// unsafe { -/// let layout = Layout::new::<u16>(); -/// let ptr = alloc(layout); -/// if ptr.is_null() { -/// handle_alloc_error(layout); -/// } -/// -/// *(ptr as *mut u16) = 42; -/// assert_eq!(*(ptr as *mut u16), 42); -/// -/// dealloc(ptr, layout); -/// } -/// ``` -#[stable(feature = "global_alloc", since = "1.28.0")] -#[must_use = "losing the pointer will leak memory"] -#[inline] -pub unsafe fn alloc(layout: Layout) -> *mut u8 { - unsafe { __rust_alloc(layout.size(), layout.align()) } -} - -/// Deallocate memory with the global allocator. -/// -/// This function forwards calls to the [`GlobalAlloc::dealloc`] method -/// of the allocator registered with the `#[global_allocator]` attribute -/// if there is one, or the `std` crate’s default. -/// -/// This function is expected to be deprecated in favor of the `dealloc` method -/// of the [`Global`] type when it and the [`Allocator`] trait become stable. -/// -/// # Safety -/// -/// See [`GlobalAlloc::dealloc`]. -#[stable(feature = "global_alloc", since = "1.28.0")] -#[inline] -pub unsafe fn dealloc(ptr: *mut u8, layout: Layout) { - unsafe { __rust_dealloc(ptr, layout.size(), layout.align()) } -} - -/// Reallocate memory with the global allocator. -/// -/// This function forwards calls to the [`GlobalAlloc::realloc`] method -/// of the allocator registered with the `#[global_allocator]` attribute -/// if there is one, or the `std` crate’s default. -/// -/// This function is expected to be deprecated in favor of the `realloc` method -/// of the [`Global`] type when it and the [`Allocator`] trait become stable. -/// -/// # Safety -/// -/// See [`GlobalAlloc::realloc`]. -#[stable(feature = "global_alloc", since = "1.28.0")] -#[must_use = "losing the pointer will leak memory"] -#[inline] -pub unsafe fn realloc(ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { - unsafe { __rust_realloc(ptr, layout.size(), layout.align(), new_size) } -} - -/// Allocate zero-initialized memory with the global allocator. -/// -/// This function forwards calls to the [`GlobalAlloc::alloc_zeroed`] method -/// of the allocator registered with the `#[global_allocator]` attribute -/// if there is one, or the `std` crate’s default. -/// -/// This function is expected to be deprecated in favor of the `alloc_zeroed` method -/// of the [`Global`] type when it and the [`Allocator`] trait become stable. -/// -/// # Safety -/// -/// See [`GlobalAlloc::alloc_zeroed`]. -/// -/// # Examples -/// -/// ``` -/// use std::alloc::{alloc_zeroed, dealloc, Layout}; -/// -/// unsafe { -/// let layout = Layout::new::<u16>(); -/// let ptr = alloc_zeroed(layout); -/// -/// assert_eq!(*(ptr as *mut u16), 0); -/// -/// dealloc(ptr, layout); -/// } -/// ``` -#[stable(feature = "global_alloc", since = "1.28.0")] -#[must_use = "losing the pointer will leak memory"] -#[inline] -pub unsafe fn alloc_zeroed(layout: Layout) -> *mut u8 { - unsafe { __rust_alloc_zeroed(layout.size(), layout.align()) } -} - -#[cfg(not(test))] -impl Global { - #[inline] - fn alloc_impl(&self, layout: Layout, zeroed: bool) -> Result<NonNull<[u8]>, AllocError> { - match layout.size() { - 0 => Ok(NonNull::slice_from_raw_parts(layout.dangling(), 0)), - // SAFETY: `layout` is non-zero in size, - size => unsafe { - let raw_ptr = if zeroed { alloc_zeroed(layout) } else { alloc(layout) }; - let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?; - Ok(NonNull::slice_from_raw_parts(ptr, size)) - }, - } - } - - // SAFETY: Same as `Allocator::grow` - #[inline] - unsafe fn grow_impl( - &self, - ptr: NonNull<u8>, - old_layout: Layout, - new_layout: Layout, - zeroed: bool, - ) -> Result<NonNull<[u8]>, AllocError> { - debug_assert!( - new_layout.size() >= old_layout.size(), - "`new_layout.size()` must be greater than or equal to `old_layout.size()`" - ); - - match old_layout.size() { - 0 => self.alloc_impl(new_layout, zeroed), - - // SAFETY: `new_size` is non-zero as `old_size` is greater than or equal to `new_size` - // as required by safety conditions. Other conditions must be upheld by the caller - old_size if old_layout.align() == new_layout.align() => unsafe { - let new_size = new_layout.size(); - - // `realloc` probably checks for `new_size >= old_layout.size()` or something similar. - intrinsics::assume(new_size >= old_layout.size()); - - let raw_ptr = realloc(ptr.as_ptr(), old_layout, new_size); - let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?; - if zeroed { - raw_ptr.add(old_size).write_bytes(0, new_size - old_size); - } - Ok(NonNull::slice_from_raw_parts(ptr, new_size)) - }, - - // SAFETY: because `new_layout.size()` must be greater than or equal to `old_size`, - // both the old and new memory allocation are valid for reads and writes for `old_size` - // bytes. Also, because the old allocation wasn't yet deallocated, it cannot overlap - // `new_ptr`. Thus, the call to `copy_nonoverlapping` is safe. The safety contract - // for `dealloc` must be upheld by the caller. - old_size => unsafe { - let new_ptr = self.alloc_impl(new_layout, zeroed)?; - ptr::copy_nonoverlapping(ptr.as_ptr(), new_ptr.as_mut_ptr(), old_size); - self.deallocate(ptr, old_layout); - Ok(new_ptr) - }, - } - } -} - -#[unstable(feature = "allocator_api", issue = "32838")] -#[cfg(not(test))] -unsafe impl Allocator for Global { - #[inline] - fn allocate(&self, layout: Layout) -> Result<NonNull<[u8]>, AllocError> { - self.alloc_impl(layout, false) - } - - #[inline] - fn allocate_zeroed(&self, layout: Layout) -> Result<NonNull<[u8]>, AllocError> { - self.alloc_impl(layout, true) - } - - #[inline] - unsafe fn deallocate(&self, ptr: NonNull<u8>, layout: Layout) { - if layout.size() != 0 { - // SAFETY: `layout` is non-zero in size, - // other conditions must be upheld by the caller - unsafe { dealloc(ptr.as_ptr(), layout) } - } - } - - #[inline] - unsafe fn grow( - &self, - ptr: NonNull<u8>, - old_layout: Layout, - new_layout: Layout, - ) -> Result<NonNull<[u8]>, AllocError> { - // SAFETY: all conditions must be upheld by the caller - unsafe { self.grow_impl(ptr, old_layout, new_layout, false) } - } - - #[inline] - unsafe fn grow_zeroed( - &self, - ptr: NonNull<u8>, - old_layout: Layout, - new_layout: Layout, - ) -> Result<NonNull<[u8]>, AllocError> { - // SAFETY: all conditions must be upheld by the caller - unsafe { self.grow_impl(ptr, old_layout, new_layout, true) } - } - - #[inline] - unsafe fn shrink( - &self, - ptr: NonNull<u8>, - old_layout: Layout, - new_layout: Layout, - ) -> Result<NonNull<[u8]>, AllocError> { - debug_assert!( - new_layout.size() <= old_layout.size(), - "`new_layout.size()` must be smaller than or equal to `old_layout.size()`" - ); - - match new_layout.size() { - // SAFETY: conditions must be upheld by the caller - 0 => unsafe { - self.deallocate(ptr, old_layout); - Ok(NonNull::slice_from_raw_parts(new_layout.dangling(), 0)) - }, - - // SAFETY: `new_size` is non-zero. Other conditions must be upheld by the caller - new_size if old_layout.align() == new_layout.align() => unsafe { - // `realloc` probably checks for `new_size <= old_layout.size()` or something similar. - intrinsics::assume(new_size <= old_layout.size()); - - let raw_ptr = realloc(ptr.as_ptr(), old_layout, new_size); - let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?; - Ok(NonNull::slice_from_raw_parts(ptr, new_size)) - }, - - // SAFETY: because `new_size` must be smaller than or equal to `old_layout.size()`, - // both the old and new memory allocation are valid for reads and writes for `new_size` - // bytes. Also, because the old allocation wasn't yet deallocated, it cannot overlap - // `new_ptr`. Thus, the call to `copy_nonoverlapping` is safe. The safety contract - // for `dealloc` must be upheld by the caller. - new_size => unsafe { - let new_ptr = self.allocate(new_layout)?; - ptr::copy_nonoverlapping(ptr.as_ptr(), new_ptr.as_mut_ptr(), new_size); - self.deallocate(ptr, old_layout); - Ok(new_ptr) - }, - } - } -} - -/// The allocator for unique pointers. -#[cfg(all(not(no_global_oom_handling), not(test)))] -#[lang = "exchange_malloc"] -#[inline] -unsafe fn exchange_malloc(size: usize, align: usize) -> *mut u8 { - let layout = unsafe { Layout::from_size_align_unchecked(size, align) }; - match Global.allocate(layout) { - Ok(ptr) => ptr.as_mut_ptr(), - Err(_) => handle_alloc_error(layout), - } -} - -#[cfg_attr(not(test), lang = "box_free")] -#[inline] -#[rustc_const_unstable(feature = "const_box", issue = "92521")] -// This signature has to be the same as `Box`, otherwise an ICE will happen. -// When an additional parameter to `Box` is added (like `A: Allocator`), this has to be added here as -// well. -// For example if `Box` is changed to `struct Box<T: ?Sized, A: Allocator>(Unique<T>, A)`, -// this function has to be changed to `fn box_free<T: ?Sized, A: Allocator>(Unique<T>, A)` as well. -pub(crate) const unsafe fn box_free<T: ?Sized, A: ~const Allocator + ~const Destruct>( - ptr: Unique<T>, - alloc: A, -) { - unsafe { - let size = size_of_val(ptr.as_ref()); - let align = min_align_of_val(ptr.as_ref()); - let layout = Layout::from_size_align_unchecked(size, align); - alloc.deallocate(From::from(ptr.cast()), layout) - } -} - -// # Allocation error handler - -#[cfg(not(no_global_oom_handling))] -extern "Rust" { - // This is the magic symbol to call the global alloc error handler. rustc generates - // it to call `__rg_oom` if there is a `#[alloc_error_handler]`, or to call the - // default implementations below (`__rdl_oom`) otherwise. - fn __rust_alloc_error_handler(size: usize, align: usize) -> !; -} - -/// Abort on memory allocation error or failure. -/// -/// Callers of memory allocation APIs wishing to abort computation -/// in response to an allocation error are encouraged to call this function, -/// rather than directly invoking `panic!` or similar. -/// -/// The default behavior of this function is to print a message to standard error -/// and abort the process. -/// It can be replaced with [`set_alloc_error_hook`] and [`take_alloc_error_hook`]. -/// -/// [`set_alloc_error_hook`]: ../../std/alloc/fn.set_alloc_error_hook.html -/// [`take_alloc_error_hook`]: ../../std/alloc/fn.take_alloc_error_hook.html -#[stable(feature = "global_alloc", since = "1.28.0")] -#[rustc_const_unstable(feature = "const_alloc_error", issue = "92523")] -#[cfg(all(not(no_global_oom_handling), not(test)))] -#[cold] -pub const fn handle_alloc_error(layout: Layout) -> ! { - const fn ct_error(_: Layout) -> ! { - panic!("allocation failed"); - } - - fn rt_error(layout: Layout) -> ! { - unsafe { - __rust_alloc_error_handler(layout.size(), layout.align()); - } - } - - unsafe { core::intrinsics::const_eval_select((layout,), ct_error, rt_error) } -} - -// For alloc test `std::alloc::handle_alloc_error` can be used directly. -#[cfg(all(not(no_global_oom_handling), test))] -pub use std::alloc::handle_alloc_error; - -#[cfg(all(not(no_global_oom_handling), not(test)))] -#[doc(hidden)] -#[allow(unused_attributes)] -#[unstable(feature = "alloc_internals", issue = "none")] -pub mod __alloc_error_handler { - // called via generated `__rust_alloc_error_handler` if there is no - // `#[alloc_error_handler]`. - #[rustc_std_internal_symbol] - pub unsafe fn __rdl_oom(size: usize, _align: usize) -> ! { - extern "Rust" { - // This symbol is emitted by rustc next to __rust_alloc_error_handler. - // Its value depends on the -Zoom={panic,abort} compiler option. - static __rust_alloc_error_handler_should_panic: u8; - } - - #[allow(unused_unsafe)] - if unsafe { __rust_alloc_error_handler_should_panic != 0 } { - panic!("memory allocation of {size} bytes failed") - } else { - core::panicking::panic_nounwind_fmt(format_args!( - "memory allocation of {size} bytes failed" - )) - } - } -} - -/// Specialize clones into pre-allocated, uninitialized memory. -/// Used by `Box::clone` and `Rc`/`Arc::make_mut`. -pub(crate) trait WriteCloneIntoRaw: Sized { - unsafe fn write_clone_into_raw(&self, target: *mut Self); -} - -impl<T: Clone> WriteCloneIntoRaw for T { - #[inline] - default unsafe fn write_clone_into_raw(&self, target: *mut Self) { - // Having allocated *first* may allow the optimizer to create - // the cloned value in-place, skipping the local and move. - unsafe { target.write(self.clone()) }; - } -} - -impl<T: Copy> WriteCloneIntoRaw for T { - #[inline] - unsafe fn write_clone_into_raw(&self, target: *mut Self) { - // We can always copy in-place, without ever involving a local value. - unsafe { target.copy_from_nonoverlapping(self, 1) }; - } -} diff --git a/rust/alloc/boxed.rs b/rust/alloc/boxed.rs deleted file mode 100644 index 14af9860c36c..000000000000 --- a/rust/alloc/boxed.rs +++ /dev/null @@ -1,2432 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 OR MIT - -//! The `Box<T>` type for heap allocation. -//! -//! [`Box<T>`], casually referred to as a 'box', provides the simplest form of -//! heap allocation in Rust. Boxes provide ownership for this allocation, and -//! drop their contents when they go out of scope. Boxes also ensure that they -//! never allocate more than `isize::MAX` bytes. -//! -//! # Examples -//! -//! Move a value from the stack to the heap by creating a [`Box`]: -//! -//! ``` -//! let val: u8 = 5; -//! let boxed: Box<u8> = Box::new(val); -//! ``` -//! -//! Move a value from a [`Box`] back to the stack by [dereferencing]: -//! -//! ``` -//! let boxed: Box<u8> = Box::new(5); -//! let val: u8 = *boxed; -//! ``` -//! -//! Creating a recursive data structure: -//! -//! ``` -//! #[derive(Debug)] -//! enum List<T> { -//! Cons(T, Box<List<T>>), -//! Nil, -//! } -//! -//! let list: List<i32> = List::Cons(1, Box::new(List::Cons(2, Box::new(List::Nil)))); -//! println!("{list:?}"); -//! ``` -//! -//! This will print `Cons(1, Cons(2, Nil))`. -//! -//! Recursive structures must be boxed, because if the definition of `Cons` -//! looked like this: -//! -//! ```compile_fail,E0072 -//! # enum List<T> { -//! Cons(T, List<T>), -//! # } -//! ``` -//! -//! It wouldn't work. This is because the size of a `List` depends on how many -//! elements are in the list, and so we don't know how much memory to allocate -//! for a `Cons`. By introducing a [`Box<T>`], which has a defined size, we know how -//! big `Cons` needs to be. -//! -//! # Memory layout -//! -//! For non-zero-sized values, a [`Box`] will use the [`Global`] allocator for -//! its allocation. It is valid to convert both ways between a [`Box`] and a -//! raw pointer allocated with the [`Global`] allocator, given that the -//! [`Layout`] used with the allocator is correct for the type. More precisely, -//! a `value: *mut T` that has been allocated with the [`Global`] allocator -//! with `Layout::for_value(&*value)` may be converted into a box using -//! [`Box::<T>::from_raw(value)`]. Conversely, the memory backing a `value: *mut -//! T` obtained from [`Box::<T>::into_raw`] may be deallocated using the -//! [`Global`] allocator with [`Layout::for_value(&*value)`]. -//! -//! For zero-sized values, the `Box` pointer still has to be [valid] for reads -//! and writes and sufficiently aligned. In particular, casting any aligned -//! non-zero integer literal to a raw pointer produces a valid pointer, but a -//! pointer pointing into previously allocated memory that since got freed is -//! not valid. The recommended way to build a Box to a ZST if `Box::new` cannot -//! be used is to use [`ptr::NonNull::dangling`]. -//! -//! So long as `T: Sized`, a `Box<T>` is guaranteed to be represented -//! as a single pointer and is also ABI-compatible with C pointers -//! (i.e. the C type `T*`). This means that if you have extern "C" -//! Rust functions that will be called from C, you can define those -//! Rust functions using `Box<T>` types, and use `T*` as corresponding -//! type on the C side. As an example, consider this C header which -//! declares functions that create and destroy some kind of `Foo` -//! value: -//! -//! ```c -//! /* C header */ -//! -//! /* Returns ownership to the caller */ -//! struct Foo* foo_new(void); -//! -//! /* Takes ownership from the caller; no-op when invoked with null */ -//! void foo_delete(struct Foo*); -//! ``` -//! -//! These two functions might be implemented in Rust as follows. Here, the -//! `struct Foo*` type from C is translated to `Box<Foo>`, which captures -//! the ownership constraints. Note also that the nullable argument to -//! `foo_delete` is represented in Rust as `Option<Box<Foo>>`, since `Box<Foo>` -//! cannot be null. -//! -//! ``` -//! #[repr(C)] -//! pub struct Foo; -//! -//! #[no_mangle] -//! pub extern "C" fn foo_new() -> Box<Foo> { -//! Box::new(Foo) -//! } -//! -//! #[no_mangle] -//! pub extern "C" fn foo_delete(_: Option<Box<Foo>>) {} -//! ``` -//! -//! Even though `Box<T>` has the same representation and C ABI as a C pointer, -//! this does not mean that you can convert an arbitrary `T*` into a `Box<T>` -//! and expect things to work. `Box<T>` values will always be fully aligned, -//! non-null pointers. Moreover, the destructor for `Box<T>` will attempt to -//! free the value with the global allocator. In general, the best practice -//! is to only use `Box<T>` for pointers that originated from the global -//! allocator. -//! -//! **Important.** At least at present, you should avoid using -//! `Box<T>` types for functions that are defined in C but invoked -//! from Rust. In those cases, you should directly mirror the C types -//! as closely as possible. Using types like `Box<T>` where the C -//! definition is just using `T*` can lead to undefined behavior, as -//! described in [rust-lang/unsafe-code-guidelines#198][ucg#198]. -//! -//! # Considerations for unsafe code -//! -//! **Warning: This section is not normative and is subject to change, possibly -//! being relaxed in the future! It is a simplified summary of the rules -//! currently implemented in the compiler.** -//! -//! The aliasing rules for `Box<T>` are the same as for `&mut T`. `Box<T>` -//! asserts uniqueness over its content. Using raw pointers derived from a box -//! after that box has been mutated through, moved or borrowed as `&mut T` -//! is not allowed. For more guidance on working with box from unsafe code, see -//! [rust-lang/unsafe-code-guidelines#326][ucg#326]. -//! -//! -//! [ucg#198]: https://github.com/rust-lang/unsafe-code-guidelines/issues/198 -//! [ucg#326]: https://github.com/rust-lang/unsafe-code-guidelines/issues/326 -//! [dereferencing]: core::ops::Deref -//! [`Box::<T>::from_raw(value)`]: Box::from_raw -//! [`Global`]: crate::alloc::Global -//! [`Layout`]: crate::alloc::Layout -//! [`Layout::for_value(&*value)`]: crate::alloc::Layout::for_value -//! [valid]: ptr#safety - -#![stable(feature = "rust1", since = "1.0.0")] - -use core::any::Any; -use core::async_iter::AsyncIterator; -use core::borrow; -use core::cmp::Ordering; -use core::convert::{From, TryFrom}; -use core::error::Error; -use core::fmt; -use core::future::Future; -use core::hash::{Hash, Hasher}; -#[cfg(not(no_global_oom_handling))] -use core::iter::FromIterator; -use core::iter::{FusedIterator, Iterator}; -use core::marker::Tuple; -use core::marker::{Destruct, Unpin, Unsize}; -use core::mem; -use core::ops::{ - CoerceUnsized, Deref, DerefMut, DispatchFromDyn, Generator, GeneratorState, Receiver, -}; -use core::pin::Pin; -use core::ptr::{self, Unique}; -use core::task::{Context, Poll}; - -#[cfg(not(no_global_oom_handling))] -use crate::alloc::{handle_alloc_error, WriteCloneIntoRaw}; -use crate::alloc::{AllocError, Allocator, Global, Layout}; -#[cfg(not(no_global_oom_handling))] -use crate::borrow::Cow; -use crate::raw_vec::RawVec; -#[cfg(not(no_global_oom_handling))] -use crate::str::from_boxed_utf8_unchecked; -#[cfg(not(no_global_oom_handling))] -use crate::string::String; -#[cfg(not(no_global_oom_handling))] -use crate::vec::Vec; - -#[cfg(not(no_thin))] -#[unstable(feature = "thin_box", issue = "92791")] -pub use thin::ThinBox; - -#[cfg(not(no_thin))] -mod thin; - -/// A pointer type that uniquely owns a heap allocation of type `T`. -/// -/// See the [module-level documentation](../../std/boxed/index.html) for more. -#[lang = "owned_box"] -#[fundamental] -#[stable(feature = "rust1", since = "1.0.0")] -// The declaration of the `Box` struct must be kept in sync with the -// `alloc::alloc::box_free` function or ICEs will happen. See the comment -// on `box_free` for more details. -pub struct Box< - T: ?Sized, - #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator = Global, ->(Unique<T>, A); - -impl<T> Box<T> { - /// Allocates memory on the heap and then places `x` into it. - /// - /// This doesn't actually allocate if `T` is zero-sized. - /// - /// # Examples - /// - /// ``` - /// let five = Box::new(5); - /// ``` - #[cfg(all(not(no_global_oom_handling)))] - #[inline(always)] - #[stable(feature = "rust1", since = "1.0.0")] - #[must_use] - pub fn new(x: T) -> Self { - #[rustc_box] - Box::new(x) - } - - /// Constructs a new box with uninitialized contents. - /// - /// # Examples - /// - /// ``` - /// #![feature(new_uninit)] - /// - /// let mut five = Box::<u32>::new_uninit(); - /// - /// let five = unsafe { - /// // Deferred initialization: - /// five.as_mut_ptr().write(5); - /// - /// five.assume_init() - /// }; - /// - /// assert_eq!(*five, 5) - /// ``` - #[cfg(not(no_global_oom_handling))] - #[unstable(feature = "new_uninit", issue = "63291")] - #[must_use] - #[inline] - pub fn new_uninit() -> Box<mem::MaybeUninit<T>> { - Self::new_uninit_in(Global) - } - - /// Constructs a new `Box` with uninitialized contents, with the memory - /// being filled with `0` bytes. - /// - /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and incorrect usage - /// of this method. - /// - /// # Examples - /// - /// ``` - /// #![feature(new_uninit)] - /// - /// let zero = Box::<u32>::new_zeroed(); - /// let zero = unsafe { zero.assume_init() }; - /// - /// assert_eq!(*zero, 0) - /// ``` - /// - /// [zeroed]: mem::MaybeUninit::zeroed - #[cfg(not(no_global_oom_handling))] - #[inline] - #[unstable(feature = "new_uninit", issue = "63291")] - #[must_use] - pub fn new_zeroed() -> Box<mem::MaybeUninit<T>> { - Self::new_zeroed_in(Global) - } - - /// Constructs a new `Pin<Box<T>>`. If `T` does not implement [`Unpin`], then - /// `x` will be pinned in memory and unable to be moved. - /// - /// Constructing and pinning of the `Box` can also be done in two steps: `Box::pin(x)` - /// does the same as <code>[Box::into_pin]\([Box::new]\(x))</code>. Consider using - /// [`into_pin`](Box::into_pin) if you already have a `Box<T>`, or if you want to - /// construct a (pinned) `Box` in a different way than with [`Box::new`]. - #[cfg(not(no_global_oom_handling))] - #[stable(feature = "pin", since = "1.33.0")] - #[must_use] - #[inline(always)] - pub fn pin(x: T) -> Pin<Box<T>> { - (#[rustc_box] - Box::new(x)) - .into() - } - - /// Allocates memory on the heap then places `x` into it, - /// returning an error if the allocation fails - /// - /// This doesn't actually allocate if `T` is zero-sized. - /// - /// # Examples - /// - /// ``` - /// #![feature(allocator_api)] - /// - /// let five = Box::try_new(5)?; - /// # Ok::<(), std::alloc::AllocError>(()) - /// ``` - #[unstable(feature = "allocator_api", issue = "32838")] - #[inline] - pub fn try_new(x: T) -> Result<Self, AllocError> { - Self::try_new_in(x, Global) - } - - /// Constructs a new box with uninitialized contents on the heap, - /// returning an error if the allocation fails - /// - /// # Examples - /// - /// ``` - /// #![feature(allocator_api, new_uninit)] - /// - /// let mut five = Box::<u32>::try_new_uninit()?; - /// - /// let five = unsafe { - /// // Deferred initialization: - /// five.as_mut_ptr().write(5); - /// - /// five.assume_init() - /// }; - /// - /// assert_eq!(*five, 5); - /// # Ok::<(), std::alloc::AllocError>(()) - /// ``` - #[unstable(feature = "allocator_api", issue = "32838")] - // #[unstable(feature = "new_uninit", issue = "63291")] - #[inline] - pub fn try_new_uninit() -> Result<Box<mem::MaybeUninit<T>>, AllocError> { - Box::try_new_uninit_in(Global) - } - - /// Constructs a new `Box` with uninitialized contents, with the memory - /// being filled with `0` bytes on the heap - /// - /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and incorrect usage - /// of this method. - /// - /// # Examples - /// - /// ``` - /// #![feature(allocator_api, new_uninit)] - /// - /// let zero = Box::<u32>::try_new_zeroed()?; - /// let zero = unsafe { zero.assume_init() }; - /// - /// assert_eq!(*zero, 0); - /// # Ok::<(), std::alloc::AllocError>(()) - /// ``` - /// - /// [zeroed]: mem::MaybeUninit::zeroed - #[unstable(feature = "allocator_api", issue = "32838")] - // #[unstable(feature = "new_uninit", issue = "63291")] - #[inline] - pub fn try_new_zeroed() -> Result<Box<mem::MaybeUninit<T>>, AllocError> { - Box::try_new_zeroed_in(Global) - } -} - -impl<T, A: Allocator> Box<T, A> { - /// Allocates memory in the given allocator then places `x` into it. - /// - /// This doesn't actually allocate if `T` is zero-sized. - /// - /// # Examples - /// - /// ``` - /// #![feature(allocator_api)] - /// - /// use std::alloc::System; - /// - /// let five = Box::new_in(5, System); - /// ``` - #[cfg(not(no_global_oom_handling))] - #[unstable(feature = "allocator_api", issue = "32838")] - #[rustc_const_unstable(feature = "const_box", issue = "92521")] - #[must_use] - #[inline] - pub const fn new_in(x: T, alloc: A) -> Self - where - A: ~const Allocator + ~const Destruct, - { - let mut boxed = Self::new_uninit_in(alloc); - unsafe { - boxed.as_mut_ptr().write(x); - boxed.assume_init() - } - } - - /// Allocates memory in the given allocator then places `x` into it, - /// returning an error if the allocation fails - /// - /// This doesn't actually allocate if `T` is zero-sized. - /// - /// # Examples - /// - /// ``` - /// #![feature(allocator_api)] - /// - /// use std::alloc::System; - /// - /// let five = Box::try_new_in(5, System)?; - /// # Ok::<(), std::alloc::AllocError>(()) - /// ``` - #[unstable(feature = "allocator_api", issue = "32838")] - #[rustc_const_unstable(feature = "const_box", issue = "92521")] - #[inline] - pub const fn try_new_in(x: T, alloc: A) -> Result<Self, AllocError> - where - T: ~const Destruct, - A: ~const Allocator + ~const Destruct, - { - let mut boxed = Self::try_new_uninit_in(alloc)?; - unsafe { - boxed.as_mut_ptr().write(x); - Ok(boxed.assume_init()) - } - } - - /// Constructs a new box with uninitialized contents in the provided allocator. - /// - /// # Examples - /// - /// ``` - /// #![feature(allocator_api, new_uninit)] - /// - /// use std::alloc::System; - /// - /// let mut five = Box::<u32, _>::new_uninit_in(System); - /// - /// let five = unsafe { - /// // Deferred initialization: - /// five.as_mut_ptr().write(5); - /// - /// five.assume_init() - /// }; - /// - /// assert_eq!(*five, 5) - /// ``` - #[unstable(feature = "allocator_api", issue = "32838")] - #[rustc_const_unstable(feature = "const_box", issue = "92521")] - #[cfg(not(no_global_oom_handling))] - #[must_use] - // #[unstable(feature = "new_uninit", issue = "63291")] - pub const fn new_uninit_in(alloc: A) -> Box<mem::MaybeUninit<T>, A> - where - A: ~const Allocator + ~const Destruct, - { - let layout = Layout::new::<mem::MaybeUninit<T>>(); - // NOTE: Prefer match over unwrap_or_else since closure sometimes not inlineable. - // That would make code size bigger. - match Box::try_new_uninit_in(alloc) { - Ok(m) => m, - Err(_) => handle_alloc_error(layout), - } - } - - /// Constructs a new box with uninitialized contents in the provided allocator, - /// returning an error if the allocation fails - /// - /// # Examples - /// - /// ``` - /// #![feature(allocator_api, new_uninit)] - /// - /// use std::alloc::System; - /// - /// let mut five = Box::<u32, _>::try_new_uninit_in(System)?; - /// - /// let five = unsafe { - /// // Deferred initialization: - /// five.as_mut_ptr().write(5); - /// - /// five.assume_init() - /// }; - /// - /// assert_eq!(*five, 5); - /// # Ok::<(), std::alloc::AllocError>(()) - /// ``` - #[unstable(feature = "allocator_api", issue = "32838")] - // #[unstable(feature = "new_uninit", issue = "63291")] - #[rustc_const_unstable(feature = "const_box", issue = "92521")] - pub const fn try_new_uninit_in(alloc: A) -> Result<Box<mem::MaybeUninit<T>, A>, AllocError> - where - A: ~const Allocator + ~const Destruct, - { - let layout = Layout::new::<mem::MaybeUninit<T>>(); - let ptr = alloc.allocate(layout)?.cast(); - unsafe { Ok(Box::from_raw_in(ptr.as_ptr(), alloc)) } - } - - /// Constructs a new `Box` with uninitialized contents, with the memory - /// being filled with `0` bytes in the provided allocator. - /// - /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and incorrect usage - /// of this method. - /// - /// # Examples - /// - /// ``` - /// #![feature(allocator_api, new_uninit)] - /// - /// use std::alloc::System; - /// - /// let zero = Box::<u32, _>::new_zeroed_in(System); - /// let zero = unsafe { zero.assume_init() }; - /// - /// assert_eq!(*zero, 0) - /// ``` - /// - /// [zeroed]: mem::MaybeUninit::zeroed - #[unstable(feature = "allocator_api", issue = "32838")] - #[rustc_const_unstable(feature = "const_box", issue = "92521")] - #[cfg(not(no_global_oom_handling))] - // #[unstable(feature = "new_uninit", issue = "63291")] - #[must_use] - pub const fn new_zeroed_in(alloc: A) -> Box<mem::MaybeUninit<T>, A> - where - A: ~const Allocator + ~const Destruct, - { - let layout = Layout::new::<mem::MaybeUninit<T>>(); - // NOTE: Prefer match over unwrap_or_else since closure sometimes not inlineable. - // That would make code size bigger. - match Box::try_new_zeroed_in(alloc) { - Ok(m) => m, - Err(_) => handle_alloc_error(layout), - } - } - - /// Constructs a new `Box` with uninitialized contents, with the memory - /// being filled with `0` bytes in the provided allocator, - /// returning an error if the allocation fails, - /// - /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and incorrect usage - /// of this method. - /// - /// # Examples - /// - /// ``` - /// #![feature(allocator_api, new_uninit)] - /// - /// use std::alloc::System; - /// - /// let zero = Box::<u32, _>::try_new_zeroed_in(System)?; - /// let zero = unsafe { zero.assume_init() }; - /// - /// assert_eq!(*zero, 0); - /// # Ok::<(), std::alloc::AllocError>(()) - /// ``` - /// - /// [zeroed]: mem::MaybeUninit::zeroed - #[unstable(feature = "allocator_api", issue = "32838")] - // #[unstable(feature = "new_uninit", issue = "63291")] - #[rustc_const_unstable(feature = "const_box", issue = "92521")] - pub const fn try_new_zeroed_in(alloc: A) -> Result<Box<mem::MaybeUninit<T>, A>, AllocError> - where - A: ~const Allocator + ~const Destruct, - { - let layout = Layout::new::<mem::MaybeUninit<T>>(); - let ptr = alloc.allocate_zeroed(layout)?.cast(); - unsafe { Ok(Box::from_raw_in(ptr.as_ptr(), alloc)) } - } - - /// Constructs a new `Pin<Box<T, A>>`. If `T` does not implement [`Unpin`], then - /// `x` will be pinned in memory and unable to be moved. - /// - /// Constructing and pinning of the `Box` can also be done in two steps: `Box::pin_in(x, alloc)` - /// does the same as <code>[Box::into_pin]\([Box::new_in]\(x, alloc))</code>. Consider using - /// [`into_pin`](Box::into_pin) if you already have a `Box<T, A>`, or if you want to - /// construct a (pinned) `Box` in a different way than with [`Box::new_in`]. - #[cfg(not(no_global_oom_handling))] - #[unstable(feature = "allocator_api", issue = "32838")] - #[rustc_const_unstable(feature = "const_box", issue = "92521")] - #[must_use] - #[inline(always)] - pub const fn pin_in(x: T, alloc: A) -> Pin<Self> - where - A: 'static + ~const Allocator + ~const Destruct, - { - Self::into_pin(Self::new_in(x, alloc)) - } - - /// Converts a `Box<T>` into a `Box<[T]>` - /// - /// This conversion does not allocate on the heap and happens in place. - #[unstable(feature = "box_into_boxed_slice", issue = "71582")] - #[rustc_const_unstable(feature = "const_box", issue = "92521")] - pub const fn into_boxed_slice(boxed: Self) -> Box<[T], A> { - let (raw, alloc) = Box::into_raw_with_allocator(boxed); - unsafe { Box::from_raw_in(raw as *mut [T; 1], alloc) } - } - - /// Consumes the `Box`, returning the wrapped value. - /// - /// # Examples - /// - /// ``` - /// #![feature(box_into_inner)] - /// - /// let c = Box::new(5); - /// - /// assert_eq!(Box::into_inner(c), 5); - /// ``` - #[unstable(feature = "box_into_inner", issue = "80437")] - #[rustc_const_unstable(feature = "const_box", issue = "92521")] - #[inline] - pub const fn into_inner(boxed: Self) -> T - where - Self: ~const Destruct, - { - *boxed - } -} - -impl<T> Box<[T]> { - /// Constructs a new boxed slice with uninitialized contents. - /// - /// # Examples - /// - /// ``` - /// #![feature(new_uninit)] - /// - /// let mut values = Box::<[u32]>::new_uninit_slice(3); - /// - /// let values = unsafe { - /// // Deferred initialization: - /// values[0].as_mut_ptr().write(1); - /// values[1].as_mut_ptr().write(2); - /// values[2].as_mut_ptr().write(3); - /// - /// values.assume_init() - /// }; - /// - /// assert_eq!(*values, [1, 2, 3]) - /// ``` - #[cfg(not(no_global_oom_handling))] - #[unstable(feature = "new_uninit", issue = "63291")] - #[must_use] - pub fn new_uninit_slice(len: usize) -> Box<[mem::MaybeUninit<T>]> { - unsafe { RawVec::with_capacity(len).into_box(len) } - } - - /// Constructs a new boxed slice with uninitialized contents, with the memory - /// being filled with `0` bytes. - /// - /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and incorrect usage - /// of this method. - /// - /// # Examples - /// - /// ``` - /// #![feature(new_uninit)] - /// - /// let values = Box::<[u32]>::new_zeroed_slice(3); - /// let values = unsafe { values.assume_init() }; - /// - /// assert_eq!(*values, [0, 0, 0]) - /// ``` - /// - /// [zeroed]: mem::MaybeUninit::zeroed - #[cfg(not(no_global_oom_handling))] - #[unstable(feature = "new_uninit", issue = "63291")] - #[must_use] - pub fn new_zeroed_slice(len: usize) -> Box<[mem::MaybeUninit<T>]> { - unsafe { RawVec::with_capacity_zeroed(len).into_box(len) } - } - - /// Constructs a new boxed slice with uninitialized contents. Returns an error if - /// the allocation fails - /// - /// # Examples - /// - /// ``` - /// #![feature(allocator_api, new_uninit)] - /// - /// let mut values = Box::<[u32]>::try_new_uninit_slice(3)?; - /// let values = unsafe { - /// // Deferred initialization: - /// values[0].as_mut_ptr().write(1); - /// values[1].as_mut_ptr().write(2); - /// values[2].as_mut_ptr().write(3); - /// values.assume_init() - /// }; - /// - /// assert_eq!(*values, [1, 2, 3]); - /// # Ok::<(), std::alloc::AllocError>(()) - /// ``` - #[unstable(feature = "allocator_api", issue = "32838")] - #[inline] - pub fn try_new_uninit_slice(len: usize) -> Result<Box<[mem::MaybeUninit<T>]>, AllocError> { - unsafe { - let layout = match Layout::array::<mem::MaybeUninit<T>>(len) { - Ok(l) => l, - Err(_) => return Err(AllocError), - }; - let ptr = Global.allocate(layout)?; - Ok(RawVec::from_raw_parts_in(ptr.as_mut_ptr() as *mut _, len, Global).into_box(len)) - } - } - - /// Constructs a new boxed slice with uninitialized contents, with the memory - /// being filled with `0` bytes. Returns an error if the allocation fails - /// - /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and incorrect usage - /// of this method. - /// - /// # Examples - /// - /// ``` - /// #![feature(allocator_api, new_uninit)] - /// - /// let values = Box::<[u32]>::try_new_zeroed_slice(3)?; - /// let values = unsafe { values.assume_init() }; - /// - /// assert_eq!(*values, [0, 0, 0]); - /// # Ok::<(), std::alloc::AllocError>(()) - /// ``` - /// - /// [zeroed]: mem::MaybeUninit::zeroed - #[unstable(feature = "allocator_api", issue = "32838")] - #[inline] - pub fn try_new_zeroed_slice(len: usize) -> Result<Box<[mem::MaybeUninit<T>]>, AllocError> { - unsafe { - let layout = match Layout::array::<mem::MaybeUninit<T>>(len) { - Ok(l) => l, - Err(_) => return Err(AllocError), - }; - let ptr = Global.allocate_zeroed(layout)?; - Ok(RawVec::from_raw_parts_in(ptr.as_mut_ptr() as *mut _, len, Global).into_box(len)) - } - } -} - -impl<T, A: Allocator> Box<[T], A> { - /// Constructs a new boxed slice with uninitialized contents in the provided allocator. - /// - /// # Examples - /// - /// ``` - /// #![feature(allocator_api, new_uninit)] - /// - /// use std::alloc::System; - /// - /// let mut values = Box::<[u32], _>::new_uninit_slice_in(3, System); - /// - /// let values = unsafe { - /// // Deferred initialization: - /// values[0].as_mut_ptr().write(1); - /// values[1].as_mut_ptr().write(2); - /// values[2].as_mut_ptr().write(3); - /// - /// values.assume_init() - /// }; - /// - /// assert_eq!(*values, [1, 2, 3]) - /// ``` - #[cfg(not(no_global_oom_handling))] - #[unstable(feature = "allocator_api", issue = "32838")] - // #[unstable(feature = "new_uninit", issue = "63291")] - #[must_use] - pub fn new_uninit_slice_in(len: usize, alloc: A) -> Box<[mem::MaybeUninit<T>], A> { - unsafe { RawVec::with_capacity_in(len, alloc).into_box(len) } - } - - /// Constructs a new boxed slice with uninitialized contents in the provided allocator, - /// with the memory being filled with `0` bytes. - /// - /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and incorrect usage - /// of this method. - /// - /// # Examples - /// - /// ``` - /// #![feature(allocator_api, new_uninit)] - /// - /// use std::alloc::System; - /// - /// let values = Box::<[u32], _>::new_zeroed_slice_in(3, System); - /// let values = unsafe { values.assume_init() }; - /// - /// assert_eq!(*values, [0, 0, 0]) - /// ``` - /// - /// [zeroed]: mem::MaybeUninit::zeroed - #[cfg(not(no_global_oom_handling))] - #[unstable(feature = "allocator_api", issue = "32838")] - // #[unstable(feature = "new_uninit", issue = "63291")] - #[must_use] - pub fn new_zeroed_slice_in(len: usize, alloc: A) -> Box<[mem::MaybeUninit<T>], A> { - unsafe { RawVec::with_capacity_zeroed_in(len, alloc).into_box(len) } - } -} - -impl<T, A: Allocator> Box<mem::MaybeUninit<T>, A> { - /// Converts to `Box<T, A>`. - /// - /// # Safety - /// - /// As with [`MaybeUninit::assume_init`], - /// it is up to the caller to guarantee that the value - /// really is in an initialized state. - /// Calling this when the content is not yet fully initialized - /// causes immediate undefined behavior. - /// - /// [`MaybeUninit::assume_init`]: mem::MaybeUninit::assume_init - /// - /// # Examples - /// - /// ``` - /// #![feature(new_uninit)] - /// - /// let mut five = Box::<u32>::new_uninit(); - /// - /// let five: Box<u32> = unsafe { - /// // Deferred initialization: - /// five.as_mut_ptr().write(5); - /// - /// five.assume_init() - /// }; - /// - /// assert_eq!(*five, 5) - /// ``` - #[unstable(feature = "new_uninit", issue = "63291")] - #[rustc_const_unstable(feature = "const_box", issue = "92521")] - #[inline] - pub const unsafe fn assume_init(self) -> Box<T, A> { - let (raw, alloc) = Box::into_raw_with_allocator(self); - unsafe { Box::from_raw_in(raw as *mut T, alloc) } - } - - /// Writes the value and converts to `Box<T, A>`. - /// - /// This method converts the box similarly to [`Box::assume_init`] but - /// writes `value` into it before conversion thus guaranteeing safety. - /// In some scenarios use of this method may improve performance because - /// the compiler may be able to optimize copying from stack. - /// - /// # Examples - /// - /// ``` - /// #![feature(new_uninit)] - /// - /// let big_box = Box::<[usize; 1024]>::new_uninit(); - /// - /// let mut array = [0; 1024]; - /// for (i, place) in array.iter_mut().enumerate() { - /// *place = i; - /// } - /// - /// // The optimizer may be able to elide this copy, so previous code writes - /// // to heap directly. - /// let big_box = Box::write(big_box, array); - /// - /// for (i, x) in big_box.iter().enumerate() { - /// assert_eq!(*x, i); - /// } - /// ``` - #[unstable(feature = "new_uninit", issue = "63291")] - #[rustc_const_unstable(feature = "const_box", issue = "92521")] - #[inline] - pub const fn write(mut boxed: Self, value: T) -> Box<T, A> { - unsafe { - (*boxed).write(value); - boxed.assume_init() - } - } -} - -impl<T, A: Allocator> Box<[mem::MaybeUninit<T>], A> { - /// Converts to `Box<[T], A>`. - /// - /// # Safety - /// - /// As with [`MaybeUninit::assume_init`], - /// it is up to the caller to guarantee that the values - /// really are in an initialized state. - /// Calling this when the content is not yet fully initialized - /// causes immediate undefined behavior. - /// - /// [`MaybeUninit::assume_init`]: mem::MaybeUninit::assume_init - /// - /// # Examples - /// - /// ``` - /// #![feature(new_uninit)] - /// - /// let mut values = Box::<[u32]>::new_uninit_slice(3); - /// - /// let values = unsafe { - /// // Deferred initialization: - /// values[0].as_mut_ptr().write(1); - /// values[1].as_mut_ptr().write(2); - /// values[2].as_mut_ptr().write(3); - /// - /// values.assume_init() - /// }; - /// - /// assert_eq!(*values, [1, 2, 3]) - /// ``` - #[unstable(feature = "new_uninit", issue = "63291")] - #[inline] - pub unsafe fn assume_init(self) -> Box<[T], A> { - let (raw, alloc) = Box::into_raw_with_allocator(self); - unsafe { Box::from_raw_in(raw as *mut [T], alloc) } - } -} - -impl<T: ?Sized> Box<T> { - /// Constructs a box from a raw pointer. - /// - /// After calling this function, the raw pointer is owned by the - /// resulting `Box`. Specifically, the `Box` destructor will call - /// the destructor of `T` and free the allocated memory. For this - /// to be safe, the memory must have been allocated in accordance - /// with the [memory layout] used by `Box` . - /// - /// # Safety - /// - /// This function is unsafe because improper use may lead to - /// memory problems. For example, a double-free may occur if the - /// function is called twice on the same raw pointer. - /// - /// The safety conditions are described in the [memory layout] section. - /// - /// # Examples - /// - /// Recreate a `Box` which was previously converted to a raw pointer - /// using [`Box::into_raw`]: - /// ``` - /// let x = Box::new(5); - /// let ptr = Box::into_raw(x); - /// let x = unsafe { Box::from_raw(ptr) }; - /// ``` - /// Manually create a `Box` from scratch by using the global allocator: - /// ``` - /// use std::alloc::{alloc, Layout}; - /// - /// unsafe { - /// let ptr = alloc(Layout::new::<i32>()) as *mut i32; - /// // In general .write is required to avoid attempting to destruct - /// // the (uninitialized) previous contents of `ptr`, though for this - /// // simple example `*ptr = 5` would have worked as well. - /// ptr.write(5); - /// let x = Box::from_raw(ptr); - /// } - /// ``` - /// - /// [memory layout]: self#memory-layout - /// [`Layout`]: crate::Layout - #[stable(feature = "box_raw", since = "1.4.0")] - #[inline] - #[must_use = "call `drop(Box::from_raw(ptr))` if you intend to drop the `Box`"] - pub unsafe fn from_raw(raw: *mut T) -> Self { - unsafe { Self::from_raw_in(raw, Global) } - } -} - -impl<T: ?Sized, A: Allocator> Box<T, A> { - /// Constructs a box from a raw pointer in the given allocator. - /// - /// After calling this function, the raw pointer is owned by the - /// resulting `Box`. Specifically, the `Box` destructor will call - /// the destructor of `T` and free the allocated memory. For this - /// to be safe, the memory must have been allocated in accordance - /// with the [memory layout] used by `Box` . - /// - /// # Safety - /// - /// This function is unsafe because improper use may lead to - /// memory problems. For example, a double-free may occur if the - /// function is called twice on the same raw pointer. - /// - /// - /// # Examples - /// - /// Recreate a `Box` which was previously converted to a raw pointer - /// using [`Box::into_raw_with_allocator`]: - /// ``` - /// #![feature(allocator_api)] - /// - /// use std::alloc::System; - /// - /// let x = Box::new_in(5, System); - /// let (ptr, alloc) = Box::into_raw_with_allocator(x); - /// let x = unsafe { Box::from_raw_in(ptr, alloc) }; - /// ``` - /// Manually create a `Box` from scratch by using the system allocator: - /// ``` - /// #![feature(allocator_api, slice_ptr_get)] - /// - /// use std::alloc::{Allocator, Layout, System}; - /// - /// unsafe { - /// let ptr = System.allocate(Layout::new::<i32>())?.as_mut_ptr() as *mut i32; - /// // In general .write is required to avoid attempting to destruct - /// // the (uninitialized) previous contents of `ptr`, though for this - /// // simple example `*ptr = 5` would have worked as well. - /// ptr.write(5); - /// let x = Box::from_raw_in(ptr, System); - /// } - /// # Ok::<(), std::alloc::AllocError>(()) - /// ``` - /// - /// [memory layout]: self#memory-layout - /// [`Layout`]: crate::Layout - #[unstable(feature = "allocator_api", issue = "32838")] - #[rustc_const_unstable(feature = "const_box", issue = "92521")] - #[inline] - pub const unsafe fn from_raw_in(raw: *mut T, alloc: A) -> Self { - Box(unsafe { Unique::new_unchecked(raw) }, alloc) - } - - /// Consumes the `Box`, returning a wrapped raw pointer. - /// - /// The pointer will be properly aligned and non-null. - /// - /// After calling this function, the caller is responsible for the - /// memory previously managed by the `Box`. In particular, the - /// caller should properly destroy `T` and release the memory, taking - /// into account the [memory layout] used by `Box`. The easiest way to - /// do this is to convert the raw pointer back into a `Box` with the - /// [`Box::from_raw`] function, allowing the `Box` destructor to perform - /// the cleanup. - /// - /// Note: this is an associated function, which means that you have - /// to call it as `Box::into_raw(b)` instead of `b.into_raw()`. This - /// is so that there is no conflict with a method on the inner type. - /// - /// # Examples - /// Converting the raw pointer back into a `Box` with [`Box::from_raw`] - /// for automatic cleanup: - /// ``` - /// let x = Box::new(String::from("Hello")); - /// let ptr = Box::into_raw(x); - /// let x = unsafe { Box::from_raw(ptr) }; - /// ``` - /// Manual cleanup by explicitly running the destructor and deallocating - /// the memory: - /// ``` - /// use std::alloc::{dealloc, Layout}; - /// use std::ptr; - /// - /// let x = Box::new(String::from("Hello")); - /// let p = Box::into_raw(x); - /// unsafe { - /// ptr::drop_in_place(p); - /// dealloc(p as *mut u8, Layout::new::<String>()); - /// } - /// ``` - /// - /// [memory layout]: self#memory-layout - #[stable(feature = "box_raw", since = "1.4.0")] - #[inline] - pub fn into_raw(b: Self) -> *mut T { - Self::into_raw_with_allocator(b).0 - } - - /// Consumes the `Box`, returning a wrapped raw pointer and the allocator. - /// - /// The pointer will be properly aligned and non-null. - /// - /// After calling this function, the caller is responsible for the - /// memory previously managed by the `Box`. In particular, the - /// caller should properly destroy `T` and release the memory, taking - /// into account the [memory layout] used by `Box`. The easiest way to - /// do this is to convert the raw pointer back into a `Box` with the - /// [`Box::from_raw_in`] function, allowing the `Box` destructor to perform - /// the cleanup. - /// - /// Note: this is an associated function, which means that you have - /// to call it as `Box::into_raw_with_allocator(b)` instead of `b.into_raw_with_allocator()`. This - /// is so that there is no conflict with a method on the inner type. - /// - /// # Examples - /// Converting the raw pointer back into a `Box` with [`Box::from_raw_in`] - /// for automatic cleanup: - /// ``` - /// #![feature(allocator_api)] - /// - /// use std::alloc::System; - /// - /// let x = Box::new_in(String::from("Hello"), System); - /// let (ptr, alloc) = Box::into_raw_with_allocator(x); - /// let x = unsafe { Box::from_raw_in(ptr, alloc) }; - /// ``` - /// Manual cleanup by explicitly running the destructor and deallocating - /// the memory: - /// ``` - /// #![feature(allocator_api)] - /// - /// use std::alloc::{Allocator, Layout, System}; - /// use std::ptr::{self, NonNull}; - /// - /// let x = Box::new_in(String::from("Hello"), System); - /// let (ptr, alloc) = Box::into_raw_with_allocator(x); - /// unsafe { - /// ptr::drop_in_place(ptr); - /// let non_null = NonNull::new_unchecked(ptr); - /// alloc.deallocate(non_null.cast(), Layout::new::<String>()); - /// } - /// ``` - /// - /// [memory layout]: self#memory-layout - #[unstable(feature = "allocator_api", issue = "32838")] - #[rustc_const_unstable(feature = "const_box", issue = "92521")] - #[inline] - pub const fn into_raw_with_allocator(b: Self) -> (*mut T, A) { - let (leaked, alloc) = Box::into_unique(b); - (leaked.as_ptr(), alloc) - } - - #[unstable( - feature = "ptr_internals", - issue = "none", - reason = "use `Box::leak(b).into()` or `Unique::from(Box::leak(b))` instead" - )] - #[rustc_const_unstable(feature = "const_box", issue = "92521")] - #[inline] - #[doc(hidden)] - pub const fn into_unique(b: Self) -> (Unique<T>, A) { - // Box is recognized as a "unique pointer" by Stacked Borrows, but internally it is a - // raw pointer for the type system. Turning it directly into a raw pointer would not be - // recognized as "releasing" the unique pointer to permit aliased raw accesses, - // so all raw pointer methods have to go through `Box::leak`. Turning *that* to a raw pointer - // behaves correctly. - let alloc = unsafe { ptr::read(&b.1) }; - (Unique::from(Box::leak(b)), alloc) - } - - /// Returns a reference to the underlying allocator. - /// - /// Note: this is an associated function, which means that you have - /// to call it as `Box::allocator(&b)` instead of `b.allocator()`. This - /// is so that there is no conflict with a method on the inner type. - #[unstable(feature = "allocator_api", issue = "32838")] - #[rustc_const_unstable(feature = "const_box", issue = "92521")] - #[inline] - pub const fn allocator(b: &Self) -> &A { - &b.1 - } - - /// Consumes and leaks the `Box`, returning a mutable reference, - /// `&'a mut T`. Note that the type `T` must outlive the chosen lifetime - /// `'a`. If the type has only static references, or none at all, then this - /// may be chosen to be `'static`. - /// - /// This function is mainly useful for data that lives for the remainder of - /// the program's life. Dropping the returned reference will cause a memory - /// leak. If this is not acceptable, the reference should first be wrapped - /// with the [`Box::from_raw`] function producing a `Box`. This `Box` can - /// then be dropped which will properly destroy `T` and release the - /// allocated memory. - /// - /// Note: this is an associated function, which means that you have - /// to call it as `Box::leak(b)` instead of `b.leak()`. This - /// is so that there is no conflict with a method on the inner type. - /// - /// # Examples - /// - /// Simple usage: - /// - /// ``` - /// let x = Box::new(41); - /// let static_ref: &'static mut usize = Box::leak(x); - /// *static_ref += 1; - /// assert_eq!(*static_ref, 42); - /// ``` - /// - /// Unsized data: - /// - /// ``` - /// let x = vec![1, 2, 3].into_boxed_slice(); - /// let static_ref = Box::leak(x); - /// static_ref[0] = 4; - /// assert_eq!(*static_ref, [4, 2, 3]); - /// ``` - #[stable(feature = "box_leak", since = "1.26.0")] - #[rustc_const_unstable(feature = "const_box", issue = "92521")] - #[inline] - pub const fn leak<'a>(b: Self) -> &'a mut T - where - A: 'a, - { - unsafe { &mut *mem::ManuallyDrop::new(b).0.as_ptr() } - } - - /// Converts a `Box<T>` into a `Pin<Box<T>>`. If `T` does not implement [`Unpin`], then - /// `*boxed` will be pinned in memory and unable to be moved. - /// - /// This conversion does not allocate on the heap and happens in place. - /// - /// This is also available via [`From`]. - /// - /// Constructing and pinning a `Box` with <code>Box::into_pin([Box::new]\(x))</code> - /// can also be written more concisely using <code>[Box::pin]\(x)</code>. - /// This `into_pin` method is useful if you already have a `Box<T>`, or you are - /// constructing a (pinned) `Box` in a different way than with [`Box::new`]. - /// - /// # Notes - /// - /// It's not recommended that crates add an impl like `From<Box<T>> for Pin<T>`, - /// as it'll introduce an ambiguity when calling `Pin::from`. - /// A demonstration of such a poor impl is shown below. - /// - /// ```compile_fail - /// # use std::pin::Pin; - /// struct Foo; // A type defined in this crate. - /// impl From<Box<()>> for Pin<Foo> { - /// fn from(_: Box<()>) -> Pin<Foo> { - /// Pin::new(Foo) - /// } - /// } - /// - /// let foo = Box::new(()); - /// let bar = Pin::from(foo); - /// ``` - #[stable(feature = "box_into_pin", since = "1.63.0")] - #[rustc_const_unstable(feature = "const_box", issue = "92521")] - pub const fn into_pin(boxed: Self) -> Pin<Self> - where - A: 'static, - { - // It's not possible to move or replace the insides of a `Pin<Box<T>>` - // when `T: !Unpin`, so it's safe to pin it directly without any - // additional requirements. - unsafe { Pin::new_unchecked(boxed) } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -unsafe impl<#[may_dangle] T: ?Sized, A: Allocator> Drop for Box<T, A> { - fn drop(&mut self) { - // FIXME: Do nothing, drop is currently performed by compiler. - } -} - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "rust1", since = "1.0.0")] -impl<T: Default> Default for Box<T> { - /// Creates a `Box<T>`, with the `Default` value for T. - fn default() -> Self { - #[rustc_box] - Box::new(T::default()) - } -} - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "rust1", since = "1.0.0")] -#[rustc_const_unstable(feature = "const_default_impls", issue = "87864")] -impl<T> const Default for Box<[T]> { - fn default() -> Self { - let ptr: Unique<[T]> = Unique::<[T; 0]>::dangling(); - Box(ptr, Global) - } -} - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "default_box_extra", since = "1.17.0")] -#[rustc_const_unstable(feature = "const_default_impls", issue = "87864")] -impl const Default for Box<str> { - fn default() -> Self { - // SAFETY: This is the same as `Unique::cast<U>` but with an unsized `U = str`. - let ptr: Unique<str> = unsafe { - let bytes: Unique<[u8]> = Unique::<[u8; 0]>::dangling(); - Unique::new_unchecked(bytes.as_ptr() as *mut str) - }; - Box(ptr, Global) - } -} - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "rust1", since = "1.0.0")] -impl<T: Clone, A: Allocator + Clone> Clone for Box<T, A> { - /// Returns a new box with a `clone()` of this box's contents. - /// - /// # Examples - /// - /// ``` - /// let x = Box::new(5); - /// let y = x.clone(); - /// - /// // The value is the same - /// assert_eq!(x, y); - /// - /// // But they are unique objects - /// assert_ne!(&*x as *const i32, &*y as *const i32); - /// ``` - #[inline] - fn clone(&self) -> Self { - // Pre-allocate memory to allow writing the cloned value directly. - let mut boxed = Self::new_uninit_in(self.1.clone()); - unsafe { - (**self).write_clone_into_raw(boxed.as_mut_ptr()); - boxed.assume_init() - } - } - - /// Copies `source`'s contents into `self` without creating a new allocation. - /// - /// # Examples - /// - /// ``` - /// let x = Box::new(5); - /// let mut y = Box::new(10); - /// let yp: *const i32 = &*y; - /// - /// y.clone_from(&x); - /// - /// // The value is the same - /// assert_eq!(x, y); - /// - /// // And no allocation occurred - /// assert_eq!(yp, &*y); - /// ``` - #[inline] - fn clone_from(&mut self, source: &Self) { - (**self).clone_from(&(**source)); - } -} - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "box_slice_clone", since = "1.3.0")] -impl Clone for Box<str> { - fn clone(&self) -> Self { - // this makes a copy of the data - let buf: Box<[u8]> = self.as_bytes().into(); - unsafe { from_boxed_utf8_unchecked(buf) } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<T: ?Sized + PartialEq, A: Allocator> PartialEq for Box<T, A> { - #[inline] - fn eq(&self, other: &Self) -> bool { - PartialEq::eq(&**self, &**other) - } - #[inline] - fn ne(&self, other: &Self) -> bool { - PartialEq::ne(&**self, &**other) - } -} -#[stable(feature = "rust1", since = "1.0.0")] -impl<T: ?Sized + PartialOrd, A: Allocator> PartialOrd for Box<T, A> { - #[inline] - fn partial_cmp(&self, other: &Self) -> Option<Ordering> { - PartialOrd::partial_cmp(&**self, &**other) - } - #[inline] - fn lt(&self, other: &Self) -> bool { - PartialOrd::lt(&**self, &**other) - } - #[inline] - fn le(&self, other: &Self) -> bool { - PartialOrd::le(&**self, &**other) - } - #[inline] - fn ge(&self, other: &Self) -> bool { - PartialOrd::ge(&**self, &**other) - } - #[inline] - fn gt(&self, other: &Self) -> bool { - PartialOrd::gt(&**self, &**other) - } -} -#[stable(feature = "rust1", since = "1.0.0")] -impl<T: ?Sized + Ord, A: Allocator> Ord for Box<T, A> { - #[inline] - fn cmp(&self, other: &Self) -> Ordering { - Ord::cmp(&**self, &**other) - } -} -#[stable(feature = "rust1", since = "1.0.0")] -impl<T: ?Sized + Eq, A: Allocator> Eq for Box<T, A> {} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<T: ?Sized + Hash, A: Allocator> Hash for Box<T, A> { - fn hash<H: Hasher>(&self, state: &mut H) { - (**self).hash(state); - } -} - -#[stable(feature = "indirect_hasher_impl", since = "1.22.0")] -impl<T: ?Sized + Hasher, A: Allocator> Hasher for Box<T, A> { - fn finish(&self) -> u64 { - (**self).finish() - } - fn write(&mut self, bytes: &[u8]) { - (**self).write(bytes) - } - fn write_u8(&mut self, i: u8) { - (**self).write_u8(i) - } - fn write_u16(&mut self, i: u16) { - (**self).write_u16(i) - } - fn write_u32(&mut self, i: u32) { - (**self).write_u32(i) - } - fn write_u64(&mut self, i: u64) { - (**self).write_u64(i) - } - fn write_u128(&mut self, i: u128) { - (**self).write_u128(i) - } - fn write_usize(&mut self, i: usize) { - (**self).write_usize(i) - } - fn write_i8(&mut self, i: i8) { - (**self).write_i8(i) - } - fn write_i16(&mut self, i: i16) { - (**self).write_i16(i) - } - fn write_i32(&mut self, i: i32) { - (**self).write_i32(i) - } - fn write_i64(&mut self, i: i64) { - (**self).write_i64(i) - } - fn write_i128(&mut self, i: i128) { - (**self).write_i128(i) - } - fn write_isize(&mut self, i: isize) { - (**self).write_isize(i) - } - fn write_length_prefix(&mut self, len: usize) { - (**self).write_length_prefix(len) - } - fn write_str(&mut self, s: &str) { - (**self).write_str(s) - } -} - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "from_for_ptrs", since = "1.6.0")] -impl<T> From<T> for Box<T> { - /// Converts a `T` into a `Box<T>` - /// - /// The conversion allocates on the heap and moves `t` - /// from the stack into it. - /// - /// # Examples - /// - /// ```rust - /// let x = 5; - /// let boxed = Box::new(5); - /// - /// assert_eq!(Box::from(x), boxed); - /// ``` - fn from(t: T) -> Self { - Box::new(t) - } -} - -#[stable(feature = "pin", since = "1.33.0")] -#[rustc_const_unstable(feature = "const_box", issue = "92521")] -impl<T: ?Sized, A: Allocator> const From<Box<T, A>> for Pin<Box<T, A>> -where - A: 'static, -{ - /// Converts a `Box<T>` into a `Pin<Box<T>>`. If `T` does not implement [`Unpin`], then - /// `*boxed` will be pinned in memory and unable to be moved. - /// - /// This conversion does not allocate on the heap and happens in place. - /// - /// This is also available via [`Box::into_pin`]. - /// - /// Constructing and pinning a `Box` with <code><Pin<Box\<T>>>::from([Box::new]\(x))</code> - /// can also be written more concisely using <code>[Box::pin]\(x)</code>. - /// This `From` implementation is useful if you already have a `Box<T>`, or you are - /// constructing a (pinned) `Box` in a different way than with [`Box::new`]. - fn from(boxed: Box<T, A>) -> Self { - Box::into_pin(boxed) - } -} - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "box_from_slice", since = "1.17.0")] -impl<T: Copy> From<&[T]> for Box<[T]> { - /// Converts a `&[T]` into a `Box<[T]>` - /// - /// This conversion allocates on the heap - /// and performs a copy of `slice` and its contents. - /// - /// # Examples - /// ```rust - /// // create a &[u8] which will be used to create a Box<[u8]> - /// let slice: &[u8] = &[104, 101, 108, 108, 111]; - /// let boxed_slice: Box<[u8]> = Box::from(slice); - /// - /// println!("{boxed_slice:?}"); - /// ``` - fn from(slice: &[T]) -> Box<[T]> { - let len = slice.len(); - let buf = RawVec::with_capacity(len); - unsafe { - ptr::copy_nonoverlapping(slice.as_ptr(), buf.ptr(), len); - buf.into_box(slice.len()).assume_init() - } - } -} - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "box_from_cow", since = "1.45.0")] -impl<T: Copy> From<Cow<'_, [T]>> for Box<[T]> { - /// Converts a `Cow<'_, [T]>` into a `Box<[T]>` - /// - /// When `cow` is the `Cow::Borrowed` variant, this - /// conversion allocates on the heap and copies the - /// underlying slice. Otherwise, it will try to reuse the owned - /// `Vec`'s allocation. - #[inline] - fn from(cow: Cow<'_, [T]>) -> Box<[T]> { - match cow { - Cow::Borrowed(slice) => Box::from(slice), - Cow::Owned(slice) => Box::from(slice), - } - } -} - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "box_from_slice", since = "1.17.0")] -impl From<&str> for Box<str> { - /// Converts a `&str` into a `Box<str>` - /// - /// This conversion allocates on the heap - /// and performs a copy of `s`. - /// - /// # Examples - /// - /// ```rust - /// let boxed: Box<str> = Box::from("hello"); - /// println!("{boxed}"); - /// ``` - #[inline] - fn from(s: &str) -> Box<str> { - unsafe { from_boxed_utf8_unchecked(Box::from(s.as_bytes())) } - } -} - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "box_from_cow", since = "1.45.0")] -impl From<Cow<'_, str>> for Box<str> { - /// Converts a `Cow<'_, str>` into a `Box<str>` - /// - /// When `cow` is the `Cow::Borrowed` variant, this - /// conversion allocates on the heap and copies the - /// underlying `str`. Otherwise, it will try to reuse the owned - /// `String`'s allocation. - /// - /// # Examples - /// - /// ```rust - /// use std::borrow::Cow; - /// - /// let unboxed = Cow::Borrowed("hello"); - /// let boxed: Box<str> = Box::from(unboxed); - /// println!("{boxed}"); - /// ``` - /// - /// ```rust - /// # use std::borrow::Cow; - /// let unboxed = Cow::Owned("hello".to_string()); - /// let boxed: Box<str> = Box::from(unboxed); - /// println!("{boxed}"); - /// ``` - #[inline] - fn from(cow: Cow<'_, str>) -> Box<str> { - match cow { - Cow::Borrowed(s) => Box::from(s), - Cow::Owned(s) => Box::from(s), - } - } -} - -#[stable(feature = "boxed_str_conv", since = "1.19.0")] -impl<A: Allocator> From<Box<str, A>> for Box<[u8], A> { - /// Converts a `Box<str>` into a `Box<[u8]>` - /// - /// This conversion does not allocate on the heap and happens in place. - /// - /// # Examples - /// ```rust - /// // create a Box<str> which will be used to create a Box<[u8]> - /// let boxed: Box<str> = Box::from("hello"); - /// let boxed_str: Box<[u8]> = Box::from(boxed); - /// - /// // create a &[u8] which will be used to create a Box<[u8]> - /// let slice: &[u8] = &[104, 101, 108, 108, 111]; - /// let boxed_slice = Box::from(slice); - /// - /// assert_eq!(boxed_slice, boxed_str); - /// ``` - #[inline] - fn from(s: Box<str, A>) -> Self { - let (raw, alloc) = Box::into_raw_with_allocator(s); - unsafe { Box::from_raw_in(raw as *mut [u8], alloc) } - } -} - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "box_from_array", since = "1.45.0")] -impl<T, const N: usize> From<[T; N]> for Box<[T]> { - /// Converts a `[T; N]` into a `Box<[T]>` - /// - /// This conversion moves the array to newly heap-allocated memory. - /// - /// # Examples - /// - /// ```rust - /// let boxed: Box<[u8]> = Box::from([4, 2]); - /// println!("{boxed:?}"); - /// ``` - fn from(array: [T; N]) -> Box<[T]> { - #[rustc_box] - Box::new(array) - } -} - -/// Casts a boxed slice to a boxed array. -/// -/// # Safety -/// -/// `boxed_slice.len()` must be exactly `N`. -unsafe fn boxed_slice_as_array_unchecked<T, A: Allocator, const N: usize>( - boxed_slice: Box<[T], A>, -) -> Box<[T; N], A> { - debug_assert_eq!(boxed_slice.len(), N); - - let (ptr, alloc) = Box::into_raw_with_allocator(boxed_slice); - // SAFETY: Pointer and allocator came from an existing box, - // and our safety condition requires that the length is exactly `N` - unsafe { Box::from_raw_in(ptr as *mut [T; N], alloc) } -} - -#[stable(feature = "boxed_slice_try_from", since = "1.43.0")] -impl<T, const N: usize> TryFrom<Box<[T]>> for Box<[T; N]> { - type Error = Box<[T]>; - - /// Attempts to convert a `Box<[T]>` into a `Box<[T; N]>`. - /// - /// The conversion occurs in-place and does not require a - /// new memory allocation. - /// - /// # Errors - /// - /// Returns the old `Box<[T]>` in the `Err` variant if - /// `boxed_slice.len()` does not equal `N`. - fn try_from(boxed_slice: Box<[T]>) -> Result<Self, Self::Error> { - if boxed_slice.len() == N { - Ok(unsafe { boxed_slice_as_array_unchecked(boxed_slice) }) - } else { - Err(boxed_slice) - } - } -} - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "boxed_array_try_from_vec", since = "1.66.0")] -impl<T, const N: usize> TryFrom<Vec<T>> for Box<[T; N]> { - type Error = Vec<T>; - - /// Attempts to convert a `Vec<T>` into a `Box<[T; N]>`. - /// - /// Like [`Vec::into_boxed_slice`], this is in-place if `vec.capacity() == N`, - /// but will require a reallocation otherwise. - /// - /// # Errors - /// - /// Returns the original `Vec<T>` in the `Err` variant if - /// `boxed_slice.len()` does not equal `N`. - /// - /// # Examples - /// - /// This can be used with [`vec!`] to create an array on the heap: - /// - /// ``` - /// let state: Box<[f32; 100]> = vec![1.0; 100].try_into().unwrap(); - /// assert_eq!(state.len(), 100); - /// ``` - fn try_from(vec: Vec<T>) -> Result<Self, Self::Error> { - if vec.len() == N { - let boxed_slice = vec.into_boxed_slice(); - Ok(unsafe { boxed_slice_as_array_unchecked(boxed_slice) }) - } else { - Err(vec) - } - } -} - -impl<A: Allocator> Box<dyn Any, A> { - /// Attempt to downcast the box to a concrete type. - /// - /// # Examples - /// - /// ``` - /// use std::any::Any; - /// - /// fn print_if_string(value: Box<dyn Any>) { - /// if let Ok(string) = value.downcast::<String>() { - /// println!("String ({}): {}", string.len(), string); - /// } - /// } - /// - /// let my_string = "Hello World".to_string(); - /// print_if_string(Box::new(my_string)); - /// print_if_string(Box::new(0i8)); - /// ``` - #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - pub fn downcast<T: Any>(self) -> Result<Box<T, A>, Self> { - if self.is::<T>() { unsafe { Ok(self.downcast_unchecked::<T>()) } } else { Err(self) } - } - - /// Downcasts the box to a concrete type. - /// - /// For a safe alternative see [`downcast`]. - /// - /// # Examples - /// - /// ``` - /// #![feature(downcast_unchecked)] - /// - /// use std::any::Any; - /// - /// let x: Box<dyn Any> = Box::new(1_usize); - /// - /// unsafe { - /// assert_eq!(*x.downcast_unchecked::<usize>(), 1); - /// } - /// ``` - /// - /// # Safety - /// - /// The contained value must be of type `T`. Calling this method - /// with the incorrect type is *undefined behavior*. - /// - /// [`downcast`]: Self::downcast - #[inline] - #[unstable(feature = "downcast_unchecked", issue = "90850")] - pub unsafe fn downcast_unchecked<T: Any>(self) -> Box<T, A> { - debug_assert!(self.is::<T>()); - unsafe { - let (raw, alloc): (*mut dyn Any, _) = Box::into_raw_with_allocator(self); - Box::from_raw_in(raw as *mut T, alloc) - } - } -} - -impl<A: Allocator> Box<dyn Any + Send, A> { - /// Attempt to downcast the box to a concrete type. - /// - /// # Examples - /// - /// ``` - /// use std::any::Any; - /// - /// fn print_if_string(value: Box<dyn Any + Send>) { - /// if let Ok(string) = value.downcast::<String>() { - /// println!("String ({}): {}", string.len(), string); - /// } - /// } - /// - /// let my_string = "Hello World".to_string(); - /// print_if_string(Box::new(my_string)); - /// print_if_string(Box::new(0i8)); - /// ``` - #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - pub fn downcast<T: Any>(self) -> Result<Box<T, A>, Self> { - if self.is::<T>() { unsafe { Ok(self.downcast_unchecked::<T>()) } } else { Err(self) } - } - - /// Downcasts the box to a concrete type. - /// - /// For a safe alternative see [`downcast`]. - /// - /// # Examples - /// - /// ``` - /// #![feature(downcast_unchecked)] - /// - /// use std::any::Any; - /// - /// let x: Box<dyn Any + Send> = Box::new(1_usize); - /// - /// unsafe { - /// assert_eq!(*x.downcast_unchecked::<usize>(), 1); - /// } - /// ``` - /// - /// # Safety - /// - /// The contained value must be of type `T`. Calling this method - /// with the incorrect type is *undefined behavior*. - /// - /// [`downcast`]: Self::downcast - #[inline] - #[unstable(feature = "downcast_unchecked", issue = "90850")] - pub unsafe fn downcast_unchecked<T: Any>(self) -> Box<T, A> { - debug_assert!(self.is::<T>()); - unsafe { - let (raw, alloc): (*mut (dyn Any + Send), _) = Box::into_raw_with_allocator(self); - Box::from_raw_in(raw as *mut T, alloc) - } - } -} - -impl<A: Allocator> Box<dyn Any + Send + Sync, A> { - /// Attempt to downcast the box to a concrete type. - /// - /// # Examples - /// - /// ``` - /// use std::any::Any; - /// - /// fn print_if_string(value: Box<dyn Any + Send + Sync>) { - /// if let Ok(string) = value.downcast::<String>() { - /// println!("String ({}): {}", string.len(), string); - /// } - /// } - /// - /// let my_string = "Hello World".to_string(); - /// print_if_string(Box::new(my_string)); - /// print_if_string(Box::new(0i8)); - /// ``` - #[inline] - #[stable(feature = "box_send_sync_any_downcast", since = "1.51.0")] - pub fn downcast<T: Any>(self) -> Result<Box<T, A>, Self> { - if self.is::<T>() { unsafe { Ok(self.downcast_unchecked::<T>()) } } else { Err(self) } - } - - /// Downcasts the box to a concrete type. - /// - /// For a safe alternative see [`downcast`]. - /// - /// # Examples - /// - /// ``` - /// #![feature(downcast_unchecked)] - /// - /// use std::any::Any; - /// - /// let x: Box<dyn Any + Send + Sync> = Box::new(1_usize); - /// - /// unsafe { - /// assert_eq!(*x.downcast_unchecked::<usize>(), 1); - /// } - /// ``` - /// - /// # Safety - /// - /// The contained value must be of type `T`. Calling this method - /// with the incorrect type is *undefined behavior*. - /// - /// [`downcast`]: Self::downcast - #[inline] - #[unstable(feature = "downcast_unchecked", issue = "90850")] - pub unsafe fn downcast_unchecked<T: Any>(self) -> Box<T, A> { - debug_assert!(self.is::<T>()); - unsafe { - let (raw, alloc): (*mut (dyn Any + Send + Sync), _) = - Box::into_raw_with_allocator(self); - Box::from_raw_in(raw as *mut T, alloc) - } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<T: fmt::Display + ?Sized, A: Allocator> fmt::Display for Box<T, A> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(&**self, f) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<T: fmt::Debug + ?Sized, A: Allocator> fmt::Debug for Box<T, A> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Debug::fmt(&**self, f) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<T: ?Sized, A: Allocator> fmt::Pointer for Box<T, A> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - // It's not possible to extract the inner Uniq directly from the Box, - // instead we cast it to a *const which aliases the Unique - let ptr: *const T = &**self; - fmt::Pointer::fmt(&ptr, f) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -#[rustc_const_unstable(feature = "const_box", issue = "92521")] -impl<T: ?Sized, A: Allocator> const Deref for Box<T, A> { - type Target = T; - - fn deref(&self) -> &T { - &**self - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -#[rustc_const_unstable(feature = "const_box", issue = "92521")] -impl<T: ?Sized, A: Allocator> const DerefMut for Box<T, A> { - fn deref_mut(&mut self) -> &mut T { - &mut **self - } -} - -#[unstable(feature = "receiver_trait", issue = "none")] -impl<T: ?Sized, A: Allocator> Receiver for Box<T, A> {} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<I: Iterator + ?Sized, A: Allocator> Iterator for Box<I, A> { - type Item = I::Item; - fn next(&mut self) -> Option<I::Item> { - (**self).next() - } - fn size_hint(&self) -> (usize, Option<usize>) { - (**self).size_hint() - } - fn nth(&mut self, n: usize) -> Option<I::Item> { - (**self).nth(n) - } - fn last(self) -> Option<I::Item> { - BoxIter::last(self) - } -} - -trait BoxIter { - type Item; - fn last(self) -> Option<Self::Item>; -} - -impl<I: Iterator + ?Sized, A: Allocator> BoxIter for Box<I, A> { - type Item = I::Item; - default fn last(self) -> Option<I::Item> { - #[inline] - fn some<T>(_: Option<T>, x: T) -> Option<T> { - Some(x) - } - - self.fold(None, some) - } -} - -/// Specialization for sized `I`s that uses `I`s implementation of `last()` -/// instead of the default. -#[stable(feature = "rust1", since = "1.0.0")] -impl<I: Iterator, A: Allocator> BoxIter for Box<I, A> { - fn last(self) -> Option<I::Item> { - (*self).last() - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<I: DoubleEndedIterator + ?Sized, A: Allocator> DoubleEndedIterator for Box<I, A> { - fn next_back(&mut self) -> Option<I::Item> { - (**self).next_back() - } - fn nth_back(&mut self, n: usize) -> Option<I::Item> { - (**self).nth_back(n) - } -} -#[stable(feature = "rust1", since = "1.0.0")] -impl<I: ExactSizeIterator + ?Sized, A: Allocator> ExactSizeIterator for Box<I, A> { - fn len(&self) -> usize { - (**self).len() - } - fn is_empty(&self) -> bool { - (**self).is_empty() - } -} - -#[stable(feature = "fused", since = "1.26.0")] -impl<I: FusedIterator + ?Sized, A: Allocator> FusedIterator for Box<I, A> {} - -#[stable(feature = "boxed_closure_impls", since = "1.35.0")] -impl<Args: Tuple, F: FnOnce<Args> + ?Sized, A: Allocator> FnOnce<Args> for Box<F, A> { - type Output = <F as FnOnce<Args>>::Output; - - extern "rust-call" fn call_once(self, args: Args) -> Self::Output { - <F as FnOnce<Args>>::call_once(*self, args) - } -} - -#[stable(feature = "boxed_closure_impls", since = "1.35.0")] -impl<Args: Tuple, F: FnMut<Args> + ?Sized, A: Allocator> FnMut<Args> for Box<F, A> { - extern "rust-call" fn call_mut(&mut self, args: Args) -> Self::Output { - <F as FnMut<Args>>::call_mut(self, args) - } -} - -#[stable(feature = "boxed_closure_impls", since = "1.35.0")] -impl<Args: Tuple, F: Fn<Args> + ?Sized, A: Allocator> Fn<Args> for Box<F, A> { - extern "rust-call" fn call(&self, args: Args) -> Self::Output { - <F as Fn<Args>>::call(self, args) - } -} - -#[unstable(feature = "coerce_unsized", issue = "18598")] -impl<T: ?Sized + Unsize<U>, U: ?Sized, A: Allocator> CoerceUnsized<Box<U, A>> for Box<T, A> {} - -#[unstable(feature = "dispatch_from_dyn", issue = "none")] -impl<T: ?Sized + Unsize<U>, U: ?Sized> DispatchFromDyn<Box<U>> for Box<T, Global> {} - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "boxed_slice_from_iter", since = "1.32.0")] -impl<I> FromIterator<I> for Box<[I]> { - fn from_iter<T: IntoIterator<Item = I>>(iter: T) -> Self { - iter.into_iter().collect::<Vec<_>>().into_boxed_slice() - } -} - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "box_slice_clone", since = "1.3.0")] -impl<T: Clone, A: Allocator + Clone> Clone for Box<[T], A> { - fn clone(&self) -> Self { - let alloc = Box::allocator(self).clone(); - self.to_vec_in(alloc).into_boxed_slice() - } - - fn clone_from(&mut self, other: &Self) { - if self.len() == other.len() { - self.clone_from_slice(&other); - } else { - *self = other.clone(); - } - } -} - -#[stable(feature = "box_borrow", since = "1.1.0")] -impl<T: ?Sized, A: Allocator> borrow::Borrow<T> for Box<T, A> { - fn borrow(&self) -> &T { - &**self - } -} - -#[stable(feature = "box_borrow", since = "1.1.0")] -impl<T: ?Sized, A: Allocator> borrow::BorrowMut<T> for Box<T, A> { - fn borrow_mut(&mut self) -> &mut T { - &mut **self - } -} - -#[stable(since = "1.5.0", feature = "smart_ptr_as_ref")] -impl<T: ?Sized, A: Allocator> AsRef<T> for Box<T, A> { - fn as_ref(&self) -> &T { - &**self - } -} - -#[stable(since = "1.5.0", feature = "smart_ptr_as_ref")] -impl<T: ?Sized, A: Allocator> AsMut<T> for Box<T, A> { - fn as_mut(&mut self) -> &mut T { - &mut **self - } -} - -/* Nota bene - * - * We could have chosen not to add this impl, and instead have written a - * function of Pin<Box<T>> to Pin<T>. Such a function would not be sound, - * because Box<T> implements Unpin even when T does not, as a result of - * this impl. - * - * We chose this API instead of the alternative for a few reasons: - * - Logically, it is helpful to understand pinning in regard to the - * memory region being pointed to. For this reason none of the - * standard library pointer types support projecting through a pin - * (Box<T> is the only pointer type in std for which this would be - * safe.) - * - It is in practice very useful to have Box<T> be unconditionally - * Unpin because of trait objects, for which the structural auto - * trait functionality does not apply (e.g., Box<dyn Foo> would - * otherwise not be Unpin). - * - * Another type with the same semantics as Box but only a conditional - * implementation of `Unpin` (where `T: Unpin`) would be valid/safe, and - * could have a method to project a Pin<T> from it. - */ -#[stable(feature = "pin", since = "1.33.0")] -impl<T: ?Sized, A: Allocator> Unpin for Box<T, A> where A: 'static {} - -#[unstable(feature = "generator_trait", issue = "43122")] -impl<G: ?Sized + Generator<R> + Unpin, R, A: Allocator> Generator<R> for Box<G, A> -where - A: 'static, -{ - type Yield = G::Yield; - type Return = G::Return; - - fn resume(mut self: Pin<&mut Self>, arg: R) -> GeneratorState<Self::Yield, Self::Return> { - G::resume(Pin::new(&mut *self), arg) - } -} - -#[unstable(feature = "generator_trait", issue = "43122")] -impl<G: ?Sized + Generator<R>, R, A: Allocator> Generator<R> for Pin<Box<G, A>> -where - A: 'static, -{ - type Yield = G::Yield; - type Return = G::Return; - - fn resume(mut self: Pin<&mut Self>, arg: R) -> GeneratorState<Self::Yield, Self::Return> { - G::resume((*self).as_mut(), arg) - } -} - -#[stable(feature = "futures_api", since = "1.36.0")] -impl<F: ?Sized + Future + Unpin, A: Allocator> Future for Box<F, A> -where - A: 'static, -{ - type Output = F::Output; - - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> { - F::poll(Pin::new(&mut *self), cx) - } -} - -#[unstable(feature = "async_iterator", issue = "79024")] -impl<S: ?Sized + AsyncIterator + Unpin> AsyncIterator for Box<S> { - type Item = S::Item; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> { - Pin::new(&mut **self).poll_next(cx) - } - - fn size_hint(&self) -> (usize, Option<usize>) { - (**self).size_hint() - } -} - -impl dyn Error { - #[inline] - #[stable(feature = "error_downcast", since = "1.3.0")] - #[rustc_allow_incoherent_impl] - /// Attempts to downcast the box to a concrete type. - pub fn downcast<T: Error + 'static>(self: Box<Self>) -> Result<Box<T>, Box<dyn Error>> { - if self.is::<T>() { - unsafe { - let raw: *mut dyn Error = Box::into_raw(self); - Ok(Box::from_raw(raw as *mut T)) - } - } else { - Err(self) - } - } -} - -impl dyn Error + Send { - #[inline] - #[stable(feature = "error_downcast", since = "1.3.0")] - #[rustc_allow_incoherent_impl] - /// Attempts to downcast the box to a concrete type. - pub fn downcast<T: Error + 'static>(self: Box<Self>) -> Result<Box<T>, Box<dyn Error + Send>> { - let err: Box<dyn Error> = self; - <dyn Error>::downcast(err).map_err(|s| unsafe { - // Reapply the `Send` marker. - mem::transmute::<Box<dyn Error>, Box<dyn Error + Send>>(s) - }) - } -} - -impl dyn Error + Send + Sync { - #[inline] - #[stable(feature = "error_downcast", since = "1.3.0")] - #[rustc_allow_incoherent_impl] - /// Attempts to downcast the box to a concrete type. - pub fn downcast<T: Error + 'static>(self: Box<Self>) -> Result<Box<T>, Box<Self>> { - let err: Box<dyn Error> = self; - <dyn Error>::downcast(err).map_err(|s| unsafe { - // Reapply the `Send + Sync` marker. - mem::transmute::<Box<dyn Error>, Box<dyn Error + Send + Sync>>(s) - }) - } -} - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a, E: Error + 'a> From<E> for Box<dyn Error + 'a> { - /// Converts a type of [`Error`] into a box of dyn [`Error`]. - /// - /// # Examples - /// - /// ``` - /// use std::error::Error; - /// use std::fmt; - /// use std::mem; - /// - /// #[derive(Debug)] - /// struct AnError; - /// - /// impl fmt::Display for AnError { - /// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - /// write!(f, "An error") - /// } - /// } - /// - /// impl Error for AnError {} - /// - /// let an_error = AnError; - /// assert!(0 == mem::size_of_val(&an_error)); - /// let a_boxed_error = Box::<dyn Error>::from(an_error); - /// assert!(mem::size_of::<Box<dyn Error>>() == mem::size_of_val(&a_boxed_error)) - /// ``` - fn from(err: E) -> Box<dyn Error + 'a> { - Box::new(err) - } -} - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a, E: Error + Send + Sync + 'a> From<E> for Box<dyn Error + Send + Sync + 'a> { - /// Converts a type of [`Error`] + [`Send`] + [`Sync`] into a box of - /// dyn [`Error`] + [`Send`] + [`Sync`]. - /// - /// # Examples - /// - /// ``` - /// use std::error::Error; - /// use std::fmt; - /// use std::mem; - /// - /// #[derive(Debug)] - /// struct AnError; - /// - /// impl fmt::Display for AnError { - /// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - /// write!(f, "An error") - /// } - /// } - /// - /// impl Error for AnError {} - /// - /// unsafe impl Send for AnError {} - /// - /// unsafe impl Sync for AnError {} - /// - /// let an_error = AnError; - /// assert!(0 == mem::size_of_val(&an_error)); - /// let a_boxed_error = Box::<dyn Error + Send + Sync>::from(an_error); - /// assert!( - /// mem::size_of::<Box<dyn Error + Send + Sync>>() == mem::size_of_val(&a_boxed_error)) - /// ``` - fn from(err: E) -> Box<dyn Error + Send + Sync + 'a> { - Box::new(err) - } -} - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "rust1", since = "1.0.0")] -impl From<String> for Box<dyn Error + Send + Sync> { - /// Converts a [`String`] into a box of dyn [`Error`] + [`Send`] + [`Sync`]. - /// - /// # Examples - /// - /// ``` - /// use std::error::Error; - /// use std::mem; - /// - /// let a_string_error = "a string error".to_string(); - /// let a_boxed_error = Box::<dyn Error + Send + Sync>::from(a_string_error); - /// assert!( - /// mem::size_of::<Box<dyn Error + Send + Sync>>() == mem::size_of_val(&a_boxed_error)) - /// ``` - #[inline] - fn from(err: String) -> Box<dyn Error + Send + Sync> { - struct StringError(String); - - impl Error for StringError { - #[allow(deprecated)] - fn description(&self) -> &str { - &self.0 - } - } - - impl fmt::Display for StringError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(&self.0, f) - } - } - - // Purposefully skip printing "StringError(..)" - impl fmt::Debug for StringError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Debug::fmt(&self.0, f) - } - } - - Box::new(StringError(err)) - } -} - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "string_box_error", since = "1.6.0")] -impl From<String> for Box<dyn Error> { - /// Converts a [`String`] into a box of dyn [`Error`]. - /// - /// # Examples - /// - /// ``` - /// use std::error::Error; - /// use std::mem; - /// - /// let a_string_error = "a string error".to_string(); - /// let a_boxed_error = Box::<dyn Error>::from(a_string_error); - /// assert!(mem::size_of::<Box<dyn Error>>() == mem::size_of_val(&a_boxed_error)) - /// ``` - fn from(str_err: String) -> Box<dyn Error> { - let err1: Box<dyn Error + Send + Sync> = From::from(str_err); - let err2: Box<dyn Error> = err1; - err2 - } -} - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a> From<&str> for Box<dyn Error + Send + Sync + 'a> { - /// Converts a [`str`] into a box of dyn [`Error`] + [`Send`] + [`Sync`]. - /// - /// [`str`]: prim@str - /// - /// # Examples - /// - /// ``` - /// use std::error::Error; - /// use std::mem; - /// - /// let a_str_error = "a str error"; - /// let a_boxed_error = Box::<dyn Error + Send + Sync>::from(a_str_error); - /// assert!( - /// mem::size_of::<Box<dyn Error + Send + Sync>>() == mem::size_of_val(&a_boxed_error)) - /// ``` - #[inline] - fn from(err: &str) -> Box<dyn Error + Send + Sync + 'a> { - From::from(String::from(err)) - } -} - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "string_box_error", since = "1.6.0")] -impl From<&str> for Box<dyn Error> { - /// Converts a [`str`] into a box of dyn [`Error`]. - /// - /// [`str`]: prim@str - /// - /// # Examples - /// - /// ``` - /// use std::error::Error; - /// use std::mem; - /// - /// let a_str_error = "a str error"; - /// let a_boxed_error = Box::<dyn Error>::from(a_str_error); - /// assert!(mem::size_of::<Box<dyn Error>>() == mem::size_of_val(&a_boxed_error)) - /// ``` - fn from(err: &str) -> Box<dyn Error> { - From::from(String::from(err)) - } -} - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "cow_box_error", since = "1.22.0")] -impl<'a, 'b> From<Cow<'b, str>> for Box<dyn Error + Send + Sync + 'a> { - /// Converts a [`Cow`] into a box of dyn [`Error`] + [`Send`] + [`Sync`]. - /// - /// # Examples - /// - /// ``` - /// use std::error::Error; - /// use std::mem; - /// use std::borrow::Cow; - /// - /// let a_cow_str_error = Cow::from("a str error"); - /// let a_boxed_error = Box::<dyn Error + Send + Sync>::from(a_cow_str_error); - /// assert!( - /// mem::size_of::<Box<dyn Error + Send + Sync>>() == mem::size_of_val(&a_boxed_error)) - /// ``` - fn from(err: Cow<'b, str>) -> Box<dyn Error + Send + Sync + 'a> { - From::from(String::from(err)) - } -} - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "cow_box_error", since = "1.22.0")] -impl<'a> From<Cow<'a, str>> for Box<dyn Error> { - /// Converts a [`Cow`] into a box of dyn [`Error`]. - /// - /// # Examples - /// - /// ``` - /// use std::error::Error; - /// use std::mem; - /// use std::borrow::Cow; - /// - /// let a_cow_str_error = Cow::from("a str error"); - /// let a_boxed_error = Box::<dyn Error>::from(a_cow_str_error); - /// assert!(mem::size_of::<Box<dyn Error>>() == mem::size_of_val(&a_boxed_error)) - /// ``` - fn from(err: Cow<'a, str>) -> Box<dyn Error> { - From::from(String::from(err)) - } -} - -#[stable(feature = "box_error", since = "1.8.0")] -impl<T: core::error::Error> core::error::Error for Box<T> { - #[allow(deprecated, deprecated_in_future)] - fn description(&self) -> &str { - core::error::Error::description(&**self) - } - - #[allow(deprecated)] - fn cause(&self) -> Option<&dyn core::error::Error> { - core::error::Error::cause(&**self) - } - - fn source(&self) -> Option<&(dyn core::error::Error + 'static)> { - core::error::Error::source(&**self) - } -} diff --git a/rust/alloc/collections/mod.rs b/rust/alloc/collections/mod.rs deleted file mode 100644 index 2506065d158a..000000000000 --- a/rust/alloc/collections/mod.rs +++ /dev/null @@ -1,159 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 OR MIT - -//! Collection types. - -#![stable(feature = "rust1", since = "1.0.0")] - -#[cfg(not(no_global_oom_handling))] -pub mod binary_heap; -#[cfg(not(no_global_oom_handling))] -mod btree; -#[cfg(not(no_global_oom_handling))] -pub mod linked_list; -#[cfg(not(no_global_oom_handling))] -pub mod vec_deque; - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "rust1", since = "1.0.0")] -pub mod btree_map { - //! An ordered map based on a B-Tree. - #[stable(feature = "rust1", since = "1.0.0")] - pub use super::btree::map::*; -} - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "rust1", since = "1.0.0")] -pub mod btree_set { - //! An ordered set based on a B-Tree. - #[stable(feature = "rust1", since = "1.0.0")] - pub use super::btree::set::*; -} - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "rust1", since = "1.0.0")] -#[doc(no_inline)] -pub use binary_heap::BinaryHeap; - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "rust1", since = "1.0.0")] -#[doc(no_inline)] -pub use btree_map::BTreeMap; - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "rust1", since = "1.0.0")] -#[doc(no_inline)] -pub use btree_set::BTreeSet; - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "rust1", since = "1.0.0")] -#[doc(no_inline)] -pub use linked_list::LinkedList; - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "rust1", since = "1.0.0")] -#[doc(no_inline)] -pub use vec_deque::VecDeque; - -use crate::alloc::{Layout, LayoutError}; -use core::fmt::Display; - -/// The error type for `try_reserve` methods. -#[derive(Clone, PartialEq, Eq, Debug)] -#[stable(feature = "try_reserve", since = "1.57.0")] -pub struct TryReserveError { - kind: TryReserveErrorKind, -} - -impl TryReserveError { - /// Details about the allocation that caused the error - #[inline] - #[must_use] - #[unstable( - feature = "try_reserve_kind", - reason = "Uncertain how much info should be exposed", - issue = "48043" - )] - pub fn kind(&self) -> TryReserveErrorKind { - self.kind.clone() - } -} - -/// Details of the allocation that caused a `TryReserveError` -#[derive(Clone, PartialEq, Eq, Debug)] -#[unstable( - feature = "try_reserve_kind", - reason = "Uncertain how much info should be exposed", - issue = "48043" -)] -pub enum TryReserveErrorKind { - /// Error due to the computed capacity exceeding the collection's maximum - /// (usually `isize::MAX` bytes). - CapacityOverflow, - - /// The memory allocator returned an error - AllocError { - /// The layout of allocation request that failed - layout: Layout, - - #[doc(hidden)] - #[unstable( - feature = "container_error_extra", - issue = "none", - reason = "\ - Enable exposing the allocator’s custom error value \ - if an associated type is added in the future: \ - https://github.com/rust-lang/wg-allocators/issues/23" - )] - non_exhaustive: (), - }, -} - -#[unstable( - feature = "try_reserve_kind", - reason = "Uncertain how much info should be exposed", - issue = "48043" -)] -impl From<TryReserveErrorKind> for TryReserveError { - #[inline] - fn from(kind: TryReserveErrorKind) -> Self { - Self { kind } - } -} - -#[unstable(feature = "try_reserve_kind", reason = "new API", issue = "48043")] -impl From<LayoutError> for TryReserveErrorKind { - /// Always evaluates to [`TryReserveErrorKind::CapacityOverflow`]. - #[inline] - fn from(_: LayoutError) -> Self { - TryReserveErrorKind::CapacityOverflow - } -} - -#[stable(feature = "try_reserve", since = "1.57.0")] -impl Display for TryReserveError { - fn fmt( - &self, - fmt: &mut core::fmt::Formatter<'_>, - ) -> core::result::Result<(), core::fmt::Error> { - fmt.write_str("memory allocation failed")?; - let reason = match self.kind { - TryReserveErrorKind::CapacityOverflow => { - " because the computed capacity exceeded the collection's maximum" - } - TryReserveErrorKind::AllocError { .. } => { - " because the memory allocator returned an error" - } - }; - fmt.write_str(reason) - } -} - -/// An intermediate trait for specialization of `Extend`. -#[doc(hidden)] -trait SpecExtend<I: IntoIterator> { - /// Extends `self` with the contents of the given iterator. - fn spec_extend(&mut self, iter: I); -} - -#[stable(feature = "try_reserve", since = "1.57.0")] -impl core::error::Error for TryReserveError {} diff --git a/rust/alloc/lib.rs b/rust/alloc/lib.rs deleted file mode 100644 index 5f374378b0d4..000000000000 --- a/rust/alloc/lib.rs +++ /dev/null @@ -1,284 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 OR MIT - -//! # The Rust core allocation and collections library -//! -//! This library provides smart pointers and collections for managing -//! heap-allocated values. -//! -//! This library, like core, normally doesn’t need to be used directly -//! since its contents are re-exported in the [`std` crate](../std/index.html). -//! Crates that use the `#![no_std]` attribute however will typically -//! not depend on `std`, so they’d use this crate instead. -//! -//! ## Boxed values -//! -//! The [`Box`] type is a smart pointer type. There can only be one owner of a -//! [`Box`], and the owner can decide to mutate the contents, which live on the -//! heap. -//! -//! This type can be sent among threads efficiently as the size of a `Box` value -//! is the same as that of a pointer. Tree-like data structures are often built -//! with boxes because each node often has only one owner, the parent. -//! -//! ## Reference counted pointers -//! -//! The [`Rc`] type is a non-threadsafe reference-counted pointer type intended -//! for sharing memory within a thread. An [`Rc`] pointer wraps a type, `T`, and -//! only allows access to `&T`, a shared reference. -//! -//! This type is useful when inherited mutability (such as using [`Box`]) is too -//! constraining for an application, and is often paired with the [`Cell`] or -//! [`RefCell`] types in order to allow mutation. -//! -//! ## Atomically reference counted pointers -//! -//! The [`Arc`] type is the threadsafe equivalent of the [`Rc`] type. It -//! provides all the same functionality of [`Rc`], except it requires that the -//! contained type `T` is shareable. Additionally, [`Arc<T>`][`Arc`] is itself -//! sendable while [`Rc<T>`][`Rc`] is not. -//! -//! This type allows for shared access to the contained data, and is often -//! paired with synchronization primitives such as mutexes to allow mutation of -//! shared resources. -//! -//! ## Collections -//! -//! Implementations of the most common general purpose data structures are -//! defined in this library. They are re-exported through the -//! [standard collections library](../std/collections/index.html). -//! -//! ## Heap interfaces -//! -//! The [`alloc`](alloc/index.html) module defines the low-level interface to the -//! default global allocator. It is not compatible with the libc allocator API. -//! -//! [`Arc`]: sync -//! [`Box`]: boxed -//! [`Cell`]: core::cell -//! [`Rc`]: rc -//! [`RefCell`]: core::cell - -#![allow(unused_attributes)] -#![stable(feature = "alloc", since = "1.36.0")] -#![doc( - html_playground_url = "https://play.rust-lang.org/", - issue_tracker_base_url = "https://github.com/rust-lang/rust/issues/", - test(no_crate_inject, attr(allow(unused_variables), deny(warnings))) -)] -#![doc(cfg_hide( - not(test), - not(any(test, bootstrap)), - any(not(feature = "miri-test-libstd"), test, doctest), - no_global_oom_handling, - not(no_global_oom_handling), - not(no_rc), - not(no_sync), - target_has_atomic = "ptr" -))] -#![no_std] -#![needs_allocator] -// To run alloc tests without x.py without ending up with two copies of alloc, Miri needs to be -// able to "empty" this crate. See <https://github.com/rust-lang/miri-test-libstd/issues/4>. -// rustc itself never sets the feature, so this line has no affect there. -#![cfg(any(not(feature = "miri-test-libstd"), test, doctest))] -// -// Lints: -#![deny(unsafe_op_in_unsafe_fn)] -#![deny(fuzzy_provenance_casts)] -#![warn(deprecated_in_future)] -#![warn(missing_debug_implementations)] -#![warn(missing_docs)] -#![allow(explicit_outlives_requirements)] -// -// Library features: -#![feature(alloc_layout_extra)] -#![feature(allocator_api)] -#![feature(array_chunks)] -#![feature(array_into_iter_constructors)] -#![feature(array_methods)] -#![feature(array_windows)] -#![feature(assert_matches)] -#![feature(async_iterator)] -#![feature(coerce_unsized)] -#![cfg_attr(not(no_global_oom_handling), feature(const_alloc_error))] -#![feature(const_box)] -#![cfg_attr(not(no_global_oom_handling), feature(const_btree_len))] -#![cfg_attr(not(no_borrow), feature(const_cow_is_borrowed))] -#![feature(const_convert)] -#![feature(const_size_of_val)] -#![feature(const_align_of_val)] -#![feature(const_ptr_read)] -#![feature(const_maybe_uninit_zeroed)] -#![feature(const_maybe_uninit_write)] -#![feature(const_maybe_uninit_as_mut_ptr)] -#![feature(const_refs_to_cell)] -#![feature(core_intrinsics)] -#![feature(core_panic)] -#![feature(const_eval_select)] -#![feature(const_pin)] -#![feature(const_waker)] -#![feature(cstr_from_bytes_until_nul)] -#![feature(dispatch_from_dyn)] -#![feature(error_generic_member_access)] -#![feature(error_in_core)] -#![feature(exact_size_is_empty)] -#![feature(extend_one)] -#![feature(fmt_internals)] -#![feature(fn_traits)] -#![feature(hasher_prefixfree_extras)] -#![feature(inline_const)] -#![feature(inplace_iteration)] -#![cfg_attr(test, feature(is_sorted))] -#![feature(iter_advance_by)] -#![feature(iter_next_chunk)] -#![feature(iter_repeat_n)] -#![feature(layout_for_ptr)] -#![feature(maybe_uninit_slice)] -#![feature(maybe_uninit_uninit_array)] -#![feature(maybe_uninit_uninit_array_transpose)] -#![cfg_attr(test, feature(new_uninit))] -#![feature(nonnull_slice_from_raw_parts)] -#![feature(pattern)] -#![feature(pointer_byte_offsets)] -#![feature(provide_any)] -#![feature(ptr_internals)] -#![feature(ptr_metadata)] -#![feature(ptr_sub_ptr)] -#![feature(receiver_trait)] -#![feature(saturating_int_impl)] -#![feature(set_ptr_value)] -#![feature(sized_type_properties)] -#![feature(slice_from_ptr_range)] -#![feature(slice_group_by)] -#![feature(slice_ptr_get)] -#![feature(slice_ptr_len)] -#![feature(slice_range)] -#![feature(str_internals)] -#![feature(strict_provenance)] -#![feature(trusted_len)] -#![feature(trusted_random_access)] -#![feature(try_trait_v2)] -#![feature(tuple_trait)] -#![feature(unchecked_math)] -#![feature(unicode_internals)] -#![feature(unsize)] -#![feature(utf8_chunks)] -#![feature(std_internals)] -// -// Language features: -#![feature(allocator_internals)] -#![feature(allow_internal_unstable)] -#![feature(associated_type_bounds)] -#![feature(cfg_sanitize)] -#![feature(const_deref)] -#![feature(const_mut_refs)] -#![feature(const_ptr_write)] -#![feature(const_precise_live_drops)] -#![feature(const_trait_impl)] -#![feature(const_try)] -#![feature(dropck_eyepatch)] -#![feature(exclusive_range_pattern)] -#![feature(fundamental)] -#![cfg_attr(not(test), feature(generator_trait))] -#![feature(hashmap_internals)] -#![feature(lang_items)] -#![feature(min_specialization)] -#![feature(negative_impls)] -#![feature(never_type)] -#![feature(rustc_allow_const_fn_unstable)] -#![feature(rustc_attrs)] -#![feature(pointer_is_aligned)] -#![feature(slice_internals)] -#![feature(staged_api)] -#![feature(stmt_expr_attributes)] -#![cfg_attr(test, feature(test))] -#![feature(unboxed_closures)] -#![feature(unsized_fn_params)] -#![feature(c_unwind)] -#![feature(with_negative_coherence)] -#![cfg_attr(test, feature(panic_update_hook))] -// -// Rustdoc features: -#![feature(doc_cfg)] -#![feature(doc_cfg_hide)] -// Technically, this is a bug in rustdoc: rustdoc sees the documentation on `#[lang = slice_alloc]` -// blocks is for `&[T]`, which also has documentation using this feature in `core`, and gets mad -// that the feature-gate isn't enabled. Ideally, it wouldn't check for the feature gate for docs -// from other crates, but since this can only appear for lang items, it doesn't seem worth fixing. -#![feature(intra_doc_pointers)] - -// Allow testing this library -#[cfg(test)] -#[macro_use] -extern crate std; -#[cfg(test)] -extern crate test; -#[cfg(test)] -mod testing; - -// Module with internal macros used by other modules (needs to be included before other modules). -#[cfg(not(no_macros))] -#[macro_use] -mod macros; - -mod raw_vec; - -// Heaps provided for low-level allocation strategies - -pub mod alloc; - -// Primitive types using the heaps above - -// Need to conditionally define the mod from `boxed.rs` to avoid -// duplicating the lang-items when building in test cfg; but also need -// to allow code to have `use boxed::Box;` declarations. -#[cfg(not(test))] -pub mod boxed; -#[cfg(test)] -mod boxed { - pub use std::boxed::Box; -} -#[cfg(not(no_borrow))] -pub mod borrow; -pub mod collections; -#[cfg(all(not(no_rc), not(no_sync), not(no_global_oom_handling)))] -pub mod ffi; -#[cfg(not(no_fmt))] -pub mod fmt; -#[cfg(not(no_rc))] -pub mod rc; -pub mod slice; -#[cfg(not(no_str))] -pub mod str; -#[cfg(not(no_string))] -pub mod string; -#[cfg(all(not(no_rc), not(no_sync), target_has_atomic = "ptr"))] -pub mod sync; -#[cfg(all(not(no_global_oom_handling), not(no_rc), not(no_sync), target_has_atomic = "ptr"))] -pub mod task; -#[cfg(test)] -mod tests; -pub mod vec; - -#[doc(hidden)] -#[unstable(feature = "liballoc_internals", issue = "none", reason = "implementation detail")] -pub mod __export { - pub use core::format_args; -} - -#[cfg(test)] -#[allow(dead_code)] // Not used in all configurations -pub(crate) mod test_helpers { - /// Copied from `std::test_helpers::test_rng`, since these tests rely on the - /// seed not being the same for every RNG invocation too. - pub(crate) fn test_rng() -> rand_xorshift::XorShiftRng { - use std::hash::{BuildHasher, Hash, Hasher}; - let mut hasher = std::collections::hash_map::RandomState::new().build_hasher(); - std::panic::Location::caller().hash(&mut hasher); - let hc64 = hasher.finish(); - let seed_vec = - hc64.to_le_bytes().into_iter().chain(0u8..8).collect::<crate::vec::Vec<u8>>(); - let seed: [u8; 16] = seed_vec.as_slice().try_into().unwrap(); - rand::SeedableRng::from_seed(seed) - } -} diff --git a/rust/alloc/raw_vec.rs b/rust/alloc/raw_vec.rs deleted file mode 100644 index 5db87eac53b7..000000000000 --- a/rust/alloc/raw_vec.rs +++ /dev/null @@ -1,558 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 OR MIT - -#![unstable(feature = "raw_vec_internals", reason = "unstable const warnings", issue = "none")] - -use core::alloc::LayoutError; -use core::cmp; -use core::intrinsics; -use core::mem::{self, ManuallyDrop, MaybeUninit, SizedTypeProperties}; -use core::ops::Drop; -use core::ptr::{self, NonNull, Unique}; -use core::slice; - -#[cfg(not(no_global_oom_handling))] -use crate::alloc::handle_alloc_error; -use crate::alloc::{Allocator, Global, Layout}; -use crate::boxed::Box; -use crate::collections::TryReserveError; -use crate::collections::TryReserveErrorKind::*; - -#[cfg(test)] -mod tests; - -enum AllocInit { - /// The contents of the new memory are uninitialized. - Uninitialized, - /// The new memory is guaranteed to be zeroed. - #[allow(dead_code)] - Zeroed, -} - -/// A low-level utility for more ergonomically allocating, reallocating, and deallocating -/// a buffer of memory on the heap without having to worry about all the corner cases -/// involved. This type is excellent for building your own data structures like Vec and VecDeque. -/// In particular: -/// -/// * Produces `Unique::dangling()` on zero-sized types. -/// * Produces `Unique::dangling()` on zero-length allocations. -/// * Avoids freeing `Unique::dangling()`. -/// * Catches all overflows in capacity computations (promotes them to "capacity overflow" panics). -/// * Guards against 32-bit systems allocating more than isize::MAX bytes. -/// * Guards against overflowing your length. -/// * Calls `handle_alloc_error` for fallible allocations. -/// * Contains a `ptr::Unique` and thus endows the user with all related benefits. -/// * Uses the excess returned from the allocator to use the largest available capacity. -/// -/// This type does not in anyway inspect the memory that it manages. When dropped it *will* -/// free its memory, but it *won't* try to drop its contents. It is up to the user of `RawVec` -/// to handle the actual things *stored* inside of a `RawVec`. -/// -/// Note that the excess of a zero-sized types is always infinite, so `capacity()` always returns -/// `usize::MAX`. This means that you need to be careful when round-tripping this type with a -/// `Box<[T]>`, since `capacity()` won't yield the length. -#[allow(missing_debug_implementations)] -pub(crate) struct RawVec<T, A: Allocator = Global> { - ptr: Unique<T>, - cap: usize, - alloc: A, -} - -impl<T> RawVec<T, Global> { - /// HACK(Centril): This exists because stable `const fn` can only call stable `const fn`, so - /// they cannot call `Self::new()`. - /// - /// If you change `RawVec<T>::new` or dependencies, please take care to not introduce anything - /// that would truly const-call something unstable. - pub const NEW: Self = Self::new(); - - /// Creates the biggest possible `RawVec` (on the system heap) - /// without allocating. If `T` has positive size, then this makes a - /// `RawVec` with capacity `0`. If `T` is zero-sized, then it makes a - /// `RawVec` with capacity `usize::MAX`. Useful for implementing - /// delayed allocation. - #[must_use] - pub const fn new() -> Self { - Self::new_in(Global) - } - - /// Creates a `RawVec` (on the system heap) with exactly the - /// capacity and alignment requirements for a `[T; capacity]`. This is - /// equivalent to calling `RawVec::new` when `capacity` is `0` or `T` is - /// zero-sized. Note that if `T` is zero-sized this means you will - /// *not* get a `RawVec` with the requested capacity. - /// - /// # Panics - /// - /// Panics if the requested capacity exceeds `isize::MAX` bytes. - /// - /// # Aborts - /// - /// Aborts on OOM. - #[cfg(not(any(no_global_oom_handling, test)))] - #[must_use] - #[inline] - pub fn with_capacity(capacity: usize) -> Self { - Self::with_capacity_in(capacity, Global) - } - - /// Like `with_capacity`, but guarantees the buffer is zeroed. - #[cfg(not(any(no_global_oom_handling, test)))] - #[must_use] - #[inline] - pub fn with_capacity_zeroed(capacity: usize) -> Self { - Self::with_capacity_zeroed_in(capacity, Global) - } -} - -impl<T, A: Allocator> RawVec<T, A> { - // Tiny Vecs are dumb. Skip to: - // - 8 if the element size is 1, because any heap allocators is likely - // to round up a request of less than 8 bytes to at least 8 bytes. - // - 4 if elements are moderate-sized (<= 1 KiB). - // - 1 otherwise, to avoid wasting too much space for very short Vecs. - pub(crate) const MIN_NON_ZERO_CAP: usize = if mem::size_of::<T>() == 1 { - 8 - } else if mem::size_of::<T>() <= 1024 { - 4 - } else { - 1 - }; - - /// Like `new`, but parameterized over the choice of allocator for - /// the returned `RawVec`. - pub const fn new_in(alloc: A) -> Self { - // `cap: 0` means "unallocated". zero-sized types are ignored. - Self { ptr: Unique::dangling(), cap: 0, alloc } - } - - /// Like `with_capacity`, but parameterized over the choice of - /// allocator for the returned `RawVec`. - #[cfg(not(no_global_oom_handling))] - #[inline] - pub fn with_capacity_in(capacity: usize, alloc: A) -> Self { - Self::allocate_in(capacity, AllocInit::Uninitialized, alloc) - } - - /// Like `try_with_capacity`, but parameterized over the choice of - /// allocator for the returned `RawVec`. - #[inline] - pub fn try_with_capacity_in(capacity: usize, alloc: A) -> Result<Self, TryReserveError> { - Self::try_allocate_in(capacity, AllocInit::Uninitialized, alloc) - } - - /// Like `with_capacity_zeroed`, but parameterized over the choice - /// of allocator for the returned `RawVec`. - #[cfg(not(no_global_oom_handling))] - #[inline] - pub fn with_capacity_zeroed_in(capacity: usize, alloc: A) -> Self { - Self::allocate_in(capacity, AllocInit::Zeroed, alloc) - } - - /// Converts the entire buffer into `Box<[MaybeUninit<T>]>` with the specified `len`. - /// - /// Note that this will correctly reconstitute any `cap` changes - /// that may have been performed. (See description of type for details.) - /// - /// # Safety - /// - /// * `len` must be greater than or equal to the most recently requested capacity, and - /// * `len` must be less than or equal to `self.capacity()`. - /// - /// Note, that the requested capacity and `self.capacity()` could differ, as - /// an allocator could overallocate and return a greater memory block than requested. - pub unsafe fn into_box(self, len: usize) -> Box<[MaybeUninit<T>], A> { - // Sanity-check one half of the safety requirement (we cannot check the other half). - debug_assert!( - len <= self.capacity(), - "`len` must be smaller than or equal to `self.capacity()`" - ); - - let me = ManuallyDrop::new(self); - unsafe { - let slice = slice::from_raw_parts_mut(me.ptr() as *mut MaybeUninit<T>, len); - Box::from_raw_in(slice, ptr::read(&me.alloc)) - } - } - - #[cfg(not(no_global_oom_handling))] - fn allocate_in(capacity: usize, init: AllocInit, alloc: A) -> Self { - // Don't allocate here because `Drop` will not deallocate when `capacity` is 0. - if T::IS_ZST || capacity == 0 { - Self::new_in(alloc) - } else { - // We avoid `unwrap_or_else` here because it bloats the amount of - // LLVM IR generated. - let layout = match Layout::array::<T>(capacity) { - Ok(layout) => layout, - Err(_) => capacity_overflow(), - }; - match alloc_guard(layout.size()) { - Ok(_) => {} - Err(_) => capacity_overflow(), - } - let result = match init { - AllocInit::Uninitialized => alloc.allocate(layout), - AllocInit::Zeroed => alloc.allocate_zeroed(layout), - }; - let ptr = match result { - Ok(ptr) => ptr, - Err(_) => handle_alloc_error(layout), - }; - - // Allocators currently return a `NonNull<[u8]>` whose length - // matches the size requested. If that ever changes, the capacity - // here should change to `ptr.len() / mem::size_of::<T>()`. - Self { - ptr: unsafe { Unique::new_unchecked(ptr.cast().as_ptr()) }, - cap: capacity, - alloc, - } - } - } - - fn try_allocate_in(capacity: usize, init: AllocInit, alloc: A) -> Result<Self, TryReserveError> { - // Don't allocate here because `Drop` will not deallocate when `capacity` is 0. - if T::IS_ZST || capacity == 0 { - return Ok(Self::new_in(alloc)); - } - - let layout = Layout::array::<T>(capacity).map_err(|_| CapacityOverflow)?; - alloc_guard(layout.size())?; - let result = match init { - AllocInit::Uninitialized => alloc.allocate(layout), - AllocInit::Zeroed => alloc.allocate_zeroed(layout), - }; - let ptr = result.map_err(|_| AllocError { layout, non_exhaustive: () })?; - - // Allocators currently return a `NonNull<[u8]>` whose length - // matches the size requested. If that ever changes, the capacity - // here should change to `ptr.len() / mem::size_of::<T>()`. - Ok(Self { - ptr: unsafe { Unique::new_unchecked(ptr.cast().as_ptr()) }, - cap: capacity, - alloc, - }) - } - - /// Reconstitutes a `RawVec` from a pointer, capacity, and allocator. - /// - /// # Safety - /// - /// The `ptr` must be allocated (via the given allocator `alloc`), and with the given - /// `capacity`. - /// The `capacity` cannot exceed `isize::MAX` for sized types. (only a concern on 32-bit - /// systems). ZST vectors may have a capacity up to `usize::MAX`. - /// If the `ptr` and `capacity` come from a `RawVec` created via `alloc`, then this is - /// guaranteed. - #[inline] - pub unsafe fn from_raw_parts_in(ptr: *mut T, capacity: usize, alloc: A) -> Self { - Self { ptr: unsafe { Unique::new_unchecked(ptr) }, cap: capacity, alloc } - } - - /// Gets a raw pointer to the start of the allocation. Note that this is - /// `Unique::dangling()` if `capacity == 0` or `T` is zero-sized. In the former case, you must - /// be careful. - #[inline] - pub fn ptr(&self) -> *mut T { - self.ptr.as_ptr() - } - - /// Gets the capacity of the allocation. - /// - /// This will always be `usize::MAX` if `T` is zero-sized. - #[inline(always)] - pub fn capacity(&self) -> usize { - if T::IS_ZST { usize::MAX } else { self.cap } - } - - /// Returns a shared reference to the allocator backing this `RawVec`. - pub fn allocator(&self) -> &A { - &self.alloc - } - - fn current_memory(&self) -> Option<(NonNull<u8>, Layout)> { - if T::IS_ZST || self.cap == 0 { - None - } else { - // We have an allocated chunk of memory, so we can bypass runtime - // checks to get our current layout. - unsafe { - let layout = Layout::array::<T>(self.cap).unwrap_unchecked(); - Some((self.ptr.cast().into(), layout)) - } - } - } - - /// Ensures that the buffer contains at least enough space to hold `len + - /// additional` elements. If it doesn't already have enough capacity, will - /// reallocate enough space plus comfortable slack space to get amortized - /// *O*(1) behavior. Will limit this behavior if it would needlessly cause - /// itself to panic. - /// - /// If `len` exceeds `self.capacity()`, this may fail to actually allocate - /// the requested space. This is not really unsafe, but the unsafe - /// code *you* write that relies on the behavior of this function may break. - /// - /// This is ideal for implementing a bulk-push operation like `extend`. - /// - /// # Panics - /// - /// Panics if the new capacity exceeds `isize::MAX` bytes. - /// - /// # Aborts - /// - /// Aborts on OOM. - #[cfg(not(no_global_oom_handling))] - #[inline] - pub fn reserve(&mut self, len: usize, additional: usize) { - // Callers expect this function to be very cheap when there is already sufficient capacity. - // Therefore, we move all the resizing and error-handling logic from grow_amortized and - // handle_reserve behind a call, while making sure that this function is likely to be - // inlined as just a comparison and a call if the comparison fails. - #[cold] - fn do_reserve_and_handle<T, A: Allocator>( - slf: &mut RawVec<T, A>, - len: usize, - additional: usize, - ) { - handle_reserve(slf.grow_amortized(len, additional)); - } - - if self.needs_to_grow(len, additional) { - do_reserve_and_handle(self, len, additional); - } - } - - /// A specialized version of `reserve()` used only by the hot and - /// oft-instantiated `Vec::push()`, which does its own capacity check. - #[cfg(not(no_global_oom_handling))] - #[inline(never)] - pub fn reserve_for_push(&mut self, len: usize) { - handle_reserve(self.grow_amortized(len, 1)); - } - - /// The same as `reserve`, but returns on errors instead of panicking or aborting. - pub fn try_reserve(&mut self, len: usize, additional: usize) -> Result<(), TryReserveError> { - if self.needs_to_grow(len, additional) { - self.grow_amortized(len, additional) - } else { - Ok(()) - } - } - - /// The same as `reserve_for_push`, but returns on errors instead of panicking or aborting. - #[inline(never)] - pub fn try_reserve_for_push(&mut self, len: usize) -> Result<(), TryReserveError> { - self.grow_amortized(len, 1) - } - - /// Ensures that the buffer contains at least enough space to hold `len + - /// additional` elements. If it doesn't already, will reallocate the - /// minimum possible amount of memory necessary. Generally this will be - /// exactly the amount of memory necessary, but in principle the allocator - /// is free to give back more than we asked for. - /// - /// If `len` exceeds `self.capacity()`, this may fail to actually allocate - /// the requested space. This is not really unsafe, but the unsafe code - /// *you* write that relies on the behavior of this function may break. - /// - /// # Panics - /// - /// Panics if the new capacity exceeds `isize::MAX` bytes. - /// - /// # Aborts - /// - /// Aborts on OOM. - #[cfg(not(no_global_oom_handling))] - pub fn reserve_exact(&mut self, len: usize, additional: usize) { - handle_reserve(self.try_reserve_exact(len, additional)); - } - - /// The same as `reserve_exact`, but returns on errors instead of panicking or aborting. - pub fn try_reserve_exact( - &mut self, - len: usize, - additional: usize, - ) -> Result<(), TryReserveError> { - if self.needs_to_grow(len, additional) { self.grow_exact(len, additional) } else { Ok(()) } - } - - /// Shrinks the buffer down to the specified capacity. If the given amount - /// is 0, actually completely deallocates. - /// - /// # Panics - /// - /// Panics if the given amount is *larger* than the current capacity. - /// - /// # Aborts - /// - /// Aborts on OOM. - #[cfg(not(no_global_oom_handling))] - pub fn shrink_to_fit(&mut self, cap: usize) { - handle_reserve(self.shrink(cap)); - } -} - -impl<T, A: Allocator> RawVec<T, A> { - /// Returns if the buffer needs to grow to fulfill the needed extra capacity. - /// Mainly used to make inlining reserve-calls possible without inlining `grow`. - fn needs_to_grow(&self, len: usize, additional: usize) -> bool { - additional > self.capacity().wrapping_sub(len) - } - - fn set_ptr_and_cap(&mut self, ptr: NonNull<[u8]>, cap: usize) { - // Allocators currently return a `NonNull<[u8]>` whose length matches - // the size requested. If that ever changes, the capacity here should - // change to `ptr.len() / mem::size_of::<T>()`. - self.ptr = unsafe { Unique::new_unchecked(ptr.cast().as_ptr()) }; - self.cap = cap; - } - - // This method is usually instantiated many times. So we want it to be as - // small as possible, to improve compile times. But we also want as much of - // its contents to be statically computable as possible, to make the - // generated code run faster. Therefore, this method is carefully written - // so that all of the code that depends on `T` is within it, while as much - // of the code that doesn't depend on `T` as possible is in functions that - // are non-generic over `T`. - fn grow_amortized(&mut self, len: usize, additional: usize) -> Result<(), TryReserveError> { - // This is ensured by the calling contexts. - debug_assert!(additional > 0); - - if T::IS_ZST { - // Since we return a capacity of `usize::MAX` when `elem_size` is - // 0, getting to here necessarily means the `RawVec` is overfull. - return Err(CapacityOverflow.into()); - } - - // Nothing we can really do about these checks, sadly. - let required_cap = len.checked_add(additional).ok_or(CapacityOverflow)?; - - // This guarantees exponential growth. The doubling cannot overflow - // because `cap <= isize::MAX` and the type of `cap` is `usize`. - let cap = cmp::max(self.cap * 2, required_cap); - let cap = cmp::max(Self::MIN_NON_ZERO_CAP, cap); - - let new_layout = Layout::array::<T>(cap); - - // `finish_grow` is non-generic over `T`. - let ptr = finish_grow(new_layout, self.current_memory(), &mut self.alloc)?; - self.set_ptr_and_cap(ptr, cap); - Ok(()) - } - - // The constraints on this method are much the same as those on - // `grow_amortized`, but this method is usually instantiated less often so - // it's less critical. - fn grow_exact(&mut self, len: usize, additional: usize) -> Result<(), TryReserveError> { - if T::IS_ZST { - // Since we return a capacity of `usize::MAX` when the type size is - // 0, getting to here necessarily means the `RawVec` is overfull. - return Err(CapacityOverflow.into()); - } - - let cap = len.checked_add(additional).ok_or(CapacityOverflow)?; - let new_layout = Layout::array::<T>(cap); - - // `finish_grow` is non-generic over `T`. - let ptr = finish_grow(new_layout, self.current_memory(), &mut self.alloc)?; - self.set_ptr_and_cap(ptr, cap); - Ok(()) - } - - #[cfg(not(no_global_oom_handling))] - fn shrink(&mut self, cap: usize) -> Result<(), TryReserveError> { - assert!(cap <= self.capacity(), "Tried to shrink to a larger capacity"); - - let (ptr, layout) = if let Some(mem) = self.current_memory() { mem } else { return Ok(()) }; - - let ptr = unsafe { - // `Layout::array` cannot overflow here because it would have - // overflowed earlier when capacity was larger. - let new_layout = Layout::array::<T>(cap).unwrap_unchecked(); - self.alloc - .shrink(ptr, layout, new_layout) - .map_err(|_| AllocError { layout: new_layout, non_exhaustive: () })? - }; - self.set_ptr_and_cap(ptr, cap); - Ok(()) - } -} - -// This function is outside `RawVec` to minimize compile times. See the comment -// above `RawVec::grow_amortized` for details. (The `A` parameter isn't -// significant, because the number of different `A` types seen in practice is -// much smaller than the number of `T` types.) -#[inline(never)] -fn finish_grow<A>( - new_layout: Result<Layout, LayoutError>, - current_memory: Option<(NonNull<u8>, Layout)>, - alloc: &mut A, -) -> Result<NonNull<[u8]>, TryReserveError> -where - A: Allocator, -{ - // Check for the error here to minimize the size of `RawVec::grow_*`. - let new_layout = new_layout.map_err(|_| CapacityOverflow)?; - - alloc_guard(new_layout.size())?; - - let memory = if let Some((ptr, old_layout)) = current_memory { - debug_assert_eq!(old_layout.align(), new_layout.align()); - unsafe { - // The allocator checks for alignment equality - intrinsics::assume(old_layout.align() == new_layout.align()); - alloc.grow(ptr, old_layout, new_layout) - } - } else { - alloc.allocate(new_layout) - }; - - memory.map_err(|_| AllocError { layout: new_layout, non_exhaustive: () }.into()) -} - -unsafe impl<#[may_dangle] T, A: Allocator> Drop for RawVec<T, A> { - /// Frees the memory owned by the `RawVec` *without* trying to drop its contents. - fn drop(&mut self) { - if let Some((ptr, layout)) = self.current_memory() { - unsafe { self.alloc.deallocate(ptr, layout) } - } - } -} - -// Central function for reserve error handling. -#[cfg(not(no_global_oom_handling))] -#[inline] -fn handle_reserve(result: Result<(), TryReserveError>) { - match result.map_err(|e| e.kind()) { - Err(CapacityOverflow) => capacity_overflow(), - Err(AllocError { layout, .. }) => handle_alloc_error(layout), - Ok(()) => { /* yay */ } - } -} - -// We need to guarantee the following: -// * We don't ever allocate `> isize::MAX` byte-size objects. -// * We don't overflow `usize::MAX` and actually allocate too little. -// -// On 64-bit we just need to check for overflow since trying to allocate -// `> isize::MAX` bytes will surely fail. On 32-bit and 16-bit we need to add -// an extra guard for this in case we're running on a platform which can use -// all 4GB in user-space, e.g., PAE or x32. - -#[inline] -fn alloc_guard(alloc_size: usize) -> Result<(), TryReserveError> { - if usize::BITS < 64 && alloc_size > isize::MAX as usize { - Err(CapacityOverflow.into()) - } else { - Ok(()) - } -} - -// One central function responsible for reporting capacity overflows. This'll -// ensure that the code generation related to these panics is minimal as there's -// only one location which panics rather than a bunch throughout the module. -#[cfg(not(no_global_oom_handling))] -fn capacity_overflow() -> ! { - panic!("capacity overflow"); -} diff --git a/rust/alloc/slice.rs b/rust/alloc/slice.rs deleted file mode 100644 index 245e01590df7..000000000000 --- a/rust/alloc/slice.rs +++ /dev/null @@ -1,867 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 OR MIT - -//! Utilities for the slice primitive type. -//! -//! *[See also the slice primitive type](slice).* -//! -//! Most of the structs in this module are iterator types which can only be created -//! using a certain function. For example, `slice.iter()` yields an [`Iter`]. -//! -//! A few functions are provided to create a slice from a value reference -//! or from a raw pointer. -#![stable(feature = "rust1", since = "1.0.0")] -// Many of the usings in this module are only used in the test configuration. -// It's cleaner to just turn off the unused_imports warning than to fix them. -#![cfg_attr(test, allow(unused_imports, dead_code))] - -use core::borrow::{Borrow, BorrowMut}; -#[cfg(not(no_global_oom_handling))] -use core::cmp::Ordering::{self, Less}; -#[cfg(not(no_global_oom_handling))] -use core::mem::{self, SizedTypeProperties}; -#[cfg(not(no_global_oom_handling))] -use core::ptr; -#[cfg(not(no_global_oom_handling))] -use core::slice::sort; - -use crate::alloc::Allocator; -#[cfg(not(no_global_oom_handling))] -use crate::alloc::{self, Global}; -#[cfg(not(no_global_oom_handling))] -use crate::borrow::ToOwned; -use crate::boxed::Box; -use crate::vec::Vec; - -#[cfg(test)] -mod tests; - -#[unstable(feature = "slice_range", issue = "76393")] -pub use core::slice::range; -#[unstable(feature = "array_chunks", issue = "74985")] -pub use core::slice::ArrayChunks; -#[unstable(feature = "array_chunks", issue = "74985")] -pub use core::slice::ArrayChunksMut; -#[unstable(feature = "array_windows", issue = "75027")] -pub use core::slice::ArrayWindows; -#[stable(feature = "inherent_ascii_escape", since = "1.60.0")] -pub use core::slice::EscapeAscii; -#[stable(feature = "slice_get_slice", since = "1.28.0")] -pub use core::slice::SliceIndex; -#[stable(feature = "from_ref", since = "1.28.0")] -pub use core::slice::{from_mut, from_ref}; -#[unstable(feature = "slice_from_ptr_range", issue = "89792")] -pub use core::slice::{from_mut_ptr_range, from_ptr_range}; -#[stable(feature = "rust1", since = "1.0.0")] -pub use core::slice::{from_raw_parts, from_raw_parts_mut}; -#[stable(feature = "rust1", since = "1.0.0")] -pub use core::slice::{Chunks, Windows}; -#[stable(feature = "chunks_exact", since = "1.31.0")] -pub use core::slice::{ChunksExact, ChunksExactMut}; -#[stable(feature = "rust1", since = "1.0.0")] -pub use core::slice::{ChunksMut, Split, SplitMut}; -#[unstable(feature = "slice_group_by", issue = "80552")] -pub use core::slice::{GroupBy, GroupByMut}; -#[stable(feature = "rust1", since = "1.0.0")] -pub use core::slice::{Iter, IterMut}; -#[stable(feature = "rchunks", since = "1.31.0")] -pub use core::slice::{RChunks, RChunksExact, RChunksExactMut, RChunksMut}; -#[stable(feature = "slice_rsplit", since = "1.27.0")] -pub use core::slice::{RSplit, RSplitMut}; -#[stable(feature = "rust1", since = "1.0.0")] -pub use core::slice::{RSplitN, RSplitNMut, SplitN, SplitNMut}; -#[stable(feature = "split_inclusive", since = "1.51.0")] -pub use core::slice::{SplitInclusive, SplitInclusiveMut}; - -//////////////////////////////////////////////////////////////////////////////// -// Basic slice extension methods -//////////////////////////////////////////////////////////////////////////////// - -// HACK(japaric) needed for the implementation of `vec!` macro during testing -// N.B., see the `hack` module in this file for more details. -#[cfg(test)] -pub use hack::into_vec; - -// HACK(japaric) needed for the implementation of `Vec::clone` during testing -// N.B., see the `hack` module in this file for more details. -#[cfg(test)] -pub use hack::to_vec; - -// HACK(japaric): With cfg(test) `impl [T]` is not available, these three -// functions are actually methods that are in `impl [T]` but not in -// `core::slice::SliceExt` - we need to supply these functions for the -// `test_permutations` test -pub(crate) mod hack { - use core::alloc::Allocator; - - use crate::boxed::Box; - use crate::vec::Vec; - - // We shouldn't add inline attribute to this since this is used in - // `vec!` macro mostly and causes perf regression. See #71204 for - // discussion and perf results. - pub fn into_vec<T, A: Allocator>(b: Box<[T], A>) -> Vec<T, A> { - unsafe { - let len = b.len(); - let (b, alloc) = Box::into_raw_with_allocator(b); - Vec::from_raw_parts_in(b as *mut T, len, len, alloc) - } - } - - #[cfg(not(no_global_oom_handling))] - #[inline] - pub fn to_vec<T: ConvertVec, A: Allocator>(s: &[T], alloc: A) -> Vec<T, A> { - T::to_vec(s, alloc) - } - - #[cfg(not(no_global_oom_handling))] - pub trait ConvertVec { - fn to_vec<A: Allocator>(s: &[Self], alloc: A) -> Vec<Self, A> - where - Self: Sized; - } - - #[cfg(not(no_global_oom_handling))] - impl<T: Clone> ConvertVec for T { - #[inline] - default fn to_vec<A: Allocator>(s: &[Self], alloc: A) -> Vec<Self, A> { - struct DropGuard<'a, T, A: Allocator> { - vec: &'a mut Vec<T, A>, - num_init: usize, - } - impl<'a, T, A: Allocator> Drop for DropGuard<'a, T, A> { - #[inline] - fn drop(&mut self) { - // SAFETY: - // items were marked initialized in the loop below - unsafe { - self.vec.set_len(self.num_init); - } - } - } - let mut vec = Vec::with_capacity_in(s.len(), alloc); - let mut guard = DropGuard { vec: &mut vec, num_init: 0 }; - let slots = guard.vec.spare_capacity_mut(); - // .take(slots.len()) is necessary for LLVM to remove bounds checks - // and has better codegen than zip. - for (i, b) in s.iter().enumerate().take(slots.len()) { - guard.num_init = i; - slots[i].write(b.clone()); - } - core::mem::forget(guard); - // SAFETY: - // the vec was allocated and initialized above to at least this length. - unsafe { - vec.set_len(s.len()); - } - vec - } - } - - #[cfg(not(no_global_oom_handling))] - impl<T: Copy> ConvertVec for T { - #[inline] - fn to_vec<A: Allocator>(s: &[Self], alloc: A) -> Vec<Self, A> { - let mut v = Vec::with_capacity_in(s.len(), alloc); - // SAFETY: - // allocated above with the capacity of `s`, and initialize to `s.len()` in - // ptr::copy_to_non_overlapping below. - unsafe { - s.as_ptr().copy_to_nonoverlapping(v.as_mut_ptr(), s.len()); - v.set_len(s.len()); - } - v - } - } -} - -#[cfg(not(test))] -impl<T> [T] { - /// Sorts the slice. - /// - /// This sort is stable (i.e., does not reorder equal elements) and *O*(*n* \* log(*n*)) worst-case. - /// - /// When applicable, unstable sorting is preferred because it is generally faster than stable - /// sorting and it doesn't allocate auxiliary memory. - /// See [`sort_unstable`](slice::sort_unstable). - /// - /// # Current implementation - /// - /// The current algorithm is an adaptive, iterative merge sort inspired by - /// [timsort](https://en.wikipedia.org/wiki/Timsort). - /// It is designed to be very fast in cases where the slice is nearly sorted, or consists of - /// two or more sorted sequences concatenated one after another. - /// - /// Also, it allocates temporary storage half the size of `self`, but for short slices a - /// non-allocating insertion sort is used instead. - /// - /// # Examples - /// - /// ``` - /// let mut v = [-5, 4, 1, -3, 2]; - /// - /// v.sort(); - /// assert!(v == [-5, -3, 1, 2, 4]); - /// ``` - #[cfg(not(no_global_oom_handling))] - #[rustc_allow_incoherent_impl] - #[stable(feature = "rust1", since = "1.0.0")] - #[inline] - pub fn sort(&mut self) - where - T: Ord, - { - stable_sort(self, T::lt); - } - - /// Sorts the slice with a comparator function. - /// - /// This sort is stable (i.e., does not reorder equal elements) and *O*(*n* \* log(*n*)) worst-case. - /// - /// The comparator function must define a total ordering for the elements in the slice. If - /// the ordering is not total, the order of the elements is unspecified. An order is a - /// total order if it is (for all `a`, `b` and `c`): - /// - /// * total and antisymmetric: exactly one of `a < b`, `a == b` or `a > b` is true, and - /// * transitive, `a < b` and `b < c` implies `a < c`. The same must hold for both `==` and `>`. - /// - /// For example, while [`f64`] doesn't implement [`Ord`] because `NaN != NaN`, we can use - /// `partial_cmp` as our sort function when we know the slice doesn't contain a `NaN`. - /// - /// ``` - /// let mut floats = [5f64, 4.0, 1.0, 3.0, 2.0]; - /// floats.sort_by(|a, b| a.partial_cmp(b).unwrap()); - /// assert_eq!(floats, [1.0, 2.0, 3.0, 4.0, 5.0]); - /// ``` - /// - /// When applicable, unstable sorting is preferred because it is generally faster than stable - /// sorting and it doesn't allocate auxiliary memory. - /// See [`sort_unstable_by`](slice::sort_unstable_by). - /// - /// # Current implementation - /// - /// The current algorithm is an adaptive, iterative merge sort inspired by - /// [timsort](https://en.wikipedia.org/wiki/Timsort). - /// It is designed to be very fast in cases where the slice is nearly sorted, or consists of - /// two or more sorted sequences concatenated one after another. - /// - /// Also, it allocates temporary storage half the size of `self`, but for short slices a - /// non-allocating insertion sort is used instead. - /// - /// # Examples - /// - /// ``` - /// let mut v = [5, 4, 1, 3, 2]; - /// v.sort_by(|a, b| a.cmp(b)); - /// assert!(v == [1, 2, 3, 4, 5]); - /// - /// // reverse sorting - /// v.sort_by(|a, b| b.cmp(a)); - /// assert!(v == [5, 4, 3, 2, 1]); - /// ``` - #[cfg(not(no_global_oom_handling))] - #[rustc_allow_incoherent_impl] - #[stable(feature = "rust1", since = "1.0.0")] - #[inline] - pub fn sort_by<F>(&mut self, mut compare: F) - where - F: FnMut(&T, &T) -> Ordering, - { - stable_sort(self, |a, b| compare(a, b) == Less); - } - - /// Sorts the slice with a key extraction function. - /// - /// This sort is stable (i.e., does not reorder equal elements) and *O*(*m* \* *n* \* log(*n*)) - /// worst-case, where the key function is *O*(*m*). - /// - /// For expensive key functions (e.g. functions that are not simple property accesses or - /// basic operations), [`sort_by_cached_key`](slice::sort_by_cached_key) is likely to be - /// significantly faster, as it does not recompute element keys. - /// - /// When applicable, unstable sorting is preferred because it is generally faster than stable - /// sorting and it doesn't allocate auxiliary memory. - /// See [`sort_unstable_by_key`](slice::sort_unstable_by_key). - /// - /// # Current implementation - /// - /// The current algorithm is an adaptive, iterative merge sort inspired by - /// [timsort](https://en.wikipedia.org/wiki/Timsort). - /// It is designed to be very fast in cases where the slice is nearly sorted, or consists of - /// two or more sorted sequences concatenated one after another. - /// - /// Also, it allocates temporary storage half the size of `self`, but for short slices a - /// non-allocating insertion sort is used instead. - /// - /// # Examples - /// - /// ``` - /// let mut v = [-5i32, 4, 1, -3, 2]; - /// - /// v.sort_by_key(|k| k.abs()); - /// assert!(v == [1, 2, -3, 4, -5]); - /// ``` - #[cfg(not(no_global_oom_handling))] - #[rustc_allow_incoherent_impl] - #[stable(feature = "slice_sort_by_key", since = "1.7.0")] - #[inline] - pub fn sort_by_key<K, F>(&mut self, mut f: F) - where - F: FnMut(&T) -> K, - K: Ord, - { - stable_sort(self, |a, b| f(a).lt(&f(b))); - } - - /// Sorts the slice with a key extraction function. - /// - /// During sorting, the key function is called at most once per element, by using - /// temporary storage to remember the results of key evaluation. - /// The order of calls to the key function is unspecified and may change in future versions - /// of the standard library. - /// - /// This sort is stable (i.e., does not reorder equal elements) and *O*(*m* \* *n* + *n* \* log(*n*)) - /// worst-case, where the key function is *O*(*m*). - /// - /// For simple key functions (e.g., functions that are property accesses or - /// basic operations), [`sort_by_key`](slice::sort_by_key) is likely to be - /// faster. - /// - /// # Current implementation - /// - /// The current algorithm is based on [pattern-defeating quicksort][pdqsort] by Orson Peters, - /// which combines the fast average case of randomized quicksort with the fast worst case of - /// heapsort, while achieving linear time on slices with certain patterns. It uses some - /// randomization to avoid degenerate cases, but with a fixed seed to always provide - /// deterministic behavior. - /// - /// In the worst case, the algorithm allocates temporary storage in a `Vec<(K, usize)>` the - /// length of the slice. - /// - /// # Examples - /// - /// ``` - /// let mut v = [-5i32, 4, 32, -3, 2]; - /// - /// v.sort_by_cached_key(|k| k.to_string()); - /// assert!(v == [-3, -5, 2, 32, 4]); - /// ``` - /// - /// [pdqsort]: https://github.com/orlp/pdqsort - #[cfg(not(no_global_oom_handling))] - #[rustc_allow_incoherent_impl] - #[stable(feature = "slice_sort_by_cached_key", since = "1.34.0")] - #[inline] - pub fn sort_by_cached_key<K, F>(&mut self, f: F) - where - F: FnMut(&T) -> K, - K: Ord, - { - // Helper macro for indexing our vector by the smallest possible type, to reduce allocation. - macro_rules! sort_by_key { - ($t:ty, $slice:ident, $f:ident) => {{ - let mut indices: Vec<_> = - $slice.iter().map($f).enumerate().map(|(i, k)| (k, i as $t)).collect(); - // The elements of `indices` are unique, as they are indexed, so any sort will be - // stable with respect to the original slice. We use `sort_unstable` here because - // it requires less memory allocation. - indices.sort_unstable(); - for i in 0..$slice.len() { - let mut index = indices[i].1; - while (index as usize) < i { - index = indices[index as usize].1; - } - indices[i].1 = index; - $slice.swap(i, index as usize); - } - }}; - } - - let sz_u8 = mem::size_of::<(K, u8)>(); - let sz_u16 = mem::size_of::<(K, u16)>(); - let sz_u32 = mem::size_of::<(K, u32)>(); - let sz_usize = mem::size_of::<(K, usize)>(); - - let len = self.len(); - if len < 2 { - return; - } - if sz_u8 < sz_u16 && len <= (u8::MAX as usize) { - return sort_by_key!(u8, self, f); - } - if sz_u16 < sz_u32 && len <= (u16::MAX as usize) { - return sort_by_key!(u16, self, f); - } - if sz_u32 < sz_usize && len <= (u32::MAX as usize) { - return sort_by_key!(u32, self, f); - } - sort_by_key!(usize, self, f) - } - - /// Copies `self` into a new `Vec`. - /// - /// # Examples - /// - /// ``` - /// let s = [10, 40, 30]; - /// let x = s.to_vec(); - /// // Here, `s` and `x` can be modified independently. - /// ``` - #[cfg(not(no_global_oom_handling))] - #[rustc_allow_incoherent_impl] - #[rustc_conversion_suggestion] - #[stable(feature = "rust1", since = "1.0.0")] - #[inline] - pub fn to_vec(&self) -> Vec<T> - where - T: Clone, - { - self.to_vec_in(Global) - } - - /// Copies `self` into a new `Vec` with an allocator. - /// - /// # Examples - /// - /// ``` - /// #![feature(allocator_api)] - /// - /// use std::alloc::System; - /// - /// let s = [10, 40, 30]; - /// let x = s.to_vec_in(System); - /// // Here, `s` and `x` can be modified independently. - /// ``` - #[cfg(not(no_global_oom_handling))] - #[rustc_allow_incoherent_impl] - #[inline] - #[unstable(feature = "allocator_api", issue = "32838")] - pub fn to_vec_in<A: Allocator>(&self, alloc: A) -> Vec<T, A> - where - T: Clone, - { - // N.B., see the `hack` module in this file for more details. - hack::to_vec(self, alloc) - } - - /// Converts `self` into a vector without clones or allocation. - /// - /// The resulting vector can be converted back into a box via - /// `Vec<T>`'s `into_boxed_slice` method. - /// - /// # Examples - /// - /// ``` - /// let s: Box<[i32]> = Box::new([10, 40, 30]); - /// let x = s.into_vec(); - /// // `s` cannot be used anymore because it has been converted into `x`. - /// - /// assert_eq!(x, vec![10, 40, 30]); - /// ``` - #[rustc_allow_incoherent_impl] - #[stable(feature = "rust1", since = "1.0.0")] - #[inline] - pub fn into_vec<A: Allocator>(self: Box<Self, A>) -> Vec<T, A> { - // N.B., see the `hack` module in this file for more details. - hack::into_vec(self) - } - - /// Creates a vector by copying a slice `n` times. - /// - /// # Panics - /// - /// This function will panic if the capacity would overflow. - /// - /// # Examples - /// - /// Basic usage: - /// - /// ``` - /// assert_eq!([1, 2].repeat(3), vec![1, 2, 1, 2, 1, 2]); - /// ``` - /// - /// A panic upon overflow: - /// - /// ```should_panic - /// // this will panic at runtime - /// b"0123456789abcdef".repeat(usize::MAX); - /// ``` - #[rustc_allow_incoherent_impl] - #[cfg(not(no_global_oom_handling))] - #[stable(feature = "repeat_generic_slice", since = "1.40.0")] - pub fn repeat(&self, n: usize) -> Vec<T> - where - T: Copy, - { - if n == 0 { - return Vec::new(); - } - - // If `n` is larger than zero, it can be split as - // `n = 2^expn + rem (2^expn > rem, expn >= 0, rem >= 0)`. - // `2^expn` is the number represented by the leftmost '1' bit of `n`, - // and `rem` is the remaining part of `n`. - - // Using `Vec` to access `set_len()`. - let capacity = self.len().checked_mul(n).expect("capacity overflow"); - let mut buf = Vec::with_capacity(capacity); - - // `2^expn` repetition is done by doubling `buf` `expn`-times. - buf.extend(self); - { - let mut m = n >> 1; - // If `m > 0`, there are remaining bits up to the leftmost '1'. - while m > 0 { - // `buf.extend(buf)`: - unsafe { - ptr::copy_nonoverlapping( - buf.as_ptr(), - (buf.as_mut_ptr() as *mut T).add(buf.len()), - buf.len(), - ); - // `buf` has capacity of `self.len() * n`. - let buf_len = buf.len(); - buf.set_len(buf_len * 2); - } - - m >>= 1; - } - } - - // `rem` (`= n - 2^expn`) repetition is done by copying - // first `rem` repetitions from `buf` itself. - let rem_len = capacity - buf.len(); // `self.len() * rem` - if rem_len > 0 { - // `buf.extend(buf[0 .. rem_len])`: - unsafe { - // This is non-overlapping since `2^expn > rem`. - ptr::copy_nonoverlapping( - buf.as_ptr(), - (buf.as_mut_ptr() as *mut T).add(buf.len()), - rem_len, - ); - // `buf.len() + rem_len` equals to `buf.capacity()` (`= self.len() * n`). - buf.set_len(capacity); - } - } - buf - } - - /// Flattens a slice of `T` into a single value `Self::Output`. - /// - /// # Examples - /// - /// ``` - /// assert_eq!(["hello", "world"].concat(), "helloworld"); - /// assert_eq!([[1, 2], [3, 4]].concat(), [1, 2, 3, 4]); - /// ``` - #[rustc_allow_incoherent_impl] - #[stable(feature = "rust1", since = "1.0.0")] - pub fn concat<Item: ?Sized>(&self) -> <Self as Concat<Item>>::Output - where - Self: Concat<Item>, - { - Concat::concat(self) - } - - /// Flattens a slice of `T` into a single value `Self::Output`, placing a - /// given separator between each. - /// - /// # Examples - /// - /// ``` - /// assert_eq!(["hello", "world"].join(" "), "hello world"); - /// assert_eq!([[1, 2], [3, 4]].join(&0), [1, 2, 0, 3, 4]); - /// assert_eq!([[1, 2], [3, 4]].join(&[0, 0][..]), [1, 2, 0, 0, 3, 4]); - /// ``` - #[rustc_allow_incoherent_impl] - #[stable(feature = "rename_connect_to_join", since = "1.3.0")] - pub fn join<Separator>(&self, sep: Separator) -> <Self as Join<Separator>>::Output - where - Self: Join<Separator>, - { - Join::join(self, sep) - } - - /// Flattens a slice of `T` into a single value `Self::Output`, placing a - /// given separator between each. - /// - /// # Examples - /// - /// ``` - /// # #![allow(deprecated)] - /// assert_eq!(["hello", "world"].connect(" "), "hello world"); - /// assert_eq!([[1, 2], [3, 4]].connect(&0), [1, 2, 0, 3, 4]); - /// ``` - #[rustc_allow_incoherent_impl] - #[stable(feature = "rust1", since = "1.0.0")] - #[deprecated(since = "1.3.0", note = "renamed to join")] - pub fn connect<Separator>(&self, sep: Separator) -> <Self as Join<Separator>>::Output - where - Self: Join<Separator>, - { - Join::join(self, sep) - } -} - -#[cfg(not(test))] -impl [u8] { - /// Returns a vector containing a copy of this slice where each byte - /// is mapped to its ASCII upper case equivalent. - /// - /// ASCII letters 'a' to 'z' are mapped to 'A' to 'Z', - /// but non-ASCII letters are unchanged. - /// - /// To uppercase the value in-place, use [`make_ascii_uppercase`]. - /// - /// [`make_ascii_uppercase`]: slice::make_ascii_uppercase - #[cfg(not(no_global_oom_handling))] - #[rustc_allow_incoherent_impl] - #[must_use = "this returns the uppercase bytes as a new Vec, \ - without modifying the original"] - #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] - #[inline] - pub fn to_ascii_uppercase(&self) -> Vec<u8> { - let mut me = self.to_vec(); - me.make_ascii_uppercase(); - me - } - - /// Returns a vector containing a copy of this slice where each byte - /// is mapped to its ASCII lower case equivalent. - /// - /// ASCII letters 'A' to 'Z' are mapped to 'a' to 'z', - /// but non-ASCII letters are unchanged. - /// - /// To lowercase the value in-place, use [`make_ascii_lowercase`]. - /// - /// [`make_ascii_lowercase`]: slice::make_ascii_lowercase - #[cfg(not(no_global_oom_handling))] - #[rustc_allow_incoherent_impl] - #[must_use = "this returns the lowercase bytes as a new Vec, \ - without modifying the original"] - #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] - #[inline] - pub fn to_ascii_lowercase(&self) -> Vec<u8> { - let mut me = self.to_vec(); - me.make_ascii_lowercase(); - me - } -} - -//////////////////////////////////////////////////////////////////////////////// -// Extension traits for slices over specific kinds of data -//////////////////////////////////////////////////////////////////////////////// - -/// Helper trait for [`[T]::concat`](slice::concat). -/// -/// Note: the `Item` type parameter is not used in this trait, -/// but it allows impls to be more generic. -/// Without it, we get this error: -/// -/// ```error -/// error[E0207]: the type parameter `T` is not constrained by the impl trait, self type, or predica -/// --> library/alloc/src/slice.rs:608:6 -/// | -/// 608 | impl<T: Clone, V: Borrow<[T]>> Concat for [V] { -/// | ^ unconstrained type parameter -/// ``` -/// -/// This is because there could exist `V` types with multiple `Borrow<[_]>` impls, -/// such that multiple `T` types would apply: -/// -/// ``` -/// # #[allow(dead_code)] -/// pub struct Foo(Vec<u32>, Vec<String>); -/// -/// impl std::borrow::Borrow<[u32]> for Foo { -/// fn borrow(&self) -> &[u32] { &self.0 } -/// } -/// -/// impl std::borrow::Borrow<[String]> for Foo { -/// fn borrow(&self) -> &[String] { &self.1 } -/// } -/// ``` -#[unstable(feature = "slice_concat_trait", issue = "27747")] -pub trait Concat<Item: ?Sized> { - #[unstable(feature = "slice_concat_trait", issue = "27747")] - /// The resulting type after concatenation - type Output; - - /// Implementation of [`[T]::concat`](slice::concat) - #[unstable(feature = "slice_concat_trait", issue = "27747")] - fn concat(slice: &Self) -> Self::Output; -} - -/// Helper trait for [`[T]::join`](slice::join) -#[unstable(feature = "slice_concat_trait", issue = "27747")] -pub trait Join<Separator> { - #[unstable(feature = "slice_concat_trait", issue = "27747")] - /// The resulting type after concatenation - type Output; - - /// Implementation of [`[T]::join`](slice::join) - #[unstable(feature = "slice_concat_trait", issue = "27747")] - fn join(slice: &Self, sep: Separator) -> Self::Output; -} - -#[cfg(not(no_global_oom_handling))] -#[unstable(feature = "slice_concat_ext", issue = "27747")] -impl<T: Clone, V: Borrow<[T]>> Concat<T> for [V] { - type Output = Vec<T>; - - fn concat(slice: &Self) -> Vec<T> { - let size = slice.iter().map(|slice| slice.borrow().len()).sum(); - let mut result = Vec::with_capacity(size); - for v in slice { - result.extend_from_slice(v.borrow()) - } - result - } -} - -#[cfg(not(no_global_oom_handling))] -#[unstable(feature = "slice_concat_ext", issue = "27747")] -impl<T: Clone, V: Borrow<[T]>> Join<&T> for [V] { - type Output = Vec<T>; - - fn join(slice: &Self, sep: &T) -> Vec<T> { - let mut iter = slice.iter(); - let first = match iter.next() { - Some(first) => first, - None => return vec![], - }; - let size = slice.iter().map(|v| v.borrow().len()).sum::<usize>() + slice.len() - 1; - let mut result = Vec::with_capacity(size); - result.extend_from_slice(first.borrow()); - - for v in iter { - result.push(sep.clone()); - result.extend_from_slice(v.borrow()) - } - result - } -} - -#[cfg(not(no_global_oom_handling))] -#[unstable(feature = "slice_concat_ext", issue = "27747")] -impl<T: Clone, V: Borrow<[T]>> Join<&[T]> for [V] { - type Output = Vec<T>; - - fn join(slice: &Self, sep: &[T]) -> Vec<T> { - let mut iter = slice.iter(); - let first = match iter.next() { - Some(first) => first, - None => return vec![], - }; - let size = - slice.iter().map(|v| v.borrow().len()).sum::<usize>() + sep.len() * (slice.len() - 1); - let mut result = Vec::with_capacity(size); - result.extend_from_slice(first.borrow()); - - for v in iter { - result.extend_from_slice(sep); - result.extend_from_slice(v.borrow()) - } - result - } -} - -//////////////////////////////////////////////////////////////////////////////// -// Standard trait implementations for slices -//////////////////////////////////////////////////////////////////////////////// - -#[stable(feature = "rust1", since = "1.0.0")] -impl<T, A: Allocator> Borrow<[T]> for Vec<T, A> { - fn borrow(&self) -> &[T] { - &self[..] - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<T, A: Allocator> BorrowMut<[T]> for Vec<T, A> { - fn borrow_mut(&mut self) -> &mut [T] { - &mut self[..] - } -} - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "rust1", since = "1.0.0")] -impl<T: Clone> ToOwned for [T] { - type Owned = Vec<T>; - #[cfg(not(test))] - fn to_owned(&self) -> Vec<T> { - self.to_vec() - } - - #[cfg(test)] - fn to_owned(&self) -> Vec<T> { - hack::to_vec(self, Global) - } - - fn clone_into(&self, target: &mut Vec<T>) { - // drop anything in target that will not be overwritten - target.truncate(self.len()); - - // target.len <= self.len due to the truncate above, so the - // slices here are always in-bounds. - let (init, tail) = self.split_at(target.len()); - - // reuse the contained values' allocations/resources. - target.clone_from_slice(init); - target.extend_from_slice(tail); - } -} - -//////////////////////////////////////////////////////////////////////////////// -// Sorting -//////////////////////////////////////////////////////////////////////////////// - -#[inline] -#[cfg(not(no_global_oom_handling))] -fn stable_sort<T, F>(v: &mut [T], mut is_less: F) -where - F: FnMut(&T, &T) -> bool, -{ - if T::IS_ZST { - // Sorting has no meaningful behavior on zero-sized types. Do nothing. - return; - } - - let elem_alloc_fn = |len: usize| -> *mut T { - // SAFETY: Creating the layout is safe as long as merge_sort never calls this with len > - // v.len(). Alloc in general will only be used as 'shadow-region' to store temporary swap - // elements. - unsafe { alloc::alloc(alloc::Layout::array::<T>(len).unwrap_unchecked()) as *mut T } - }; - - let elem_dealloc_fn = |buf_ptr: *mut T, len: usize| { - // SAFETY: Creating the layout is safe as long as merge_sort never calls this with len > - // v.len(). The caller must ensure that buf_ptr was created by elem_alloc_fn with the same - // len. - unsafe { - alloc::dealloc(buf_ptr as *mut u8, alloc::Layout::array::<T>(len).unwrap_unchecked()); - } - }; - - let run_alloc_fn = |len: usize| -> *mut sort::TimSortRun { - // SAFETY: Creating the layout is safe as long as merge_sort never calls this with an - // obscene length or 0. - unsafe { - alloc::alloc(alloc::Layout::array::<sort::TimSortRun>(len).unwrap_unchecked()) - as *mut sort::TimSortRun - } - }; - - let run_dealloc_fn = |buf_ptr: *mut sort::TimSortRun, len: usize| { - // SAFETY: The caller must ensure that buf_ptr was created by elem_alloc_fn with the same - // len. - unsafe { - alloc::dealloc( - buf_ptr as *mut u8, - alloc::Layout::array::<sort::TimSortRun>(len).unwrap_unchecked(), - ); - } - }; - - sort::merge_sort(v, &mut is_less, elem_alloc_fn, elem_dealloc_fn, run_alloc_fn, run_dealloc_fn); -} diff --git a/rust/alloc/vec/drain.rs b/rust/alloc/vec/drain.rs deleted file mode 100644 index d503d2f478ce..000000000000 --- a/rust/alloc/vec/drain.rs +++ /dev/null @@ -1,257 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 OR MIT - -use crate::alloc::{Allocator, Global}; -use core::fmt; -use core::iter::{FusedIterator, TrustedLen}; -use core::mem::{self, ManuallyDrop, SizedTypeProperties}; -use core::ptr::{self, NonNull}; -use core::slice::{self}; - -use super::Vec; - -/// A draining iterator for `Vec<T>`. -/// -/// This `struct` is created by [`Vec::drain`]. -/// See its documentation for more. -/// -/// # Example -/// -/// ``` -/// let mut v = vec![0, 1, 2]; -/// let iter: std::vec::Drain<_> = v.drain(..); -/// ``` -#[stable(feature = "drain", since = "1.6.0")] -pub struct Drain< - 'a, - T: 'a, - #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator + 'a = Global, -> { - /// Index of tail to preserve - pub(super) tail_start: usize, - /// Length of tail - pub(super) tail_len: usize, - /// Current remaining range to remove - pub(super) iter: slice::Iter<'a, T>, - pub(super) vec: NonNull<Vec<T, A>>, -} - -#[stable(feature = "collection_debug", since = "1.17.0")] -impl<T: fmt::Debug, A: Allocator> fmt::Debug for Drain<'_, T, A> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_tuple("Drain").field(&self.iter.as_slice()).finish() - } -} - -impl<'a, T, A: Allocator> Drain<'a, T, A> { - /// Returns the remaining items of this iterator as a slice. - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec!['a', 'b', 'c']; - /// let mut drain = vec.drain(..); - /// assert_eq!(drain.as_slice(), &['a', 'b', 'c']); - /// let _ = drain.next().unwrap(); - /// assert_eq!(drain.as_slice(), &['b', 'c']); - /// ``` - #[must_use] - #[stable(feature = "vec_drain_as_slice", since = "1.46.0")] - pub fn as_slice(&self) -> &[T] { - self.iter.as_slice() - } - - /// Returns a reference to the underlying allocator. - #[unstable(feature = "allocator_api", issue = "32838")] - #[must_use] - #[inline] - pub fn allocator(&self) -> &A { - unsafe { self.vec.as_ref().allocator() } - } - - /// Keep unyielded elements in the source `Vec`. - /// - /// # Examples - /// - /// ``` - /// #![feature(drain_keep_rest)] - /// - /// let mut vec = vec!['a', 'b', 'c']; - /// let mut drain = vec.drain(..); - /// - /// assert_eq!(drain.next().unwrap(), 'a'); - /// - /// // This call keeps 'b' and 'c' in the vec. - /// drain.keep_rest(); - /// - /// // If we wouldn't call `keep_rest()`, - /// // `vec` would be empty. - /// assert_eq!(vec, ['b', 'c']); - /// ``` - #[unstable(feature = "drain_keep_rest", issue = "101122")] - pub fn keep_rest(self) { - // At this moment layout looks like this: - // - // [head] [yielded by next] [unyielded] [yielded by next_back] [tail] - // ^-- start \_________/-- unyielded_len \____/-- self.tail_len - // ^-- unyielded_ptr ^-- tail - // - // Normally `Drop` impl would drop [unyielded] and then move [tail] to the `start`. - // Here we want to - // 1. Move [unyielded] to `start` - // 2. Move [tail] to a new start at `start + len(unyielded)` - // 3. Update length of the original vec to `len(head) + len(unyielded) + len(tail)` - // a. In case of ZST, this is the only thing we want to do - // 4. Do *not* drop self, as everything is put in a consistent state already, there is nothing to do - let mut this = ManuallyDrop::new(self); - - unsafe { - let source_vec = this.vec.as_mut(); - - let start = source_vec.len(); - let tail = this.tail_start; - - let unyielded_len = this.iter.len(); - let unyielded_ptr = this.iter.as_slice().as_ptr(); - - // ZSTs have no identity, so we don't need to move them around. - let needs_move = mem::size_of::<T>() != 0; - - if needs_move { - let start_ptr = source_vec.as_mut_ptr().add(start); - - // memmove back unyielded elements - if unyielded_ptr != start_ptr { - let src = unyielded_ptr; - let dst = start_ptr; - - ptr::copy(src, dst, unyielded_len); - } - - // memmove back untouched tail - if tail != (start + unyielded_len) { - let src = source_vec.as_ptr().add(tail); - let dst = start_ptr.add(unyielded_len); - ptr::copy(src, dst, this.tail_len); - } - } - - source_vec.set_len(start + unyielded_len + this.tail_len); - } - } -} - -#[stable(feature = "vec_drain_as_slice", since = "1.46.0")] -impl<'a, T, A: Allocator> AsRef<[T]> for Drain<'a, T, A> { - fn as_ref(&self) -> &[T] { - self.as_slice() - } -} - -#[stable(feature = "drain", since = "1.6.0")] -unsafe impl<T: Sync, A: Sync + Allocator> Sync for Drain<'_, T, A> {} -#[stable(feature = "drain", since = "1.6.0")] -unsafe impl<T: Send, A: Send + Allocator> Send for Drain<'_, T, A> {} - -#[stable(feature = "drain", since = "1.6.0")] -impl<T, A: Allocator> Iterator for Drain<'_, T, A> { - type Item = T; - - #[inline] - fn next(&mut self) -> Option<T> { - self.iter.next().map(|elt| unsafe { ptr::read(elt as *const _) }) - } - - fn size_hint(&self) -> (usize, Option<usize>) { - self.iter.size_hint() - } -} - -#[stable(feature = "drain", since = "1.6.0")] -impl<T, A: Allocator> DoubleEndedIterator for Drain<'_, T, A> { - #[inline] - fn next_back(&mut self) -> Option<T> { - self.iter.next_back().map(|elt| unsafe { ptr::read(elt as *const _) }) - } -} - -#[stable(feature = "drain", since = "1.6.0")] -impl<T, A: Allocator> Drop for Drain<'_, T, A> { - fn drop(&mut self) { - /// Moves back the un-`Drain`ed elements to restore the original `Vec`. - struct DropGuard<'r, 'a, T, A: Allocator>(&'r mut Drain<'a, T, A>); - - impl<'r, 'a, T, A: Allocator> Drop for DropGuard<'r, 'a, T, A> { - fn drop(&mut self) { - if self.0.tail_len > 0 { - unsafe { - let source_vec = self.0.vec.as_mut(); - // memmove back untouched tail, update to new length - let start = source_vec.len(); - let tail = self.0.tail_start; - if tail != start { - let src = source_vec.as_ptr().add(tail); - let dst = source_vec.as_mut_ptr().add(start); - ptr::copy(src, dst, self.0.tail_len); - } - source_vec.set_len(start + self.0.tail_len); - } - } - } - } - - let iter = mem::replace(&mut self.iter, (&mut []).iter()); - let drop_len = iter.len(); - - let mut vec = self.vec; - - if T::IS_ZST { - // ZSTs have no identity, so we don't need to move them around, we only need to drop the correct amount. - // this can be achieved by manipulating the Vec length instead of moving values out from `iter`. - unsafe { - let vec = vec.as_mut(); - let old_len = vec.len(); - vec.set_len(old_len + drop_len + self.tail_len); - vec.truncate(old_len + self.tail_len); - } - - return; - } - - // ensure elements are moved back into their appropriate places, even when drop_in_place panics - let _guard = DropGuard(self); - - if drop_len == 0 { - return; - } - - // as_slice() must only be called when iter.len() is > 0 because - // it also gets touched by vec::Splice which may turn it into a dangling pointer - // which would make it and the vec pointer point to different allocations which would - // lead to invalid pointer arithmetic below. - let drop_ptr = iter.as_slice().as_ptr(); - - unsafe { - // drop_ptr comes from a slice::Iter which only gives us a &[T] but for drop_in_place - // a pointer with mutable provenance is necessary. Therefore we must reconstruct - // it from the original vec but also avoid creating a &mut to the front since that could - // invalidate raw pointers to it which some unsafe code might rely on. - let vec_ptr = vec.as_mut().as_mut_ptr(); - let drop_offset = drop_ptr.sub_ptr(vec_ptr); - let to_drop = ptr::slice_from_raw_parts_mut(vec_ptr.add(drop_offset), drop_len); - ptr::drop_in_place(to_drop); - } - } -} - -#[stable(feature = "drain", since = "1.6.0")] -impl<T, A: Allocator> ExactSizeIterator for Drain<'_, T, A> { - fn is_empty(&self) -> bool { - self.iter.is_empty() - } -} - -#[unstable(feature = "trusted_len", issue = "37572")] -unsafe impl<T, A: Allocator> TrustedLen for Drain<'_, T, A> {} - -#[stable(feature = "fused", since = "1.26.0")] -impl<T, A: Allocator> FusedIterator for Drain<'_, T, A> {} diff --git a/rust/alloc/vec/drain_filter.rs b/rust/alloc/vec/drain_filter.rs deleted file mode 100644 index 4b019220657d..000000000000 --- a/rust/alloc/vec/drain_filter.rs +++ /dev/null @@ -1,201 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 OR MIT - -use crate::alloc::{Allocator, Global}; -use core::mem::{self, ManuallyDrop}; -use core::ptr; -use core::slice; - -use super::Vec; - -/// An iterator which uses a closure to determine if an element should be removed. -/// -/// This struct is created by [`Vec::drain_filter`]. -/// See its documentation for more. -/// -/// # Example -/// -/// ``` -/// #![feature(drain_filter)] -/// -/// let mut v = vec![0, 1, 2]; -/// let iter: std::vec::DrainFilter<_, _> = v.drain_filter(|x| *x % 2 == 0); -/// ``` -#[unstable(feature = "drain_filter", reason = "recently added", issue = "43244")] -#[derive(Debug)] -pub struct DrainFilter< - 'a, - T, - F, - #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator = Global, -> where - F: FnMut(&mut T) -> bool, -{ - pub(super) vec: &'a mut Vec<T, A>, - /// The index of the item that will be inspected by the next call to `next`. - pub(super) idx: usize, - /// The number of items that have been drained (removed) thus far. - pub(super) del: usize, - /// The original length of `vec` prior to draining. - pub(super) old_len: usize, - /// The filter test predicate. - pub(super) pred: F, - /// A flag that indicates a panic has occurred in the filter test predicate. - /// This is used as a hint in the drop implementation to prevent consumption - /// of the remainder of the `DrainFilter`. Any unprocessed items will be - /// backshifted in the `vec`, but no further items will be dropped or - /// tested by the filter predicate. - pub(super) panic_flag: bool, -} - -impl<T, F, A: Allocator> DrainFilter<'_, T, F, A> -where - F: FnMut(&mut T) -> bool, -{ - /// Returns a reference to the underlying allocator. - #[unstable(feature = "allocator_api", issue = "32838")] - #[inline] - pub fn allocator(&self) -> &A { - self.vec.allocator() - } - - /// Keep unyielded elements in the source `Vec`. - /// - /// # Examples - /// - /// ``` - /// #![feature(drain_filter)] - /// #![feature(drain_keep_rest)] - /// - /// let mut vec = vec!['a', 'b', 'c']; - /// let mut drain = vec.drain_filter(|_| true); - /// - /// assert_eq!(drain.next().unwrap(), 'a'); - /// - /// // This call keeps 'b' and 'c' in the vec. - /// drain.keep_rest(); - /// - /// // If we wouldn't call `keep_rest()`, - /// // `vec` would be empty. - /// assert_eq!(vec, ['b', 'c']); - /// ``` - #[unstable(feature = "drain_keep_rest", issue = "101122")] - pub fn keep_rest(self) { - // At this moment layout looks like this: - // - // _____________________/-- old_len - // / \ - // [kept] [yielded] [tail] - // \_______/ ^-- idx - // \-- del - // - // Normally `Drop` impl would drop [tail] (via .for_each(drop), ie still calling `pred`) - // - // 1. Move [tail] after [kept] - // 2. Update length of the original vec to `old_len - del` - // a. In case of ZST, this is the only thing we want to do - // 3. Do *not* drop self, as everything is put in a consistent state already, there is nothing to do - let mut this = ManuallyDrop::new(self); - - unsafe { - // ZSTs have no identity, so we don't need to move them around. - let needs_move = mem::size_of::<T>() != 0; - - if needs_move && this.idx < this.old_len && this.del > 0 { - let ptr = this.vec.as_mut_ptr(); - let src = ptr.add(this.idx); - let dst = src.sub(this.del); - let tail_len = this.old_len - this.idx; - src.copy_to(dst, tail_len); - } - - let new_len = this.old_len - this.del; - this.vec.set_len(new_len); - } - } -} - -#[unstable(feature = "drain_filter", reason = "recently added", issue = "43244")] -impl<T, F, A: Allocator> Iterator for DrainFilter<'_, T, F, A> -where - F: FnMut(&mut T) -> bool, -{ - type Item = T; - - fn next(&mut self) -> Option<T> { - unsafe { - while self.idx < self.old_len { - let i = self.idx; - let v = slice::from_raw_parts_mut(self.vec.as_mut_ptr(), self.old_len); - self.panic_flag = true; - let drained = (self.pred)(&mut v[i]); - self.panic_flag = false; - // Update the index *after* the predicate is called. If the index - // is updated prior and the predicate panics, the element at this - // index would be leaked. - self.idx += 1; - if drained { - self.del += 1; - return Some(ptr::read(&v[i])); - } else if self.del > 0 { - let del = self.del; - let src: *const T = &v[i]; - let dst: *mut T = &mut v[i - del]; - ptr::copy_nonoverlapping(src, dst, 1); - } - } - None - } - } - - fn size_hint(&self) -> (usize, Option<usize>) { - (0, Some(self.old_len - self.idx)) - } -} - -#[unstable(feature = "drain_filter", reason = "recently added", issue = "43244")] -impl<T, F, A: Allocator> Drop for DrainFilter<'_, T, F, A> -where - F: FnMut(&mut T) -> bool, -{ - fn drop(&mut self) { - struct BackshiftOnDrop<'a, 'b, T, F, A: Allocator> - where - F: FnMut(&mut T) -> bool, - { - drain: &'b mut DrainFilter<'a, T, F, A>, - } - - impl<'a, 'b, T, F, A: Allocator> Drop for BackshiftOnDrop<'a, 'b, T, F, A> - where - F: FnMut(&mut T) -> bool, - { - fn drop(&mut self) { - unsafe { - if self.drain.idx < self.drain.old_len && self.drain.del > 0 { - // This is a pretty messed up state, and there isn't really an - // obviously right thing to do. We don't want to keep trying - // to execute `pred`, so we just backshift all the unprocessed - // elements and tell the vec that they still exist. The backshift - // is required to prevent a double-drop of the last successfully - // drained item prior to a panic in the predicate. - let ptr = self.drain.vec.as_mut_ptr(); - let src = ptr.add(self.drain.idx); - let dst = src.sub(self.drain.del); - let tail_len = self.drain.old_len - self.drain.idx; - src.copy_to(dst, tail_len); - } - self.drain.vec.set_len(self.drain.old_len - self.drain.del); - } - } - } - - let backshift = BackshiftOnDrop { drain: self }; - - // Attempt to consume any remaining elements if the filter predicate - // has not yet panicked. We'll backshift any remaining elements - // whether we've already panicked or if the consumption here panics. - if !backshift.drain.panic_flag { - backshift.drain.for_each(drop); - } - } -} diff --git a/rust/alloc/vec/into_iter.rs b/rust/alloc/vec/into_iter.rs deleted file mode 100644 index 34a2a70d6ded..000000000000 --- a/rust/alloc/vec/into_iter.rs +++ /dev/null @@ -1,435 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 OR MIT - -#[cfg(not(no_global_oom_handling))] -use super::AsVecIntoIter; -use crate::alloc::{Allocator, Global}; -#[cfg(not(no_global_oom_handling))] -use crate::collections::VecDeque; -use crate::raw_vec::RawVec; -use core::array; -use core::fmt; -use core::iter::{ - FusedIterator, InPlaceIterable, SourceIter, TrustedLen, TrustedRandomAccessNoCoerce, -}; -use core::marker::PhantomData; -use core::mem::{self, ManuallyDrop, MaybeUninit, SizedTypeProperties}; -#[cfg(not(no_global_oom_handling))] -use core::ops::Deref; -use core::ptr::{self, NonNull}; -use core::slice::{self}; - -/// An iterator that moves out of a vector. -/// -/// This `struct` is created by the `into_iter` method on [`Vec`](super::Vec) -/// (provided by the [`IntoIterator`] trait). -/// -/// # Example -/// -/// ``` -/// let v = vec![0, 1, 2]; -/// let iter: std::vec::IntoIter<_> = v.into_iter(); -/// ``` -#[stable(feature = "rust1", since = "1.0.0")] -#[rustc_insignificant_dtor] -pub struct IntoIter< - T, - #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator = Global, -> { - pub(super) buf: NonNull<T>, - pub(super) phantom: PhantomData<T>, - pub(super) cap: usize, - // the drop impl reconstructs a RawVec from buf, cap and alloc - // to avoid dropping the allocator twice we need to wrap it into ManuallyDrop - pub(super) alloc: ManuallyDrop<A>, - pub(super) ptr: *const T, - pub(super) end: *const T, // If T is a ZST, this is actually ptr+len. This encoding is picked so that - // ptr == end is a quick test for the Iterator being empty, that works - // for both ZST and non-ZST. -} - -#[stable(feature = "vec_intoiter_debug", since = "1.13.0")] -impl<T: fmt::Debug, A: Allocator> fmt::Debug for IntoIter<T, A> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_tuple("IntoIter").field(&self.as_slice()).finish() - } -} - -impl<T, A: Allocator> IntoIter<T, A> { - /// Returns the remaining items of this iterator as a slice. - /// - /// # Examples - /// - /// ``` - /// let vec = vec!['a', 'b', 'c']; - /// let mut into_iter = vec.into_iter(); - /// assert_eq!(into_iter.as_slice(), &['a', 'b', 'c']); - /// let _ = into_iter.next().unwrap(); - /// assert_eq!(into_iter.as_slice(), &['b', 'c']); - /// ``` - #[stable(feature = "vec_into_iter_as_slice", since = "1.15.0")] - pub fn as_slice(&self) -> &[T] { - unsafe { slice::from_raw_parts(self.ptr, self.len()) } - } - - /// Returns the remaining items of this iterator as a mutable slice. - /// - /// # Examples - /// - /// ``` - /// let vec = vec!['a', 'b', 'c']; - /// let mut into_iter = vec.into_iter(); - /// assert_eq!(into_iter.as_slice(), &['a', 'b', 'c']); - /// into_iter.as_mut_slice()[2] = 'z'; - /// assert_eq!(into_iter.next().unwrap(), 'a'); - /// assert_eq!(into_iter.next().unwrap(), 'b'); - /// assert_eq!(into_iter.next().unwrap(), 'z'); - /// ``` - #[stable(feature = "vec_into_iter_as_slice", since = "1.15.0")] - pub fn as_mut_slice(&mut self) -> &mut [T] { - unsafe { &mut *self.as_raw_mut_slice() } - } - - /// Returns a reference to the underlying allocator. - #[unstable(feature = "allocator_api", issue = "32838")] - #[inline] - pub fn allocator(&self) -> &A { - &self.alloc - } - - fn as_raw_mut_slice(&mut self) -> *mut [T] { - ptr::slice_from_raw_parts_mut(self.ptr as *mut T, self.len()) - } - - /// Drops remaining elements and relinquishes the backing allocation. - /// This method guarantees it won't panic before relinquishing - /// the backing allocation. - /// - /// This is roughly equivalent to the following, but more efficient - /// - /// ``` - /// # let mut into_iter = Vec::<u8>::with_capacity(10).into_iter(); - /// let mut into_iter = std::mem::replace(&mut into_iter, Vec::new().into_iter()); - /// (&mut into_iter).for_each(core::mem::drop); - /// std::mem::forget(into_iter); - /// ``` - /// - /// This method is used by in-place iteration, refer to the vec::in_place_collect - /// documentation for an overview. - #[cfg(not(no_global_oom_handling))] - pub(super) fn forget_allocation_drop_remaining(&mut self) { - let remaining = self.as_raw_mut_slice(); - - // overwrite the individual fields instead of creating a new - // struct and then overwriting &mut self. - // this creates less assembly - self.cap = 0; - self.buf = unsafe { NonNull::new_unchecked(RawVec::NEW.ptr()) }; - self.ptr = self.buf.as_ptr(); - self.end = self.buf.as_ptr(); - - // Dropping the remaining elements can panic, so this needs to be - // done only after updating the other fields. - unsafe { - ptr::drop_in_place(remaining); - } - } - - /// Forgets to Drop the remaining elements while still allowing the backing allocation to be freed. - pub(crate) fn forget_remaining_elements(&mut self) { - // For th ZST case, it is crucial that we mutate `end` here, not `ptr`. - // `ptr` must stay aligned, while `end` may be unaligned. - self.end = self.ptr; - } - - #[cfg(not(no_global_oom_handling))] - #[inline] - pub(crate) fn into_vecdeque(self) -> VecDeque<T, A> { - // Keep our `Drop` impl from dropping the elements and the allocator - let mut this = ManuallyDrop::new(self); - - // SAFETY: This allocation originally came from a `Vec`, so it passes - // all those checks. We have `this.buf` ≤ `this.ptr` ≤ `this.end`, - // so the `sub_ptr`s below cannot wrap, and will produce a well-formed - // range. `end` ≤ `buf + cap`, so the range will be in-bounds. - // Taking `alloc` is ok because nothing else is going to look at it, - // since our `Drop` impl isn't going to run so there's no more code. - unsafe { - let buf = this.buf.as_ptr(); - let initialized = if T::IS_ZST { - // All the pointers are the same for ZSTs, so it's fine to - // say that they're all at the beginning of the "allocation". - 0..this.len() - } else { - this.ptr.sub_ptr(buf)..this.end.sub_ptr(buf) - }; - let cap = this.cap; - let alloc = ManuallyDrop::take(&mut this.alloc); - VecDeque::from_contiguous_raw_parts_in(buf, initialized, cap, alloc) - } - } -} - -#[stable(feature = "vec_intoiter_as_ref", since = "1.46.0")] -impl<T, A: Allocator> AsRef<[T]> for IntoIter<T, A> { - fn as_ref(&self) -> &[T] { - self.as_slice() - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -unsafe impl<T: Send, A: Allocator + Send> Send for IntoIter<T, A> {} -#[stable(feature = "rust1", since = "1.0.0")] -unsafe impl<T: Sync, A: Allocator + Sync> Sync for IntoIter<T, A> {} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<T, A: Allocator> Iterator for IntoIter<T, A> { - type Item = T; - - #[inline] - fn next(&mut self) -> Option<T> { - if self.ptr == self.end { - None - } else if T::IS_ZST { - // `ptr` has to stay where it is to remain aligned, so we reduce the length by 1 by - // reducing the `end`. - self.end = self.end.wrapping_byte_sub(1); - - // Make up a value of this ZST. - Some(unsafe { mem::zeroed() }) - } else { - let old = self.ptr; - self.ptr = unsafe { self.ptr.add(1) }; - - Some(unsafe { ptr::read(old) }) - } - } - - #[inline] - fn size_hint(&self) -> (usize, Option<usize>) { - let exact = if T::IS_ZST { - self.end.addr().wrapping_sub(self.ptr.addr()) - } else { - unsafe { self.end.sub_ptr(self.ptr) } - }; - (exact, Some(exact)) - } - - #[inline] - fn advance_by(&mut self, n: usize) -> Result<(), usize> { - let step_size = self.len().min(n); - let to_drop = ptr::slice_from_raw_parts_mut(self.ptr as *mut T, step_size); - if T::IS_ZST { - // See `next` for why we sub `end` here. - self.end = self.end.wrapping_byte_sub(step_size); - } else { - // SAFETY: the min() above ensures that step_size is in bounds - self.ptr = unsafe { self.ptr.add(step_size) }; - } - // SAFETY: the min() above ensures that step_size is in bounds - unsafe { - ptr::drop_in_place(to_drop); - } - if step_size < n { - return Err(step_size); - } - Ok(()) - } - - #[inline] - fn count(self) -> usize { - self.len() - } - - #[inline] - fn next_chunk<const N: usize>(&mut self) -> Result<[T; N], core::array::IntoIter<T, N>> { - let mut raw_ary = MaybeUninit::uninit_array(); - - let len = self.len(); - - if T::IS_ZST { - if len < N { - self.forget_remaining_elements(); - // Safety: ZSTs can be conjured ex nihilo, only the amount has to be correct - return Err(unsafe { array::IntoIter::new_unchecked(raw_ary, 0..len) }); - } - - self.end = self.end.wrapping_byte_sub(N); - // Safety: ditto - return Ok(unsafe { raw_ary.transpose().assume_init() }); - } - - if len < N { - // Safety: `len` indicates that this many elements are available and we just checked that - // it fits into the array. - unsafe { - ptr::copy_nonoverlapping(self.ptr, raw_ary.as_mut_ptr() as *mut T, len); - self.forget_remaining_elements(); - return Err(array::IntoIter::new_unchecked(raw_ary, 0..len)); - } - } - - // Safety: `len` is larger than the array size. Copy a fixed amount here to fully initialize - // the array. - return unsafe { - ptr::copy_nonoverlapping(self.ptr, raw_ary.as_mut_ptr() as *mut T, N); - self.ptr = self.ptr.add(N); - Ok(raw_ary.transpose().assume_init()) - }; - } - - unsafe fn __iterator_get_unchecked(&mut self, i: usize) -> Self::Item - where - Self: TrustedRandomAccessNoCoerce, - { - // SAFETY: the caller must guarantee that `i` is in bounds of the - // `Vec<T>`, so `i` cannot overflow an `isize`, and the `self.ptr.add(i)` - // is guaranteed to pointer to an element of the `Vec<T>` and - // thus guaranteed to be valid to dereference. - // - // Also note the implementation of `Self: TrustedRandomAccess` requires - // that `T: Copy` so reading elements from the buffer doesn't invalidate - // them for `Drop`. - unsafe { - if T::IS_ZST { mem::zeroed() } else { ptr::read(self.ptr.add(i)) } - } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<T, A: Allocator> DoubleEndedIterator for IntoIter<T, A> { - #[inline] - fn next_back(&mut self) -> Option<T> { - if self.end == self.ptr { - None - } else if T::IS_ZST { - // See above for why 'ptr.offset' isn't used - self.end = self.end.wrapping_byte_sub(1); - - // Make up a value of this ZST. - Some(unsafe { mem::zeroed() }) - } else { - self.end = unsafe { self.end.sub(1) }; - - Some(unsafe { ptr::read(self.end) }) - } - } - - #[inline] - fn advance_back_by(&mut self, n: usize) -> Result<(), usize> { - let step_size = self.len().min(n); - if T::IS_ZST { - // SAFETY: same as for advance_by() - self.end = self.end.wrapping_byte_sub(step_size); - } else { - // SAFETY: same as for advance_by() - self.end = unsafe { self.end.sub(step_size) }; - } - let to_drop = ptr::slice_from_raw_parts_mut(self.end as *mut T, step_size); - // SAFETY: same as for advance_by() - unsafe { - ptr::drop_in_place(to_drop); - } - if step_size < n { - return Err(step_size); - } - Ok(()) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<T, A: Allocator> ExactSizeIterator for IntoIter<T, A> { - fn is_empty(&self) -> bool { - self.ptr == self.end - } -} - -#[stable(feature = "fused", since = "1.26.0")] -impl<T, A: Allocator> FusedIterator for IntoIter<T, A> {} - -#[unstable(feature = "trusted_len", issue = "37572")] -unsafe impl<T, A: Allocator> TrustedLen for IntoIter<T, A> {} - -#[doc(hidden)] -#[unstable(issue = "none", feature = "std_internals")] -#[rustc_unsafe_specialization_marker] -pub trait NonDrop {} - -// T: Copy as approximation for !Drop since get_unchecked does not advance self.ptr -// and thus we can't implement drop-handling -#[unstable(issue = "none", feature = "std_internals")] -impl<T: Copy> NonDrop for T {} - -#[doc(hidden)] -#[unstable(issue = "none", feature = "std_internals")] -// TrustedRandomAccess (without NoCoerce) must not be implemented because -// subtypes/supertypes of `T` might not be `NonDrop` -unsafe impl<T, A: Allocator> TrustedRandomAccessNoCoerce for IntoIter<T, A> -where - T: NonDrop, -{ - const MAY_HAVE_SIDE_EFFECT: bool = false; -} - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "vec_into_iter_clone", since = "1.8.0")] -impl<T: Clone, A: Allocator + Clone> Clone for IntoIter<T, A> { - #[cfg(not(test))] - fn clone(&self) -> Self { - self.as_slice().to_vec_in(self.alloc.deref().clone()).into_iter() - } - #[cfg(test)] - fn clone(&self) -> Self { - crate::slice::to_vec(self.as_slice(), self.alloc.deref().clone()).into_iter() - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -unsafe impl<#[may_dangle] T, A: Allocator> Drop for IntoIter<T, A> { - fn drop(&mut self) { - struct DropGuard<'a, T, A: Allocator>(&'a mut IntoIter<T, A>); - - impl<T, A: Allocator> Drop for DropGuard<'_, T, A> { - fn drop(&mut self) { - unsafe { - // `IntoIter::alloc` is not used anymore after this and will be dropped by RawVec - let alloc = ManuallyDrop::take(&mut self.0.alloc); - // RawVec handles deallocation - let _ = RawVec::from_raw_parts_in(self.0.buf.as_ptr(), self.0.cap, alloc); - } - } - } - - let guard = DropGuard(self); - // destroy the remaining elements - unsafe { - ptr::drop_in_place(guard.0.as_raw_mut_slice()); - } - // now `guard` will be dropped and do the rest - } -} - -// In addition to the SAFETY invariants of the following three unsafe traits -// also refer to the vec::in_place_collect module documentation to get an overview -#[unstable(issue = "none", feature = "inplace_iteration")] -#[doc(hidden)] -unsafe impl<T, A: Allocator> InPlaceIterable for IntoIter<T, A> {} - -#[unstable(issue = "none", feature = "inplace_iteration")] -#[doc(hidden)] -unsafe impl<T, A: Allocator> SourceIter for IntoIter<T, A> { - type Source = Self; - - #[inline] - unsafe fn as_inner(&mut self) -> &mut Self::Source { - self - } -} - -#[cfg(not(no_global_oom_handling))] -unsafe impl<T> AsVecIntoIter for IntoIter<T> { - type Item = T; - - fn as_into_iter(&mut self) -> &mut IntoIter<Self::Item> { - self - } -} diff --git a/rust/alloc/vec/is_zero.rs b/rust/alloc/vec/is_zero.rs deleted file mode 100644 index d928dcf90e80..000000000000 --- a/rust/alloc/vec/is_zero.rs +++ /dev/null @@ -1,204 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 OR MIT - -use core::num::{Saturating, Wrapping}; - -use crate::boxed::Box; - -#[rustc_specialization_trait] -pub(super) unsafe trait IsZero { - /// Whether this value's representation is all zeros, - /// or can be represented with all zeroes. - fn is_zero(&self) -> bool; -} - -macro_rules! impl_is_zero { - ($t:ty, $is_zero:expr) => { - unsafe impl IsZero for $t { - #[inline] - fn is_zero(&self) -> bool { - $is_zero(*self) - } - } - }; -} - -impl_is_zero!(i8, |x| x == 0); // It is needed to impl for arrays and tuples of i8. -impl_is_zero!(i16, |x| x == 0); -impl_is_zero!(i32, |x| x == 0); -impl_is_zero!(i64, |x| x == 0); -impl_is_zero!(i128, |x| x == 0); -impl_is_zero!(isize, |x| x == 0); - -impl_is_zero!(u8, |x| x == 0); // It is needed to impl for arrays and tuples of u8. -impl_is_zero!(u16, |x| x == 0); -impl_is_zero!(u32, |x| x == 0); -impl_is_zero!(u64, |x| x == 0); -impl_is_zero!(u128, |x| x == 0); -impl_is_zero!(usize, |x| x == 0); - -impl_is_zero!(bool, |x| x == false); -impl_is_zero!(char, |x| x == '\0'); - -impl_is_zero!(f32, |x: f32| x.to_bits() == 0); -impl_is_zero!(f64, |x: f64| x.to_bits() == 0); - -unsafe impl<T> IsZero for *const T { - #[inline] - fn is_zero(&self) -> bool { - (*self).is_null() - } -} - -unsafe impl<T> IsZero for *mut T { - #[inline] - fn is_zero(&self) -> bool { - (*self).is_null() - } -} - -unsafe impl<T: IsZero, const N: usize> IsZero for [T; N] { - #[inline] - fn is_zero(&self) -> bool { - // Because this is generated as a runtime check, it's not obvious that - // it's worth doing if the array is really long. The threshold here - // is largely arbitrary, but was picked because as of 2022-07-01 LLVM - // fails to const-fold the check in `vec![[1; 32]; n]` - // See https://github.com/rust-lang/rust/pull/97581#issuecomment-1166628022 - // Feel free to tweak if you have better evidence. - - N <= 16 && self.iter().all(IsZero::is_zero) - } -} - -// This is recursive macro. -macro_rules! impl_for_tuples { - // Stopper - () => { - // No use for implementing for empty tuple because it is ZST. - }; - ($first_arg:ident $(,$rest:ident)*) => { - unsafe impl <$first_arg: IsZero, $($rest: IsZero,)*> IsZero for ($first_arg, $($rest,)*){ - #[inline] - fn is_zero(&self) -> bool{ - // Destructure tuple to N references - // Rust allows to hide generic params by local variable names. - #[allow(non_snake_case)] - let ($first_arg, $($rest,)*) = self; - - $first_arg.is_zero() - $( && $rest.is_zero() )* - } - } - - impl_for_tuples!($($rest),*); - } -} - -impl_for_tuples!(A, B, C, D, E, F, G, H); - -// `Option<&T>` and `Option<Box<T>>` are guaranteed to represent `None` as null. -// For fat pointers, the bytes that would be the pointer metadata in the `Some` -// variant are padding in the `None` variant, so ignoring them and -// zero-initializing instead is ok. -// `Option<&mut T>` never implements `Clone`, so there's no need for an impl of -// `SpecFromElem`. - -unsafe impl<T: ?Sized> IsZero for Option<&T> { - #[inline] - fn is_zero(&self) -> bool { - self.is_none() - } -} - -unsafe impl<T: ?Sized> IsZero for Option<Box<T>> { - #[inline] - fn is_zero(&self) -> bool { - self.is_none() - } -} - -// `Option<num::NonZeroU32>` and similar have a representation guarantee that -// they're the same size as the corresponding `u32` type, as well as a guarantee -// that transmuting between `NonZeroU32` and `Option<num::NonZeroU32>` works. -// While the documentation officially makes it UB to transmute from `None`, -// we're the standard library so we can make extra inferences, and we know that -// the only niche available to represent `None` is the one that's all zeros. - -macro_rules! impl_is_zero_option_of_nonzero { - ($($t:ident,)+) => {$( - unsafe impl IsZero for Option<core::num::$t> { - #[inline] - fn is_zero(&self) -> bool { - self.is_none() - } - } - )+}; -} - -impl_is_zero_option_of_nonzero!( - NonZeroU8, - NonZeroU16, - NonZeroU32, - NonZeroU64, - NonZeroU128, - NonZeroI8, - NonZeroI16, - NonZeroI32, - NonZeroI64, - NonZeroI128, - NonZeroUsize, - NonZeroIsize, -); - -macro_rules! impl_is_zero_option_of_num { - ($($t:ty,)+) => {$( - unsafe impl IsZero for Option<$t> { - #[inline] - fn is_zero(&self) -> bool { - const { - let none: Self = unsafe { core::mem::MaybeUninit::zeroed().assume_init() }; - assert!(none.is_none()); - } - self.is_none() - } - } - )+}; -} - -impl_is_zero_option_of_num!(u8, u16, u32, u64, u128, i8, i16, i32, i64, i128, usize, isize,); - -unsafe impl<T: IsZero> IsZero for Wrapping<T> { - #[inline] - fn is_zero(&self) -> bool { - self.0.is_zero() - } -} - -unsafe impl<T: IsZero> IsZero for Saturating<T> { - #[inline] - fn is_zero(&self) -> bool { - self.0.is_zero() - } -} - -macro_rules! impl_for_optional_bool { - ($($t:ty,)+) => {$( - unsafe impl IsZero for $t { - #[inline] - fn is_zero(&self) -> bool { - // SAFETY: This is *not* a stable layout guarantee, but - // inside `core` we're allowed to rely on the current rustc - // behaviour that options of bools will be one byte with - // no padding, so long as they're nested less than 254 deep. - let raw: u8 = unsafe { core::mem::transmute(*self) }; - raw == 0 - } - } - )+}; -} -impl_for_optional_bool! { - Option<bool>, - Option<Option<bool>>, - Option<Option<Option<bool>>>, - // Could go further, but not worth the metadata overhead -} diff --git a/rust/alloc/vec/mod.rs b/rust/alloc/vec/mod.rs deleted file mode 100644 index 94995913566b..000000000000 --- a/rust/alloc/vec/mod.rs +++ /dev/null @@ -1,3609 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 OR MIT - -//! A contiguous growable array type with heap-allocated contents, written -//! `Vec<T>`. -//! -//! Vectors have *O*(1) indexing, amortized *O*(1) push (to the end) and -//! *O*(1) pop (from the end). -//! -//! Vectors ensure they never allocate more than `isize::MAX` bytes. -//! -//! # Examples -//! -//! You can explicitly create a [`Vec`] with [`Vec::new`]: -//! -//! ``` -//! let v: Vec<i32> = Vec::new(); -//! ``` -//! -//! ...or by using the [`vec!`] macro: -//! -//! ``` -//! let v: Vec<i32> = vec![]; -//! -//! let v = vec![1, 2, 3, 4, 5]; -//! -//! let v = vec![0; 10]; // ten zeroes -//! ``` -//! -//! You can [`push`] values onto the end of a vector (which will grow the vector -//! as needed): -//! -//! ``` -//! let mut v = vec![1, 2]; -//! -//! v.push(3); -//! ``` -//! -//! Popping values works in much the same way: -//! -//! ``` -//! let mut v = vec![1, 2]; -//! -//! let two = v.pop(); -//! ``` -//! -//! Vectors also support indexing (through the [`Index`] and [`IndexMut`] traits): -//! -//! ``` -//! let mut v = vec![1, 2, 3]; -//! let three = v[2]; -//! v[1] = v[1] + 5; -//! ``` -//! -//! [`push`]: Vec::push - -#![stable(feature = "rust1", since = "1.0.0")] - -#[cfg(not(no_global_oom_handling))] -use core::cmp; -use core::cmp::Ordering; -use core::convert::TryFrom; -use core::fmt; -use core::hash::{Hash, Hasher}; -use core::intrinsics::assume; -use core::iter; -#[cfg(not(no_global_oom_handling))] -use core::iter::FromIterator; -use core::marker::PhantomData; -use core::mem::{self, ManuallyDrop, MaybeUninit, SizedTypeProperties}; -use core::ops::{self, Index, IndexMut, Range, RangeBounds}; -use core::ptr::{self, NonNull}; -use core::slice::{self, SliceIndex}; - -use crate::alloc::{Allocator, Global}; -#[cfg(not(no_borrow))] -use crate::borrow::{Cow, ToOwned}; -use crate::boxed::Box; -use crate::collections::{TryReserveError, TryReserveErrorKind}; -use crate::raw_vec::RawVec; - -#[unstable(feature = "drain_filter", reason = "recently added", issue = "43244")] -pub use self::drain_filter::DrainFilter; - -mod drain_filter; - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "vec_splice", since = "1.21.0")] -pub use self::splice::Splice; - -#[cfg(not(no_global_oom_handling))] -mod splice; - -#[stable(feature = "drain", since = "1.6.0")] -pub use self::drain::Drain; - -mod drain; - -#[cfg(not(no_borrow))] -#[cfg(not(no_global_oom_handling))] -mod cow; - -#[cfg(not(no_global_oom_handling))] -pub(crate) use self::in_place_collect::AsVecIntoIter; -#[stable(feature = "rust1", since = "1.0.0")] -pub use self::into_iter::IntoIter; - -mod into_iter; - -#[cfg(not(no_global_oom_handling))] -use self::is_zero::IsZero; - -mod is_zero; - -#[cfg(not(no_global_oom_handling))] -mod in_place_collect; - -mod partial_eq; - -#[cfg(not(no_global_oom_handling))] -use self::spec_from_elem::SpecFromElem; - -#[cfg(not(no_global_oom_handling))] -mod spec_from_elem; - -use self::set_len_on_drop::SetLenOnDrop; - -mod set_len_on_drop; - -#[cfg(not(no_global_oom_handling))] -use self::in_place_drop::{InPlaceDrop, InPlaceDstBufDrop}; - -#[cfg(not(no_global_oom_handling))] -mod in_place_drop; - -#[cfg(not(no_global_oom_handling))] -use self::spec_from_iter_nested::SpecFromIterNested; - -#[cfg(not(no_global_oom_handling))] -mod spec_from_iter_nested; - -#[cfg(not(no_global_oom_handling))] -use self::spec_from_iter::SpecFromIter; - -#[cfg(not(no_global_oom_handling))] -mod spec_from_iter; - -#[cfg(not(no_global_oom_handling))] -use self::spec_extend::SpecExtend; - -use self::spec_extend::TrySpecExtend; - -mod spec_extend; - -/// A contiguous growable array type, written as `Vec<T>`, short for 'vector'. -/// -/// # Examples -/// -/// ``` -/// let mut vec = Vec::new(); -/// vec.push(1); -/// vec.push(2); -/// -/// assert_eq!(vec.len(), 2); -/// assert_eq!(vec[0], 1); -/// -/// assert_eq!(vec.pop(), Some(2)); -/// assert_eq!(vec.len(), 1); -/// -/// vec[0] = 7; -/// assert_eq!(vec[0], 7); -/// -/// vec.extend([1, 2, 3]); -/// -/// for x in &vec { -/// println!("{x}"); -/// } -/// assert_eq!(vec, [7, 1, 2, 3]); -/// ``` -/// -/// The [`vec!`] macro is provided for convenient initialization: -/// -/// ``` -/// let mut vec1 = vec![1, 2, 3]; -/// vec1.push(4); -/// let vec2 = Vec::from([1, 2, 3, 4]); -/// assert_eq!(vec1, vec2); -/// ``` -/// -/// It can also initialize each element of a `Vec<T>` with a given value. -/// This may be more efficient than performing allocation and initialization -/// in separate steps, especially when initializing a vector of zeros: -/// -/// ``` -/// let vec = vec![0; 5]; -/// assert_eq!(vec, [0, 0, 0, 0, 0]); -/// -/// // The following is equivalent, but potentially slower: -/// let mut vec = Vec::with_capacity(5); -/// vec.resize(5, 0); -/// assert_eq!(vec, [0, 0, 0, 0, 0]); -/// ``` -/// -/// For more information, see -/// [Capacity and Reallocation](#capacity-and-reallocation). -/// -/// Use a `Vec<T>` as an efficient stack: -/// -/// ``` -/// let mut stack = Vec::new(); -/// -/// stack.push(1); -/// stack.push(2); -/// stack.push(3); -/// -/// while let Some(top) = stack.pop() { -/// // Prints 3, 2, 1 -/// println!("{top}"); -/// } -/// ``` -/// -/// # Indexing -/// -/// The `Vec` type allows to access values by index, because it implements the -/// [`Index`] trait. An example will be more explicit: -/// -/// ``` -/// let v = vec![0, 2, 4, 6]; -/// println!("{}", v[1]); // it will display '2' -/// ``` -/// -/// However be careful: if you try to access an index which isn't in the `Vec`, -/// your software will panic! You cannot do this: -/// -/// ```should_panic -/// let v = vec![0, 2, 4, 6]; -/// println!("{}", v[6]); // it will panic! -/// ``` -/// -/// Use [`get`] and [`get_mut`] if you want to check whether the index is in -/// the `Vec`. -/// -/// # Slicing -/// -/// A `Vec` can be mutable. On the other hand, slices are read-only objects. -/// To get a [slice][prim@slice], use [`&`]. Example: -/// -/// ``` -/// fn read_slice(slice: &[usize]) { -/// // ... -/// } -/// -/// let v = vec![0, 1]; -/// read_slice(&v); -/// -/// // ... and that's all! -/// // you can also do it like this: -/// let u: &[usize] = &v; -/// // or like this: -/// let u: &[_] = &v; -/// ``` -/// -/// In Rust, it's more common to pass slices as arguments rather than vectors -/// when you just want to provide read access. The same goes for [`String`] and -/// [`&str`]. -/// -/// # Capacity and reallocation -/// -/// The capacity of a vector is the amount of space allocated for any future -/// elements that will be added onto the vector. This is not to be confused with -/// the *length* of a vector, which specifies the number of actual elements -/// within the vector. If a vector's length exceeds its capacity, its capacity -/// will automatically be increased, but its elements will have to be -/// reallocated. -/// -/// For example, a vector with capacity 10 and length 0 would be an empty vector -/// with space for 10 more elements. Pushing 10 or fewer elements onto the -/// vector will not change its capacity or cause reallocation to occur. However, -/// if the vector's length is increased to 11, it will have to reallocate, which -/// can be slow. For this reason, it is recommended to use [`Vec::with_capacity`] -/// whenever possible to specify how big the vector is expected to get. -/// -/// # Guarantees -/// -/// Due to its incredibly fundamental nature, `Vec` makes a lot of guarantees -/// about its design. This ensures that it's as low-overhead as possible in -/// the general case, and can be correctly manipulated in primitive ways -/// by unsafe code. Note that these guarantees refer to an unqualified `Vec<T>`. -/// If additional type parameters are added (e.g., to support custom allocators), -/// overriding their defaults may change the behavior. -/// -/// Most fundamentally, `Vec` is and always will be a (pointer, capacity, length) -/// triplet. No more, no less. The order of these fields is completely -/// unspecified, and you should use the appropriate methods to modify these. -/// The pointer will never be null, so this type is null-pointer-optimized. -/// -/// However, the pointer might not actually point to allocated memory. In particular, -/// if you construct a `Vec` with capacity 0 via [`Vec::new`], [`vec![]`][`vec!`], -/// [`Vec::with_capacity(0)`][`Vec::with_capacity`], or by calling [`shrink_to_fit`] -/// on an empty Vec, it will not allocate memory. Similarly, if you store zero-sized -/// types inside a `Vec`, it will not allocate space for them. *Note that in this case -/// the `Vec` might not report a [`capacity`] of 0*. `Vec` will allocate if and only -/// if <code>[mem::size_of::\<T>]\() * [capacity]\() > 0</code>. In general, `Vec`'s allocation -/// details are very subtle --- if you intend to allocate memory using a `Vec` -/// and use it for something else (either to pass to unsafe code, or to build your -/// own memory-backed collection), be sure to deallocate this memory by using -/// `from_raw_parts` to recover the `Vec` and then dropping it. -/// -/// If a `Vec` *has* allocated memory, then the memory it points to is on the heap -/// (as defined by the allocator Rust is configured to use by default), and its -/// pointer points to [`len`] initialized, contiguous elements in order (what -/// you would see if you coerced it to a slice), followed by <code>[capacity] - [len]</code> -/// logically uninitialized, contiguous elements. -/// -/// A vector containing the elements `'a'` and `'b'` with capacity 4 can be -/// visualized as below. The top part is the `Vec` struct, it contains a -/// pointer to the head of the allocation in the heap, length and capacity. -/// The bottom part is the allocation on the heap, a contiguous memory block. -/// -/// ```text -/// ptr len capacity -/// +--------+--------+--------+ -/// | 0x0123 | 2 | 4 | -/// +--------+--------+--------+ -/// | -/// v -/// Heap +--------+--------+--------+--------+ -/// | 'a' | 'b' | uninit | uninit | -/// +--------+--------+--------+--------+ -/// ``` -/// -/// - **uninit** represents memory that is not initialized, see [`MaybeUninit`]. -/// - Note: the ABI is not stable and `Vec` makes no guarantees about its memory -/// layout (including the order of fields). -/// -/// `Vec` will never perform a "small optimization" where elements are actually -/// stored on the stack for two reasons: -/// -/// * It would make it more difficult for unsafe code to correctly manipulate -/// a `Vec`. The contents of a `Vec` wouldn't have a stable address if it were -/// only moved, and it would be more difficult to determine if a `Vec` had -/// actually allocated memory. -/// -/// * It would penalize the general case, incurring an additional branch -/// on every access. -/// -/// `Vec` will never automatically shrink itself, even if completely empty. This -/// ensures no unnecessary allocations or deallocations occur. Emptying a `Vec` -/// and then filling it back up to the same [`len`] should incur no calls to -/// the allocator. If you wish to free up unused memory, use -/// [`shrink_to_fit`] or [`shrink_to`]. -/// -/// [`push`] and [`insert`] will never (re)allocate if the reported capacity is -/// sufficient. [`push`] and [`insert`] *will* (re)allocate if -/// <code>[len] == [capacity]</code>. That is, the reported capacity is completely -/// accurate, and can be relied on. It can even be used to manually free the memory -/// allocated by a `Vec` if desired. Bulk insertion methods *may* reallocate, even -/// when not necessary. -/// -/// `Vec` does not guarantee any particular growth strategy when reallocating -/// when full, nor when [`reserve`] is called. The current strategy is basic -/// and it may prove desirable to use a non-constant growth factor. Whatever -/// strategy is used will of course guarantee *O*(1) amortized [`push`]. -/// -/// `vec![x; n]`, `vec![a, b, c, d]`, and -/// [`Vec::with_capacity(n)`][`Vec::with_capacity`], will all produce a `Vec` -/// with exactly the requested capacity. If <code>[len] == [capacity]</code>, -/// (as is the case for the [`vec!`] macro), then a `Vec<T>` can be converted to -/// and from a [`Box<[T]>`][owned slice] without reallocating or moving the elements. -/// -/// `Vec` will not specifically overwrite any data that is removed from it, -/// but also won't specifically preserve it. Its uninitialized memory is -/// scratch space that it may use however it wants. It will generally just do -/// whatever is most efficient or otherwise easy to implement. Do not rely on -/// removed data to be erased for security purposes. Even if you drop a `Vec`, its -/// buffer may simply be reused by another allocation. Even if you zero a `Vec`'s memory -/// first, that might not actually happen because the optimizer does not consider -/// this a side-effect that must be preserved. There is one case which we will -/// not break, however: using `unsafe` code to write to the excess capacity, -/// and then increasing the length to match, is always valid. -/// -/// Currently, `Vec` does not guarantee the order in which elements are dropped. -/// The order has changed in the past and may change again. -/// -/// [`get`]: ../../std/vec/struct.Vec.html#method.get -/// [`get_mut`]: ../../std/vec/struct.Vec.html#method.get_mut -/// [`String`]: crate::string::String -/// [`&str`]: type@str -/// [`shrink_to_fit`]: Vec::shrink_to_fit -/// [`shrink_to`]: Vec::shrink_to -/// [capacity]: Vec::capacity -/// [`capacity`]: Vec::capacity -/// [mem::size_of::\<T>]: core::mem::size_of -/// [len]: Vec::len -/// [`len`]: Vec::len -/// [`push`]: Vec::push -/// [`insert`]: Vec::insert -/// [`reserve`]: Vec::reserve -/// [`MaybeUninit`]: core::mem::MaybeUninit -/// [owned slice]: Box -#[stable(feature = "rust1", since = "1.0.0")] -#[cfg_attr(not(test), rustc_diagnostic_item = "Vec")] -#[rustc_insignificant_dtor] -pub struct Vec<T, #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator = Global> { - buf: RawVec<T, A>, - len: usize, -} - -//////////////////////////////////////////////////////////////////////////////// -// Inherent methods -//////////////////////////////////////////////////////////////////////////////// - -impl<T> Vec<T> { - /// Constructs a new, empty `Vec<T>`. - /// - /// The vector will not allocate until elements are pushed onto it. - /// - /// # Examples - /// - /// ``` - /// # #![allow(unused_mut)] - /// let mut vec: Vec<i32> = Vec::new(); - /// ``` - #[inline] - #[rustc_const_stable(feature = "const_vec_new", since = "1.39.0")] - #[stable(feature = "rust1", since = "1.0.0")] - #[must_use] - pub const fn new() -> Self { - Vec { buf: RawVec::NEW, len: 0 } - } - - /// Constructs a new, empty `Vec<T>` with at least the specified capacity. - /// - /// The vector will be able to hold at least `capacity` elements without - /// reallocating. This method is allowed to allocate for more elements than - /// `capacity`. If `capacity` is 0, the vector will not allocate. - /// - /// It is important to note that although the returned vector has the - /// minimum *capacity* specified, the vector will have a zero *length*. For - /// an explanation of the difference between length and capacity, see - /// *[Capacity and reallocation]*. - /// - /// If it is important to know the exact allocated capacity of a `Vec`, - /// always use the [`capacity`] method after construction. - /// - /// For `Vec<T>` where `T` is a zero-sized type, there will be no allocation - /// and the capacity will always be `usize::MAX`. - /// - /// [Capacity and reallocation]: #capacity-and-reallocation - /// [`capacity`]: Vec::capacity - /// - /// # Panics - /// - /// Panics if the new capacity exceeds `isize::MAX` bytes. - /// - /// # Examples - /// - /// ``` - /// let mut vec = Vec::with_capacity(10); - /// - /// // The vector contains no items, even though it has capacity for more - /// assert_eq!(vec.len(), 0); - /// assert!(vec.capacity() >= 10); - /// - /// // These are all done without reallocating... - /// for i in 0..10 { - /// vec.push(i); - /// } - /// assert_eq!(vec.len(), 10); - /// assert!(vec.capacity() >= 10); - /// - /// // ...but this may make the vector reallocate - /// vec.push(11); - /// assert_eq!(vec.len(), 11); - /// assert!(vec.capacity() >= 11); - /// - /// // A vector of a zero-sized type will always over-allocate, since no - /// // allocation is necessary - /// let vec_units = Vec::<()>::with_capacity(10); - /// assert_eq!(vec_units.capacity(), usize::MAX); - /// ``` - #[cfg(not(no_global_oom_handling))] - #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - #[must_use] - pub fn with_capacity(capacity: usize) -> Self { - Self::with_capacity_in(capacity, Global) - } - - /// Tries to construct a new, empty `Vec<T>` with at least the specified capacity. - /// - /// The vector will be able to hold at least `capacity` elements without - /// reallocating. This method is allowed to allocate for more elements than - /// `capacity`. If `capacity` is 0, the vector will not allocate. - /// - /// It is important to note that although the returned vector has the - /// minimum *capacity* specified, the vector will have a zero *length*. For - /// an explanation of the difference between length and capacity, see - /// *[Capacity and reallocation]*. - /// - /// If it is important to know the exact allocated capacity of a `Vec`, - /// always use the [`capacity`] method after construction. - /// - /// For `Vec<T>` where `T` is a zero-sized type, there will be no allocation - /// and the capacity will always be `usize::MAX`. - /// - /// [Capacity and reallocation]: #capacity-and-reallocation - /// [`capacity`]: Vec::capacity - /// - /// # Examples - /// - /// ``` - /// let mut vec = Vec::try_with_capacity(10).unwrap(); - /// - /// // The vector contains no items, even though it has capacity for more - /// assert_eq!(vec.len(), 0); - /// assert!(vec.capacity() >= 10); - /// - /// // These are all done without reallocating... - /// for i in 0..10 { - /// vec.push(i); - /// } - /// assert_eq!(vec.len(), 10); - /// assert!(vec.capacity() >= 10); - /// - /// // ...but this may make the vector reallocate - /// vec.push(11); - /// assert_eq!(vec.len(), 11); - /// assert!(vec.capacity() >= 11); - /// - /// let mut result = Vec::try_with_capacity(usize::MAX); - /// assert!(result.is_err()); - /// - /// // A vector of a zero-sized type will always over-allocate, since no - /// // allocation is necessary - /// let vec_units = Vec::<()>::try_with_capacity(10).unwrap(); - /// assert_eq!(vec_units.capacity(), usize::MAX); - /// ``` - #[inline] - #[stable(feature = "kernel", since = "1.0.0")] - pub fn try_with_capacity(capacity: usize) -> Result<Self, TryReserveError> { - Self::try_with_capacity_in(capacity, Global) - } - - /// Creates a `Vec<T>` directly from a pointer, a capacity, and a length. - /// - /// # Safety - /// - /// This is highly unsafe, due to the number of invariants that aren't - /// checked: - /// - /// * `ptr` must have been allocated using the global allocator, such as via - /// the [`alloc::alloc`] function. - /// * `T` needs to have the same alignment as what `ptr` was allocated with. - /// (`T` having a less strict alignment is not sufficient, the alignment really - /// needs to be equal to satisfy the [`dealloc`] requirement that memory must be - /// allocated and deallocated with the same layout.) - /// * The size of `T` times the `capacity` (ie. the allocated size in bytes) needs - /// to be the same size as the pointer was allocated with. (Because similar to - /// alignment, [`dealloc`] must be called with the same layout `size`.) - /// * `length` needs to be less than or equal to `capacity`. - /// * The first `length` values must be properly initialized values of type `T`. - /// * `capacity` needs to be the capacity that the pointer was allocated with. - /// * The allocated size in bytes must be no larger than `isize::MAX`. - /// See the safety documentation of [`pointer::offset`]. - /// - /// These requirements are always upheld by any `ptr` that has been allocated - /// via `Vec<T>`. Other allocation sources are allowed if the invariants are - /// upheld. - /// - /// Violating these may cause problems like corrupting the allocator's - /// internal data structures. For example it is normally **not** safe - /// to build a `Vec<u8>` from a pointer to a C `char` array with length - /// `size_t`, doing so is only safe if the array was initially allocated by - /// a `Vec` or `String`. - /// It's also not safe to build one from a `Vec<u16>` and its length, because - /// the allocator cares about the alignment, and these two types have different - /// alignments. The buffer was allocated with alignment 2 (for `u16`), but after - /// turning it into a `Vec<u8>` it'll be deallocated with alignment 1. To avoid - /// these issues, it is often preferable to do casting/transmuting using - /// [`slice::from_raw_parts`] instead. - /// - /// The ownership of `ptr` is effectively transferred to the - /// `Vec<T>` which may then deallocate, reallocate or change the - /// contents of memory pointed to by the pointer at will. Ensure - /// that nothing else uses the pointer after calling this - /// function. - /// - /// [`String`]: crate::string::String - /// [`alloc::alloc`]: crate::alloc::alloc - /// [`dealloc`]: crate::alloc::GlobalAlloc::dealloc - /// - /// # Examples - /// - /// ``` - /// use std::ptr; - /// use std::mem; - /// - /// let v = vec![1, 2, 3]; - /// - // FIXME Update this when vec_into_raw_parts is stabilized - /// // Prevent running `v`'s destructor so we are in complete control - /// // of the allocation. - /// let mut v = mem::ManuallyDrop::new(v); - /// - /// // Pull out the various important pieces of information about `v` - /// let p = v.as_mut_ptr(); - /// let len = v.len(); - /// let cap = v.capacity(); - /// - /// unsafe { - /// // Overwrite memory with 4, 5, 6 - /// for i in 0..len { - /// ptr::write(p.add(i), 4 + i); - /// } - /// - /// // Put everything back together into a Vec - /// let rebuilt = Vec::from_raw_parts(p, len, cap); - /// assert_eq!(rebuilt, [4, 5, 6]); - /// } - /// ``` - /// - /// Using memory that was allocated elsewhere: - /// - /// ```rust - /// #![feature(allocator_api)] - /// - /// use std::alloc::{AllocError, Allocator, Global, Layout}; - /// - /// fn main() { - /// let layout = Layout::array::<u32>(16).expect("overflow cannot happen"); - /// - /// let vec = unsafe { - /// let mem = match Global.allocate(layout) { - /// Ok(mem) => mem.cast::<u32>().as_ptr(), - /// Err(AllocError) => return, - /// }; - /// - /// mem.write(1_000_000); - /// - /// Vec::from_raw_parts_in(mem, 1, 16, Global) - /// }; - /// - /// assert_eq!(vec, &[1_000_000]); - /// assert_eq!(vec.capacity(), 16); - /// } - /// ``` - #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - pub unsafe fn from_raw_parts(ptr: *mut T, length: usize, capacity: usize) -> Self { - unsafe { Self::from_raw_parts_in(ptr, length, capacity, Global) } - } -} - -impl<T, A: Allocator> Vec<T, A> { - /// Constructs a new, empty `Vec<T, A>`. - /// - /// The vector will not allocate until elements are pushed onto it. - /// - /// # Examples - /// - /// ``` - /// #![feature(allocator_api)] - /// - /// use std::alloc::System; - /// - /// # #[allow(unused_mut)] - /// let mut vec: Vec<i32, _> = Vec::new_in(System); - /// ``` - #[inline] - #[unstable(feature = "allocator_api", issue = "32838")] - pub const fn new_in(alloc: A) -> Self { - Vec { buf: RawVec::new_in(alloc), len: 0 } - } - - /// Constructs a new, empty `Vec<T, A>` with at least the specified capacity - /// with the provided allocator. - /// - /// The vector will be able to hold at least `capacity` elements without - /// reallocating. This method is allowed to allocate for more elements than - /// `capacity`. If `capacity` is 0, the vector will not allocate. - /// - /// It is important to note that although the returned vector has the - /// minimum *capacity* specified, the vector will have a zero *length*. For - /// an explanation of the difference between length and capacity, see - /// *[Capacity and reallocation]*. - /// - /// If it is important to know the exact allocated capacity of a `Vec`, - /// always use the [`capacity`] method after construction. - /// - /// For `Vec<T, A>` where `T` is a zero-sized type, there will be no allocation - /// and the capacity will always be `usize::MAX`. - /// - /// [Capacity and reallocation]: #capacity-and-reallocation - /// [`capacity`]: Vec::capacity - /// - /// # Panics - /// - /// Panics if the new capacity exceeds `isize::MAX` bytes. - /// - /// # Examples - /// - /// ``` - /// #![feature(allocator_api)] - /// - /// use std::alloc::System; - /// - /// let mut vec = Vec::with_capacity_in(10, System); - /// - /// // The vector contains no items, even though it has capacity for more - /// assert_eq!(vec.len(), 0); - /// assert_eq!(vec.capacity(), 10); - /// - /// // These are all done without reallocating... - /// for i in 0..10 { - /// vec.push(i); - /// } - /// assert_eq!(vec.len(), 10); - /// assert_eq!(vec.capacity(), 10); - /// - /// // ...but this may make the vector reallocate - /// vec.push(11); - /// assert_eq!(vec.len(), 11); - /// assert!(vec.capacity() >= 11); - /// - /// // A vector of a zero-sized type will always over-allocate, since no - /// // allocation is necessary - /// let vec_units = Vec::<(), System>::with_capacity_in(10, System); - /// assert_eq!(vec_units.capacity(), usize::MAX); - /// ``` - #[cfg(not(no_global_oom_handling))] - #[inline] - #[unstable(feature = "allocator_api", issue = "32838")] - pub fn with_capacity_in(capacity: usize, alloc: A) -> Self { - Vec { buf: RawVec::with_capacity_in(capacity, alloc), len: 0 } - } - - /// Tries to construct a new, empty `Vec<T, A>` with at least the specified capacity - /// with the provided allocator. - /// - /// The vector will be able to hold at least `capacity` elements without - /// reallocating. This method is allowed to allocate for more elements than - /// `capacity`. If `capacity` is 0, the vector will not allocate. - /// - /// It is important to note that although the returned vector has the - /// minimum *capacity* specified, the vector will have a zero *length*. For - /// an explanation of the difference between length and capacity, see - /// *[Capacity and reallocation]*. - /// - /// If it is important to know the exact allocated capacity of a `Vec`, - /// always use the [`capacity`] method after construction. - /// - /// For `Vec<T, A>` where `T` is a zero-sized type, there will be no allocation - /// and the capacity will always be `usize::MAX`. - /// - /// [Capacity and reallocation]: #capacity-and-reallocation - /// [`capacity`]: Vec::capacity - /// - /// # Examples - /// - /// ``` - /// #![feature(allocator_api)] - /// - /// use std::alloc::System; - /// - /// let mut vec = Vec::try_with_capacity_in(10, System).unwrap(); - /// - /// // The vector contains no items, even though it has capacity for more - /// assert_eq!(vec.len(), 0); - /// assert_eq!(vec.capacity(), 10); - /// - /// // These are all done without reallocating... - /// for i in 0..10 { - /// vec.push(i); - /// } - /// assert_eq!(vec.len(), 10); - /// assert_eq!(vec.capacity(), 10); - /// - /// // ...but this may make the vector reallocate - /// vec.push(11); - /// assert_eq!(vec.len(), 11); - /// assert!(vec.capacity() >= 11); - /// - /// let mut result = Vec::try_with_capacity_in(usize::MAX, System); - /// assert!(result.is_err()); - /// - /// // A vector of a zero-sized type will always over-allocate, since no - /// // allocation is necessary - /// let vec_units = Vec::<(), System>::try_with_capacity_in(10, System).unwrap(); - /// assert_eq!(vec_units.capacity(), usize::MAX); - /// ``` - #[inline] - #[stable(feature = "kernel", since = "1.0.0")] - pub fn try_with_capacity_in(capacity: usize, alloc: A) -> Result<Self, TryReserveError> { - Ok(Vec { buf: RawVec::try_with_capacity_in(capacity, alloc)?, len: 0 }) - } - - /// Creates a `Vec<T, A>` directly from a pointer, a capacity, a length, - /// and an allocator. - /// - /// # Safety - /// - /// This is highly unsafe, due to the number of invariants that aren't - /// checked: - /// - /// * `ptr` must be [*currently allocated*] via the given allocator `alloc`. - /// * `T` needs to have the same alignment as what `ptr` was allocated with. - /// (`T` having a less strict alignment is not sufficient, the alignment really - /// needs to be equal to satisfy the [`dealloc`] requirement that memory must be - /// allocated and deallocated with the same layout.) - /// * The size of `T` times the `capacity` (ie. the allocated size in bytes) needs - /// to be the same size as the pointer was allocated with. (Because similar to - /// alignment, [`dealloc`] must be called with the same layout `size`.) - /// * `length` needs to be less than or equal to `capacity`. - /// * The first `length` values must be properly initialized values of type `T`. - /// * `capacity` needs to [*fit*] the layout size that the pointer was allocated with. - /// * The allocated size in bytes must be no larger than `isize::MAX`. - /// See the safety documentation of [`pointer::offset`]. - /// - /// These requirements are always upheld by any `ptr` that has been allocated - /// via `Vec<T, A>`. Other allocation sources are allowed if the invariants are - /// upheld. - /// - /// Violating these may cause problems like corrupting the allocator's - /// internal data structures. For example it is **not** safe - /// to build a `Vec<u8>` from a pointer to a C `char` array with length `size_t`. - /// It's also not safe to build one from a `Vec<u16>` and its length, because - /// the allocator cares about the alignment, and these two types have different - /// alignments. The buffer was allocated with alignment 2 (for `u16`), but after - /// turning it into a `Vec<u8>` it'll be deallocated with alignment 1. - /// - /// The ownership of `ptr` is effectively transferred to the - /// `Vec<T>` which may then deallocate, reallocate or change the - /// contents of memory pointed to by the pointer at will. Ensure - /// that nothing else uses the pointer after calling this - /// function. - /// - /// [`String`]: crate::string::String - /// [`dealloc`]: crate::alloc::GlobalAlloc::dealloc - /// [*currently allocated*]: crate::alloc::Allocator#currently-allocated-memory - /// [*fit*]: crate::alloc::Allocator#memory-fitting - /// - /// # Examples - /// - /// ``` - /// #![feature(allocator_api)] - /// - /// use std::alloc::System; - /// - /// use std::ptr; - /// use std::mem; - /// - /// let mut v = Vec::with_capacity_in(3, System); - /// v.push(1); - /// v.push(2); - /// v.push(3); - /// - // FIXME Update this when vec_into_raw_parts is stabilized - /// // Prevent running `v`'s destructor so we are in complete control - /// // of the allocation. - /// let mut v = mem::ManuallyDrop::new(v); - /// - /// // Pull out the various important pieces of information about `v` - /// let p = v.as_mut_ptr(); - /// let len = v.len(); - /// let cap = v.capacity(); - /// let alloc = v.allocator(); - /// - /// unsafe { - /// // Overwrite memory with 4, 5, 6 - /// for i in 0..len { - /// ptr::write(p.add(i), 4 + i); - /// } - /// - /// // Put everything back together into a Vec - /// let rebuilt = Vec::from_raw_parts_in(p, len, cap, alloc.clone()); - /// assert_eq!(rebuilt, [4, 5, 6]); - /// } - /// ``` - /// - /// Using memory that was allocated elsewhere: - /// - /// ```rust - /// use std::alloc::{alloc, Layout}; - /// - /// fn main() { - /// let layout = Layout::array::<u32>(16).expect("overflow cannot happen"); - /// let vec = unsafe { - /// let mem = alloc(layout).cast::<u32>(); - /// if mem.is_null() { - /// return; - /// } - /// - /// mem.write(1_000_000); - /// - /// Vec::from_raw_parts(mem, 1, 16) - /// }; - /// - /// assert_eq!(vec, &[1_000_000]); - /// assert_eq!(vec.capacity(), 16); - /// } - /// ``` - #[inline] - #[unstable(feature = "allocator_api", issue = "32838")] - pub unsafe fn from_raw_parts_in(ptr: *mut T, length: usize, capacity: usize, alloc: A) -> Self { - unsafe { Vec { buf: RawVec::from_raw_parts_in(ptr, capacity, alloc), len: length } } - } - - /// Decomposes a `Vec<T>` into its raw components. - /// - /// Returns the raw pointer to the underlying data, the length of - /// the vector (in elements), and the allocated capacity of the - /// data (in elements). These are the same arguments in the same - /// order as the arguments to [`from_raw_parts`]. - /// - /// After calling this function, the caller is responsible for the - /// memory previously managed by the `Vec`. The only way to do - /// this is to convert the raw pointer, length, and capacity back - /// into a `Vec` with the [`from_raw_parts`] function, allowing - /// the destructor to perform the cleanup. - /// - /// [`from_raw_parts`]: Vec::from_raw_parts - /// - /// # Examples - /// - /// ``` - /// #![feature(vec_into_raw_parts)] - /// let v: Vec<i32> = vec![-1, 0, 1]; - /// - /// let (ptr, len, cap) = v.into_raw_parts(); - /// - /// let rebuilt = unsafe { - /// // We can now make changes to the components, such as - /// // transmuting the raw pointer to a compatible type. - /// let ptr = ptr as *mut u32; - /// - /// Vec::from_raw_parts(ptr, len, cap) - /// }; - /// assert_eq!(rebuilt, [4294967295, 0, 1]); - /// ``` - #[unstable(feature = "vec_into_raw_parts", reason = "new API", issue = "65816")] - pub fn into_raw_parts(self) -> (*mut T, usize, usize) { - let mut me = ManuallyDrop::new(self); - (me.as_mut_ptr(), me.len(), me.capacity()) - } - - /// Decomposes a `Vec<T>` into its raw components. - /// - /// Returns the raw pointer to the underlying data, the length of the vector (in elements), - /// the allocated capacity of the data (in elements), and the allocator. These are the same - /// arguments in the same order as the arguments to [`from_raw_parts_in`]. - /// - /// After calling this function, the caller is responsible for the - /// memory previously managed by the `Vec`. The only way to do - /// this is to convert the raw pointer, length, and capacity back - /// into a `Vec` with the [`from_raw_parts_in`] function, allowing - /// the destructor to perform the cleanup. - /// - /// [`from_raw_parts_in`]: Vec::from_raw_parts_in - /// - /// # Examples - /// - /// ``` - /// #![feature(allocator_api, vec_into_raw_parts)] - /// - /// use std::alloc::System; - /// - /// let mut v: Vec<i32, System> = Vec::new_in(System); - /// v.push(-1); - /// v.push(0); - /// v.push(1); - /// - /// let (ptr, len, cap, alloc) = v.into_raw_parts_with_alloc(); - /// - /// let rebuilt = unsafe { - /// // We can now make changes to the components, such as - /// // transmuting the raw pointer to a compatible type. - /// let ptr = ptr as *mut u32; - /// - /// Vec::from_raw_parts_in(ptr, len, cap, alloc) - /// }; - /// assert_eq!(rebuilt, [4294967295, 0, 1]); - /// ``` - #[unstable(feature = "allocator_api", issue = "32838")] - // #[unstable(feature = "vec_into_raw_parts", reason = "new API", issue = "65816")] - pub fn into_raw_parts_with_alloc(self) -> (*mut T, usize, usize, A) { - let mut me = ManuallyDrop::new(self); - let len = me.len(); - let capacity = me.capacity(); - let ptr = me.as_mut_ptr(); - let alloc = unsafe { ptr::read(me.allocator()) }; - (ptr, len, capacity, alloc) - } - - /// Returns the total number of elements the vector can hold without - /// reallocating. - /// - /// # Examples - /// - /// ``` - /// let mut vec: Vec<i32> = Vec::with_capacity(10); - /// vec.push(42); - /// assert_eq!(vec.capacity(), 10); - /// ``` - #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - pub fn capacity(&self) -> usize { - self.buf.capacity() - } - - /// Reserves capacity for at least `additional` more elements to be inserted - /// in the given `Vec<T>`. The collection may reserve more space to - /// speculatively avoid frequent reallocations. After calling `reserve`, - /// capacity will be greater than or equal to `self.len() + additional`. - /// Does nothing if capacity is already sufficient. - /// - /// # Panics - /// - /// Panics if the new capacity exceeds `isize::MAX` bytes. - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec![1]; - /// vec.reserve(10); - /// assert!(vec.capacity() >= 11); - /// ``` - #[cfg(not(no_global_oom_handling))] - #[stable(feature = "rust1", since = "1.0.0")] - pub fn reserve(&mut self, additional: usize) { - self.buf.reserve(self.len, additional); - } - - /// Reserves the minimum capacity for at least `additional` more elements to - /// be inserted in the given `Vec<T>`. Unlike [`reserve`], this will not - /// deliberately over-allocate to speculatively avoid frequent allocations. - /// After calling `reserve_exact`, capacity will be greater than or equal to - /// `self.len() + additional`. Does nothing if the capacity is already - /// sufficient. - /// - /// Note that the allocator may give the collection more space than it - /// requests. Therefore, capacity can not be relied upon to be precisely - /// minimal. Prefer [`reserve`] if future insertions are expected. - /// - /// [`reserve`]: Vec::reserve - /// - /// # Panics - /// - /// Panics if the new capacity exceeds `isize::MAX` bytes. - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec![1]; - /// vec.reserve_exact(10); - /// assert!(vec.capacity() >= 11); - /// ``` - #[cfg(not(no_global_oom_handling))] - #[stable(feature = "rust1", since = "1.0.0")] - pub fn reserve_exact(&mut self, additional: usize) { - self.buf.reserve_exact(self.len, additional); - } - - /// Tries to reserve capacity for at least `additional` more elements to be inserted - /// in the given `Vec<T>`. The collection may reserve more space to speculatively avoid - /// frequent reallocations. After calling `try_reserve`, capacity will be - /// greater than or equal to `self.len() + additional` if it returns - /// `Ok(())`. Does nothing if capacity is already sufficient. This method - /// preserves the contents even if an error occurs. - /// - /// # Errors - /// - /// If the capacity overflows, or the allocator reports a failure, then an error - /// is returned. - /// - /// # Examples - /// - /// ``` - /// use std::collections::TryReserveError; - /// - /// fn process_data(data: &[u32]) -> Result<Vec<u32>, TryReserveError> { - /// let mut output = Vec::new(); - /// - /// // Pre-reserve the memory, exiting if we can't - /// output.try_reserve(data.len())?; - /// - /// // Now we know this can't OOM in the middle of our complex work - /// output.extend(data.iter().map(|&val| { - /// val * 2 + 5 // very complicated - /// })); - /// - /// Ok(output) - /// } - /// # process_data(&[1, 2, 3]).expect("why is the test harness OOMing on 12 bytes?"); - /// ``` - #[stable(feature = "try_reserve", since = "1.57.0")] - pub fn try_reserve(&mut self, additional: usize) -> Result<(), TryReserveError> { - self.buf.try_reserve(self.len, additional) - } - - /// Tries to reserve the minimum capacity for at least `additional` - /// elements to be inserted in the given `Vec<T>`. Unlike [`try_reserve`], - /// this will not deliberately over-allocate to speculatively avoid frequent - /// allocations. After calling `try_reserve_exact`, capacity will be greater - /// than or equal to `self.len() + additional` if it returns `Ok(())`. - /// Does nothing if the capacity is already sufficient. - /// - /// Note that the allocator may give the collection more space than it - /// requests. Therefore, capacity can not be relied upon to be precisely - /// minimal. Prefer [`try_reserve`] if future insertions are expected. - /// - /// [`try_reserve`]: Vec::try_reserve - /// - /// # Errors - /// - /// If the capacity overflows, or the allocator reports a failure, then an error - /// is returned. - /// - /// # Examples - /// - /// ``` - /// use std::collections::TryReserveError; - /// - /// fn process_data(data: &[u32]) -> Result<Vec<u32>, TryReserveError> { - /// let mut output = Vec::new(); - /// - /// // Pre-reserve the memory, exiting if we can't - /// output.try_reserve_exact(data.len())?; - /// - /// // Now we know this can't OOM in the middle of our complex work - /// output.extend(data.iter().map(|&val| { - /// val * 2 + 5 // very complicated - /// })); - /// - /// Ok(output) - /// } - /// # process_data(&[1, 2, 3]).expect("why is the test harness OOMing on 12 bytes?"); - /// ``` - #[stable(feature = "try_reserve", since = "1.57.0")] - pub fn try_reserve_exact(&mut self, additional: usize) -> Result<(), TryReserveError> { - self.buf.try_reserve_exact(self.len, additional) - } - - /// Shrinks the capacity of the vector as much as possible. - /// - /// It will drop down as close as possible to the length but the allocator - /// may still inform the vector that there is space for a few more elements. - /// - /// # Examples - /// - /// ``` - /// let mut vec = Vec::with_capacity(10); - /// vec.extend([1, 2, 3]); - /// assert_eq!(vec.capacity(), 10); - /// vec.shrink_to_fit(); - /// assert!(vec.capacity() >= 3); - /// ``` - #[cfg(not(no_global_oom_handling))] - #[stable(feature = "rust1", since = "1.0.0")] - pub fn shrink_to_fit(&mut self) { - // The capacity is never less than the length, and there's nothing to do when - // they are equal, so we can avoid the panic case in `RawVec::shrink_to_fit` - // by only calling it with a greater capacity. - if self.capacity() > self.len { - self.buf.shrink_to_fit(self.len); - } - } - - /// Shrinks the capacity of the vector with a lower bound. - /// - /// The capacity will remain at least as large as both the length - /// and the supplied value. - /// - /// If the current capacity is less than the lower limit, this is a no-op. - /// - /// # Examples - /// - /// ``` - /// let mut vec = Vec::with_capacity(10); - /// vec.extend([1, 2, 3]); - /// assert_eq!(vec.capacity(), 10); - /// vec.shrink_to(4); - /// assert!(vec.capacity() >= 4); - /// vec.shrink_to(0); - /// assert!(vec.capacity() >= 3); - /// ``` - #[cfg(not(no_global_oom_handling))] - #[stable(feature = "shrink_to", since = "1.56.0")] - pub fn shrink_to(&mut self, min_capacity: usize) { - if self.capacity() > min_capacity { - self.buf.shrink_to_fit(cmp::max(self.len, min_capacity)); - } - } - - /// Converts the vector into [`Box<[T]>`][owned slice]. - /// - /// If the vector has excess capacity, its items will be moved into a - /// newly-allocated buffer with exactly the right capacity. - /// - /// [owned slice]: Box - /// - /// # Examples - /// - /// ``` - /// let v = vec![1, 2, 3]; - /// - /// let slice = v.into_boxed_slice(); - /// ``` - /// - /// Any excess capacity is removed: - /// - /// ``` - /// let mut vec = Vec::with_capacity(10); - /// vec.extend([1, 2, 3]); - /// - /// assert_eq!(vec.capacity(), 10); - /// let slice = vec.into_boxed_slice(); - /// assert_eq!(slice.into_vec().capacity(), 3); - /// ``` - #[cfg(not(no_global_oom_handling))] - #[stable(feature = "rust1", since = "1.0.0")] - pub fn into_boxed_slice(mut self) -> Box<[T], A> { - unsafe { - self.shrink_to_fit(); - let me = ManuallyDrop::new(self); - let buf = ptr::read(&me.buf); - let len = me.len(); - buf.into_box(len).assume_init() - } - } - - /// Shortens the vector, keeping the first `len` elements and dropping - /// the rest. - /// - /// If `len` is greater than the vector's current length, this has no - /// effect. - /// - /// The [`drain`] method can emulate `truncate`, but causes the excess - /// elements to be returned instead of dropped. - /// - /// Note that this method has no effect on the allocated capacity - /// of the vector. - /// - /// # Examples - /// - /// Truncating a five element vector to two elements: - /// - /// ``` - /// let mut vec = vec![1, 2, 3, 4, 5]; - /// vec.truncate(2); - /// assert_eq!(vec, [1, 2]); - /// ``` - /// - /// No truncation occurs when `len` is greater than the vector's current - /// length: - /// - /// ``` - /// let mut vec = vec![1, 2, 3]; - /// vec.truncate(8); - /// assert_eq!(vec, [1, 2, 3]); - /// ``` - /// - /// Truncating when `len == 0` is equivalent to calling the [`clear`] - /// method. - /// - /// ``` - /// let mut vec = vec![1, 2, 3]; - /// vec.truncate(0); - /// assert_eq!(vec, []); - /// ``` - /// - /// [`clear`]: Vec::clear - /// [`drain`]: Vec::drain - #[stable(feature = "rust1", since = "1.0.0")] - pub fn truncate(&mut self, len: usize) { - // This is safe because: - // - // * the slice passed to `drop_in_place` is valid; the `len > self.len` - // case avoids creating an invalid slice, and - // * the `len` of the vector is shrunk before calling `drop_in_place`, - // such that no value will be dropped twice in case `drop_in_place` - // were to panic once (if it panics twice, the program aborts). - unsafe { - // Note: It's intentional that this is `>` and not `>=`. - // Changing it to `>=` has negative performance - // implications in some cases. See #78884 for more. - if len > self.len { - return; - } - let remaining_len = self.len - len; - let s = ptr::slice_from_raw_parts_mut(self.as_mut_ptr().add(len), remaining_len); - self.len = len; - ptr::drop_in_place(s); - } - } - - /// Extracts a slice containing the entire vector. - /// - /// Equivalent to `&s[..]`. - /// - /// # Examples - /// - /// ``` - /// use std::io::{self, Write}; - /// let buffer = vec![1, 2, 3, 5, 8]; - /// io::sink().write(buffer.as_slice()).unwrap(); - /// ``` - #[inline] - #[stable(feature = "vec_as_slice", since = "1.7.0")] - pub fn as_slice(&self) -> &[T] { - self - } - - /// Extracts a mutable slice of the entire vector. - /// - /// Equivalent to `&mut s[..]`. - /// - /// # Examples - /// - /// ``` - /// use std::io::{self, Read}; - /// let mut buffer = vec![0; 3]; - /// io::repeat(0b101).read_exact(buffer.as_mut_slice()).unwrap(); - /// ``` - #[inline] - #[stable(feature = "vec_as_slice", since = "1.7.0")] - pub fn as_mut_slice(&mut self) -> &mut [T] { - self - } - - /// Returns a raw pointer to the vector's buffer, or a dangling raw pointer - /// valid for zero sized reads if the vector didn't allocate. - /// - /// The caller must ensure that the vector outlives the pointer this - /// function returns, or else it will end up pointing to garbage. - /// Modifying the vector may cause its buffer to be reallocated, - /// which would also make any pointers to it invalid. - /// - /// The caller must also ensure that the memory the pointer (non-transitively) points to - /// is never written to (except inside an `UnsafeCell`) using this pointer or any pointer - /// derived from it. If you need to mutate the contents of the slice, use [`as_mut_ptr`]. - /// - /// # Examples - /// - /// ``` - /// let x = vec![1, 2, 4]; - /// let x_ptr = x.as_ptr(); - /// - /// unsafe { - /// for i in 0..x.len() { - /// assert_eq!(*x_ptr.add(i), 1 << i); - /// } - /// } - /// ``` - /// - /// [`as_mut_ptr`]: Vec::as_mut_ptr - #[stable(feature = "vec_as_ptr", since = "1.37.0")] - #[inline] - pub fn as_ptr(&self) -> *const T { - // We shadow the slice method of the same name to avoid going through - // `deref`, which creates an intermediate reference. - let ptr = self.buf.ptr(); - unsafe { - assume(!ptr.is_null()); - } - ptr - } - - /// Returns an unsafe mutable pointer to the vector's buffer, or a dangling - /// raw pointer valid for zero sized reads if the vector didn't allocate. - /// - /// The caller must ensure that the vector outlives the pointer this - /// function returns, or else it will end up pointing to garbage. - /// Modifying the vector may cause its buffer to be reallocated, - /// which would also make any pointers to it invalid. - /// - /// # Examples - /// - /// ``` - /// // Allocate vector big enough for 4 elements. - /// let size = 4; - /// let mut x: Vec<i32> = Vec::with_capacity(size); - /// let x_ptr = x.as_mut_ptr(); - /// - /// // Initialize elements via raw pointer writes, then set length. - /// unsafe { - /// for i in 0..size { - /// *x_ptr.add(i) = i as i32; - /// } - /// x.set_len(size); - /// } - /// assert_eq!(&*x, &[0, 1, 2, 3]); - /// ``` - #[stable(feature = "vec_as_ptr", since = "1.37.0")] - #[inline] - pub fn as_mut_ptr(&mut self) -> *mut T { - // We shadow the slice method of the same name to avoid going through - // `deref_mut`, which creates an intermediate reference. - let ptr = self.buf.ptr(); - unsafe { - assume(!ptr.is_null()); - } - ptr - } - - /// Returns a reference to the underlying allocator. - #[unstable(feature = "allocator_api", issue = "32838")] - #[inline] - pub fn allocator(&self) -> &A { - self.buf.allocator() - } - - /// Forces the length of the vector to `new_len`. - /// - /// This is a low-level operation that maintains none of the normal - /// invariants of the type. Normally changing the length of a vector - /// is done using one of the safe operations instead, such as - /// [`truncate`], [`resize`], [`extend`], or [`clear`]. - /// - /// [`truncate`]: Vec::truncate - /// [`resize`]: Vec::resize - /// [`extend`]: Extend::extend - /// [`clear`]: Vec::clear - /// - /// # Safety - /// - /// - `new_len` must be less than or equal to [`capacity()`]. - /// - The elements at `old_len..new_len` must be initialized. - /// - /// [`capacity()`]: Vec::capacity - /// - /// # Examples - /// - /// This method can be useful for situations in which the vector - /// is serving as a buffer for other code, particularly over FFI: - /// - /// ```no_run - /// # #![allow(dead_code)] - /// # // This is just a minimal skeleton for the doc example; - /// # // don't use this as a starting point for a real library. - /// # pub struct StreamWrapper { strm: *mut std::ffi::c_void } - /// # const Z_OK: i32 = 0; - /// # extern "C" { - /// # fn deflateGetDictionary( - /// # strm: *mut std::ffi::c_void, - /// # dictionary: *mut u8, - /// # dictLength: *mut usize, - /// # ) -> i32; - /// # } - /// # impl StreamWrapper { - /// pub fn get_dictionary(&self) -> Option<Vec<u8>> { - /// // Per the FFI method's docs, "32768 bytes is always enough". - /// let mut dict = Vec::with_capacity(32_768); - /// let mut dict_length = 0; - /// // SAFETY: When `deflateGetDictionary` returns `Z_OK`, it holds that: - /// // 1. `dict_length` elements were initialized. - /// // 2. `dict_length` <= the capacity (32_768) - /// // which makes `set_len` safe to call. - /// unsafe { - /// // Make the FFI call... - /// let r = deflateGetDictionary(self.strm, dict.as_mut_ptr(), &mut dict_length); - /// if r == Z_OK { - /// // ...and update the length to what was initialized. - /// dict.set_len(dict_length); - /// Some(dict) - /// } else { - /// None - /// } - /// } - /// } - /// # } - /// ``` - /// - /// While the following example is sound, there is a memory leak since - /// the inner vectors were not freed prior to the `set_len` call: - /// - /// ``` - /// let mut vec = vec![vec![1, 0, 0], - /// vec![0, 1, 0], - /// vec![0, 0, 1]]; - /// // SAFETY: - /// // 1. `old_len..0` is empty so no elements need to be initialized. - /// // 2. `0 <= capacity` always holds whatever `capacity` is. - /// unsafe { - /// vec.set_len(0); - /// } - /// ``` - /// - /// Normally, here, one would use [`clear`] instead to correctly drop - /// the contents and thus not leak memory. - #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - pub unsafe fn set_len(&mut self, new_len: usize) { - debug_assert!(new_len <= self.capacity()); - - self.len = new_len; - } - - /// Removes an element from the vector and returns it. - /// - /// The removed element is replaced by the last element of the vector. - /// - /// This does not preserve ordering, but is *O*(1). - /// If you need to preserve the element order, use [`remove`] instead. - /// - /// [`remove`]: Vec::remove - /// - /// # Panics - /// - /// Panics if `index` is out of bounds. - /// - /// # Examples - /// - /// ``` - /// let mut v = vec!["foo", "bar", "baz", "qux"]; - /// - /// assert_eq!(v.swap_remove(1), "bar"); - /// assert_eq!(v, ["foo", "qux", "baz"]); - /// - /// assert_eq!(v.swap_remove(0), "foo"); - /// assert_eq!(v, ["baz", "qux"]); - /// ``` - #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - pub fn swap_remove(&mut self, index: usize) -> T { - #[cold] - #[inline(never)] - fn assert_failed(index: usize, len: usize) -> ! { - panic!("swap_remove index (is {index}) should be < len (is {len})"); - } - - let len = self.len(); - if index >= len { - assert_failed(index, len); - } - unsafe { - // We replace self[index] with the last element. Note that if the - // bounds check above succeeds there must be a last element (which - // can be self[index] itself). - let value = ptr::read(self.as_ptr().add(index)); - let base_ptr = self.as_mut_ptr(); - ptr::copy(base_ptr.add(len - 1), base_ptr.add(index), 1); - self.set_len(len - 1); - value - } - } - - /// Inserts an element at position `index` within the vector, shifting all - /// elements after it to the right. - /// - /// # Panics - /// - /// Panics if `index > len`. - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec![1, 2, 3]; - /// vec.insert(1, 4); - /// assert_eq!(vec, [1, 4, 2, 3]); - /// vec.insert(4, 5); - /// assert_eq!(vec, [1, 4, 2, 3, 5]); - /// ``` - #[cfg(not(no_global_oom_handling))] - #[stable(feature = "rust1", since = "1.0.0")] - pub fn insert(&mut self, index: usize, element: T) { - #[cold] - #[inline(never)] - fn assert_failed(index: usize, len: usize) -> ! { - panic!("insertion index (is {index}) should be <= len (is {len})"); - } - - let len = self.len(); - - // space for the new element - if len == self.buf.capacity() { - self.reserve(1); - } - - unsafe { - // infallible - // The spot to put the new value - { - let p = self.as_mut_ptr().add(index); - if index < len { - // Shift everything over to make space. (Duplicating the - // `index`th element into two consecutive places.) - ptr::copy(p, p.add(1), len - index); - } else if index == len { - // No elements need shifting. - } else { - assert_failed(index, len); - } - // Write it in, overwriting the first copy of the `index`th - // element. - ptr::write(p, element); - } - self.set_len(len + 1); - } - } - - /// Removes and returns the element at position `index` within the vector, - /// shifting all elements after it to the left. - /// - /// Note: Because this shifts over the remaining elements, it has a - /// worst-case performance of *O*(*n*). If you don't need the order of elements - /// to be preserved, use [`swap_remove`] instead. If you'd like to remove - /// elements from the beginning of the `Vec`, consider using - /// [`VecDeque::pop_front`] instead. - /// - /// [`swap_remove`]: Vec::swap_remove - /// [`VecDeque::pop_front`]: crate::collections::VecDeque::pop_front - /// - /// # Panics - /// - /// Panics if `index` is out of bounds. - /// - /// # Examples - /// - /// ``` - /// let mut v = vec![1, 2, 3]; - /// assert_eq!(v.remove(1), 2); - /// assert_eq!(v, [1, 3]); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - #[track_caller] - pub fn remove(&mut self, index: usize) -> T { - #[cold] - #[inline(never)] - #[track_caller] - fn assert_failed(index: usize, len: usize) -> ! { - panic!("removal index (is {index}) should be < len (is {len})"); - } - - let len = self.len(); - if index >= len { - assert_failed(index, len); - } - unsafe { - // infallible - let ret; - { - // the place we are taking from. - let ptr = self.as_mut_ptr().add(index); - // copy it out, unsafely having a copy of the value on - // the stack and in the vector at the same time. - ret = ptr::read(ptr); - - // Shift everything down to fill in that spot. - ptr::copy(ptr.add(1), ptr, len - index - 1); - } - self.set_len(len - 1); - ret - } - } - - /// Retains only the elements specified by the predicate. - /// - /// In other words, remove all elements `e` for which `f(&e)` returns `false`. - /// This method operates in place, visiting each element exactly once in the - /// original order, and preserves the order of the retained elements. - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec![1, 2, 3, 4]; - /// vec.retain(|&x| x % 2 == 0); - /// assert_eq!(vec, [2, 4]); - /// ``` - /// - /// Because the elements are visited exactly once in the original order, - /// external state may be used to decide which elements to keep. - /// - /// ``` - /// let mut vec = vec![1, 2, 3, 4, 5]; - /// let keep = [false, true, true, false, true]; - /// let mut iter = keep.iter(); - /// vec.retain(|_| *iter.next().unwrap()); - /// assert_eq!(vec, [2, 3, 5]); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn retain<F>(&mut self, mut f: F) - where - F: FnMut(&T) -> bool, - { - self.retain_mut(|elem| f(elem)); - } - - /// Retains only the elements specified by the predicate, passing a mutable reference to it. - /// - /// In other words, remove all elements `e` such that `f(&mut e)` returns `false`. - /// This method operates in place, visiting each element exactly once in the - /// original order, and preserves the order of the retained elements. - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec![1, 2, 3, 4]; - /// vec.retain_mut(|x| if *x <= 3 { - /// *x += 1; - /// true - /// } else { - /// false - /// }); - /// assert_eq!(vec, [2, 3, 4]); - /// ``` - #[stable(feature = "vec_retain_mut", since = "1.61.0")] - pub fn retain_mut<F>(&mut self, mut f: F) - where - F: FnMut(&mut T) -> bool, - { - let original_len = self.len(); - // Avoid double drop if the drop guard is not executed, - // since we may make some holes during the process. - unsafe { self.set_len(0) }; - - // Vec: [Kept, Kept, Hole, Hole, Hole, Hole, Unchecked, Unchecked] - // |<- processed len ->| ^- next to check - // |<- deleted cnt ->| - // |<- original_len ->| - // Kept: Elements which predicate returns true on. - // Hole: Moved or dropped element slot. - // Unchecked: Unchecked valid elements. - // - // This drop guard will be invoked when predicate or `drop` of element panicked. - // It shifts unchecked elements to cover holes and `set_len` to the correct length. - // In cases when predicate and `drop` never panick, it will be optimized out. - struct BackshiftOnDrop<'a, T, A: Allocator> { - v: &'a mut Vec<T, A>, - processed_len: usize, - deleted_cnt: usize, - original_len: usize, - } - - impl<T, A: Allocator> Drop for BackshiftOnDrop<'_, T, A> { - fn drop(&mut self) { - if self.deleted_cnt > 0 { - // SAFETY: Trailing unchecked items must be valid since we never touch them. - unsafe { - ptr::copy( - self.v.as_ptr().add(self.processed_len), - self.v.as_mut_ptr().add(self.processed_len - self.deleted_cnt), - self.original_len - self.processed_len, - ); - } - } - // SAFETY: After filling holes, all items are in contiguous memory. - unsafe { - self.v.set_len(self.original_len - self.deleted_cnt); - } - } - } - - let mut g = BackshiftOnDrop { v: self, processed_len: 0, deleted_cnt: 0, original_len }; - - fn process_loop<F, T, A: Allocator, const DELETED: bool>( - original_len: usize, - f: &mut F, - g: &mut BackshiftOnDrop<'_, T, A>, - ) where - F: FnMut(&mut T) -> bool, - { - while g.processed_len != original_len { - // SAFETY: Unchecked element must be valid. - let cur = unsafe { &mut *g.v.as_mut_ptr().add(g.processed_len) }; - if !f(cur) { - // Advance early to avoid double drop if `drop_in_place` panicked. - g.processed_len += 1; - g.deleted_cnt += 1; - // SAFETY: We never touch this element again after dropped. - unsafe { ptr::drop_in_place(cur) }; - // We already advanced the counter. - if DELETED { - continue; - } else { - break; - } - } - if DELETED { - // SAFETY: `deleted_cnt` > 0, so the hole slot must not overlap with current element. - // We use copy for move, and never touch this element again. - unsafe { - let hole_slot = g.v.as_mut_ptr().add(g.processed_len - g.deleted_cnt); - ptr::copy_nonoverlapping(cur, hole_slot, 1); - } - } - g.processed_len += 1; - } - } - - // Stage 1: Nothing was deleted. - process_loop::<F, T, A, false>(original_len, &mut f, &mut g); - - // Stage 2: Some elements were deleted. - process_loop::<F, T, A, true>(original_len, &mut f, &mut g); - - // All item are processed. This can be optimized to `set_len` by LLVM. - drop(g); - } - - /// Removes all but the first of consecutive elements in the vector that resolve to the same - /// key. - /// - /// If the vector is sorted, this removes all duplicates. - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec![10, 20, 21, 30, 20]; - /// - /// vec.dedup_by_key(|i| *i / 10); - /// - /// assert_eq!(vec, [10, 20, 30, 20]); - /// ``` - #[stable(feature = "dedup_by", since = "1.16.0")] - #[inline] - pub fn dedup_by_key<F, K>(&mut self, mut key: F) - where - F: FnMut(&mut T) -> K, - K: PartialEq, - { - self.dedup_by(|a, b| key(a) == key(b)) - } - - /// Removes all but the first of consecutive elements in the vector satisfying a given equality - /// relation. - /// - /// The `same_bucket` function is passed references to two elements from the vector and - /// must determine if the elements compare equal. The elements are passed in opposite order - /// from their order in the slice, so if `same_bucket(a, b)` returns `true`, `a` is removed. - /// - /// If the vector is sorted, this removes all duplicates. - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec!["foo", "bar", "Bar", "baz", "bar"]; - /// - /// vec.dedup_by(|a, b| a.eq_ignore_ascii_case(b)); - /// - /// assert_eq!(vec, ["foo", "bar", "baz", "bar"]); - /// ``` - #[stable(feature = "dedup_by", since = "1.16.0")] - pub fn dedup_by<F>(&mut self, mut same_bucket: F) - where - F: FnMut(&mut T, &mut T) -> bool, - { - let len = self.len(); - if len <= 1 { - return; - } - - /* INVARIANT: vec.len() > read >= write > write-1 >= 0 */ - struct FillGapOnDrop<'a, T, A: core::alloc::Allocator> { - /* Offset of the element we want to check if it is duplicate */ - read: usize, - - /* Offset of the place where we want to place the non-duplicate - * when we find it. */ - write: usize, - - /* The Vec that would need correction if `same_bucket` panicked */ - vec: &'a mut Vec<T, A>, - } - - impl<'a, T, A: core::alloc::Allocator> Drop for FillGapOnDrop<'a, T, A> { - fn drop(&mut self) { - /* This code gets executed when `same_bucket` panics */ - - /* SAFETY: invariant guarantees that `read - write` - * and `len - read` never overflow and that the copy is always - * in-bounds. */ - unsafe { - let ptr = self.vec.as_mut_ptr(); - let len = self.vec.len(); - - /* How many items were left when `same_bucket` panicked. - * Basically vec[read..].len() */ - let items_left = len.wrapping_sub(self.read); - - /* Pointer to first item in vec[write..write+items_left] slice */ - let dropped_ptr = ptr.add(self.write); - /* Pointer to first item in vec[read..] slice */ - let valid_ptr = ptr.add(self.read); - - /* Copy `vec[read..]` to `vec[write..write+items_left]`. - * The slices can overlap, so `copy_nonoverlapping` cannot be used */ - ptr::copy(valid_ptr, dropped_ptr, items_left); - - /* How many items have been already dropped - * Basically vec[read..write].len() */ - let dropped = self.read.wrapping_sub(self.write); - - self.vec.set_len(len - dropped); - } - } - } - - let mut gap = FillGapOnDrop { read: 1, write: 1, vec: self }; - let ptr = gap.vec.as_mut_ptr(); - - /* Drop items while going through Vec, it should be more efficient than - * doing slice partition_dedup + truncate */ - - /* SAFETY: Because of the invariant, read_ptr, prev_ptr and write_ptr - * are always in-bounds and read_ptr never aliases prev_ptr */ - unsafe { - while gap.read < len { - let read_ptr = ptr.add(gap.read); - let prev_ptr = ptr.add(gap.write.wrapping_sub(1)); - - if same_bucket(&mut *read_ptr, &mut *prev_ptr) { - // Increase `gap.read` now since the drop may panic. - gap.read += 1; - /* We have found duplicate, drop it in-place */ - ptr::drop_in_place(read_ptr); - } else { - let write_ptr = ptr.add(gap.write); - - /* Because `read_ptr` can be equal to `write_ptr`, we either - * have to use `copy` or conditional `copy_nonoverlapping`. - * Looks like the first option is faster. */ - ptr::copy(read_ptr, write_ptr, 1); - - /* We have filled that place, so go further */ - gap.write += 1; - gap.read += 1; - } - } - - /* Technically we could let `gap` clean up with its Drop, but - * when `same_bucket` is guaranteed to not panic, this bloats a little - * the codegen, so we just do it manually */ - gap.vec.set_len(gap.write); - mem::forget(gap); - } - } - - /// Appends an element to the back of a collection. - /// - /// # Panics - /// - /// Panics if the new capacity exceeds `isize::MAX` bytes. - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec![1, 2]; - /// vec.push(3); - /// assert_eq!(vec, [1, 2, 3]); - /// ``` - #[cfg(not(no_global_oom_handling))] - #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - pub fn push(&mut self, value: T) { - // This will panic or abort if we would allocate > isize::MAX bytes - // or if the length increment would overflow for zero-sized types. - if self.len == self.buf.capacity() { - self.buf.reserve_for_push(self.len); - } - unsafe { - let end = self.as_mut_ptr().add(self.len); - ptr::write(end, value); - self.len += 1; - } - } - - /// Tries to append an element to the back of a collection. - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec![1, 2]; - /// vec.try_push(3).unwrap(); - /// assert_eq!(vec, [1, 2, 3]); - /// ``` - #[inline] - #[stable(feature = "kernel", since = "1.0.0")] - pub fn try_push(&mut self, value: T) -> Result<(), TryReserveError> { - if self.len == self.buf.capacity() { - self.buf.try_reserve_for_push(self.len)?; - } - unsafe { - let end = self.as_mut_ptr().add(self.len); - ptr::write(end, value); - self.len += 1; - } - Ok(()) - } - - /// Appends an element if there is sufficient spare capacity, otherwise an error is returned - /// with the element. - /// - /// Unlike [`push`] this method will not reallocate when there's insufficient capacity. - /// The caller should use [`reserve`] or [`try_reserve`] to ensure that there is enough capacity. - /// - /// [`push`]: Vec::push - /// [`reserve`]: Vec::reserve - /// [`try_reserve`]: Vec::try_reserve - /// - /// # Examples - /// - /// A manual, panic-free alternative to [`FromIterator`]: - /// - /// ``` - /// #![feature(vec_push_within_capacity)] - /// - /// use std::collections::TryReserveError; - /// fn from_iter_fallible<T>(iter: impl Iterator<Item=T>) -> Result<Vec<T>, TryReserveError> { - /// let mut vec = Vec::new(); - /// for value in iter { - /// if let Err(value) = vec.push_within_capacity(value) { - /// vec.try_reserve(1)?; - /// // this cannot fail, the previous line either returned or added at least 1 free slot - /// let _ = vec.push_within_capacity(value); - /// } - /// } - /// Ok(vec) - /// } - /// assert_eq!(from_iter_fallible(0..100), Ok(Vec::from_iter(0..100))); - /// ``` - #[inline] - #[unstable(feature = "vec_push_within_capacity", issue = "100486")] - pub fn push_within_capacity(&mut self, value: T) -> Result<(), T> { - if self.len == self.buf.capacity() { - return Err(value); - } - unsafe { - let end = self.as_mut_ptr().add(self.len); - ptr::write(end, value); - self.len += 1; - } - Ok(()) - } - - /// Removes the last element from a vector and returns it, or [`None`] if it - /// is empty. - /// - /// If you'd like to pop the first element, consider using - /// [`VecDeque::pop_front`] instead. - /// - /// [`VecDeque::pop_front`]: crate::collections::VecDeque::pop_front - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec![1, 2, 3]; - /// assert_eq!(vec.pop(), Some(3)); - /// assert_eq!(vec, [1, 2]); - /// ``` - #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - pub fn pop(&mut self) -> Option<T> { - if self.len == 0 { - None - } else { - unsafe { - self.len -= 1; - Some(ptr::read(self.as_ptr().add(self.len()))) - } - } - } - - /// Moves all the elements of `other` into `self`, leaving `other` empty. - /// - /// # Panics - /// - /// Panics if the new capacity exceeds `isize::MAX` bytes. - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec![1, 2, 3]; - /// let mut vec2 = vec![4, 5, 6]; - /// vec.append(&mut vec2); - /// assert_eq!(vec, [1, 2, 3, 4, 5, 6]); - /// assert_eq!(vec2, []); - /// ``` - #[cfg(not(no_global_oom_handling))] - #[inline] - #[stable(feature = "append", since = "1.4.0")] - pub fn append(&mut self, other: &mut Self) { - unsafe { - self.append_elements(other.as_slice() as _); - other.set_len(0); - } - } - - /// Appends elements to `self` from other buffer. - #[cfg(not(no_global_oom_handling))] - #[inline] - unsafe fn append_elements(&mut self, other: *const [T]) { - let count = unsafe { (*other).len() }; - self.reserve(count); - let len = self.len(); - unsafe { ptr::copy_nonoverlapping(other as *const T, self.as_mut_ptr().add(len), count) }; - self.len += count; - } - - /// Tries to append elements to `self` from other buffer. - #[inline] - unsafe fn try_append_elements(&mut self, other: *const [T]) -> Result<(), TryReserveError> { - let count = unsafe { (*other).len() }; - self.try_reserve(count)?; - let len = self.len(); - unsafe { ptr::copy_nonoverlapping(other as *const T, self.as_mut_ptr().add(len), count) }; - self.len += count; - Ok(()) - } - - /// Removes the specified range from the vector in bulk, returning all - /// removed elements as an iterator. If the iterator is dropped before - /// being fully consumed, it drops the remaining removed elements. - /// - /// The returned iterator keeps a mutable borrow on the vector to optimize - /// its implementation. - /// - /// # Panics - /// - /// Panics if the starting point is greater than the end point or if - /// the end point is greater than the length of the vector. - /// - /// # Leaking - /// - /// If the returned iterator goes out of scope without being dropped (due to - /// [`mem::forget`], for example), the vector may have lost and leaked - /// elements arbitrarily, including elements outside the range. - /// - /// # Examples - /// - /// ``` - /// let mut v = vec![1, 2, 3]; - /// let u: Vec<_> = v.drain(1..).collect(); - /// assert_eq!(v, &[1]); - /// assert_eq!(u, &[2, 3]); - /// - /// // A full range clears the vector, like `clear()` does - /// v.drain(..); - /// assert_eq!(v, &[]); - /// ``` - #[stable(feature = "drain", since = "1.6.0")] - pub fn drain<R>(&mut self, range: R) -> Drain<'_, T, A> - where - R: RangeBounds<usize>, - { - // Memory safety - // - // When the Drain is first created, it shortens the length of - // the source vector to make sure no uninitialized or moved-from elements - // are accessible at all if the Drain's destructor never gets to run. - // - // Drain will ptr::read out the values to remove. - // When finished, remaining tail of the vec is copied back to cover - // the hole, and the vector length is restored to the new length. - // - let len = self.len(); - let Range { start, end } = slice::range(range, ..len); - - unsafe { - // set self.vec length's to start, to be safe in case Drain is leaked - self.set_len(start); - let range_slice = slice::from_raw_parts(self.as_ptr().add(start), end - start); - Drain { - tail_start: end, - tail_len: len - end, - iter: range_slice.iter(), - vec: NonNull::from(self), - } - } - } - - /// Clears the vector, removing all values. - /// - /// Note that this method has no effect on the allocated capacity - /// of the vector. - /// - /// # Examples - /// - /// ``` - /// let mut v = vec![1, 2, 3]; - /// - /// v.clear(); - /// - /// assert!(v.is_empty()); - /// ``` - #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - pub fn clear(&mut self) { - let elems: *mut [T] = self.as_mut_slice(); - - // SAFETY: - // - `elems` comes directly from `as_mut_slice` and is therefore valid. - // - Setting `self.len` before calling `drop_in_place` means that, - // if an element's `Drop` impl panics, the vector's `Drop` impl will - // do nothing (leaking the rest of the elements) instead of dropping - // some twice. - unsafe { - self.len = 0; - ptr::drop_in_place(elems); - } - } - - /// Returns the number of elements in the vector, also referred to - /// as its 'length'. - /// - /// # Examples - /// - /// ``` - /// let a = vec![1, 2, 3]; - /// assert_eq!(a.len(), 3); - /// ``` - #[inline] - #[stable(feature = "rust1", since = "1.0.0")] - pub fn len(&self) -> usize { - self.len - } - - /// Returns `true` if the vector contains no elements. - /// - /// # Examples - /// - /// ``` - /// let mut v = Vec::new(); - /// assert!(v.is_empty()); - /// - /// v.push(1); - /// assert!(!v.is_empty()); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// Splits the collection into two at the given index. - /// - /// Returns a newly allocated vector containing the elements in the range - /// `[at, len)`. After the call, the original vector will be left containing - /// the elements `[0, at)` with its previous capacity unchanged. - /// - /// # Panics - /// - /// Panics if `at > len`. - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec![1, 2, 3]; - /// let vec2 = vec.split_off(1); - /// assert_eq!(vec, [1]); - /// assert_eq!(vec2, [2, 3]); - /// ``` - #[cfg(not(no_global_oom_handling))] - #[inline] - #[must_use = "use `.truncate()` if you don't need the other half"] - #[stable(feature = "split_off", since = "1.4.0")] - pub fn split_off(&mut self, at: usize) -> Self - where - A: Clone, - { - #[cold] - #[inline(never)] - fn assert_failed(at: usize, len: usize) -> ! { - panic!("`at` split index (is {at}) should be <= len (is {len})"); - } - - if at > self.len() { - assert_failed(at, self.len()); - } - - if at == 0 { - // the new vector can take over the original buffer and avoid the copy - return mem::replace( - self, - Vec::with_capacity_in(self.capacity(), self.allocator().clone()), - ); - } - - let other_len = self.len - at; - let mut other = Vec::with_capacity_in(other_len, self.allocator().clone()); - - // Unsafely `set_len` and copy items to `other`. - unsafe { - self.set_len(at); - other.set_len(other_len); - - ptr::copy_nonoverlapping(self.as_ptr().add(at), other.as_mut_ptr(), other.len()); - } - other - } - - /// Resizes the `Vec` in-place so that `len` is equal to `new_len`. - /// - /// If `new_len` is greater than `len`, the `Vec` is extended by the - /// difference, with each additional slot filled with the result of - /// calling the closure `f`. The return values from `f` will end up - /// in the `Vec` in the order they have been generated. - /// - /// If `new_len` is less than `len`, the `Vec` is simply truncated. - /// - /// This method uses a closure to create new values on every push. If - /// you'd rather [`Clone`] a given value, use [`Vec::resize`]. If you - /// want to use the [`Default`] trait to generate values, you can - /// pass [`Default::default`] as the second argument. - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec![1, 2, 3]; - /// vec.resize_with(5, Default::default); - /// assert_eq!(vec, [1, 2, 3, 0, 0]); - /// - /// let mut vec = vec![]; - /// let mut p = 1; - /// vec.resize_with(4, || { p *= 2; p }); - /// assert_eq!(vec, [2, 4, 8, 16]); - /// ``` - #[cfg(not(no_global_oom_handling))] - #[stable(feature = "vec_resize_with", since = "1.33.0")] - pub fn resize_with<F>(&mut self, new_len: usize, f: F) - where - F: FnMut() -> T, - { - let len = self.len(); - if new_len > len { - self.extend_trusted(iter::repeat_with(f).take(new_len - len)); - } else { - self.truncate(new_len); - } - } - - /// Consumes and leaks the `Vec`, returning a mutable reference to the contents, - /// `&'a mut [T]`. Note that the type `T` must outlive the chosen lifetime - /// `'a`. If the type has only static references, or none at all, then this - /// may be chosen to be `'static`. - /// - /// As of Rust 1.57, this method does not reallocate or shrink the `Vec`, - /// so the leaked allocation may include unused capacity that is not part - /// of the returned slice. - /// - /// This function is mainly useful for data that lives for the remainder of - /// the program's life. Dropping the returned reference will cause a memory - /// leak. - /// - /// # Examples - /// - /// Simple usage: - /// - /// ``` - /// let x = vec![1, 2, 3]; - /// let static_ref: &'static mut [usize] = x.leak(); - /// static_ref[0] += 1; - /// assert_eq!(static_ref, &[2, 2, 3]); - /// ``` - #[stable(feature = "vec_leak", since = "1.47.0")] - #[inline] - pub fn leak<'a>(self) -> &'a mut [T] - where - A: 'a, - { - let mut me = ManuallyDrop::new(self); - unsafe { slice::from_raw_parts_mut(me.as_mut_ptr(), me.len) } - } - - /// Returns the remaining spare capacity of the vector as a slice of - /// `MaybeUninit<T>`. - /// - /// The returned slice can be used to fill the vector with data (e.g. by - /// reading from a file) before marking the data as initialized using the - /// [`set_len`] method. - /// - /// [`set_len`]: Vec::set_len - /// - /// # Examples - /// - /// ``` - /// // Allocate vector big enough for 10 elements. - /// let mut v = Vec::with_capacity(10); - /// - /// // Fill in the first 3 elements. - /// let uninit = v.spare_capacity_mut(); - /// uninit[0].write(0); - /// uninit[1].write(1); - /// uninit[2].write(2); - /// - /// // Mark the first 3 elements of the vector as being initialized. - /// unsafe { - /// v.set_len(3); - /// } - /// - /// assert_eq!(&v, &[0, 1, 2]); - /// ``` - #[stable(feature = "vec_spare_capacity", since = "1.60.0")] - #[inline] - pub fn spare_capacity_mut(&mut self) -> &mut [MaybeUninit<T>] { - // Note: - // This method is not implemented in terms of `split_at_spare_mut`, - // to prevent invalidation of pointers to the buffer. - unsafe { - slice::from_raw_parts_mut( - self.as_mut_ptr().add(self.len) as *mut MaybeUninit<T>, - self.buf.capacity() - self.len, - ) - } - } - - /// Returns vector content as a slice of `T`, along with the remaining spare - /// capacity of the vector as a slice of `MaybeUninit<T>`. - /// - /// The returned spare capacity slice can be used to fill the vector with data - /// (e.g. by reading from a file) before marking the data as initialized using - /// the [`set_len`] method. - /// - /// [`set_len`]: Vec::set_len - /// - /// Note that this is a low-level API, which should be used with care for - /// optimization purposes. If you need to append data to a `Vec` - /// you can use [`push`], [`extend`], [`extend_from_slice`], - /// [`extend_from_within`], [`insert`], [`append`], [`resize`] or - /// [`resize_with`], depending on your exact needs. - /// - /// [`push`]: Vec::push - /// [`extend`]: Vec::extend - /// [`extend_from_slice`]: Vec::extend_from_slice - /// [`extend_from_within`]: Vec::extend_from_within - /// [`insert`]: Vec::insert - /// [`append`]: Vec::append - /// [`resize`]: Vec::resize - /// [`resize_with`]: Vec::resize_with - /// - /// # Examples - /// - /// ``` - /// #![feature(vec_split_at_spare)] - /// - /// let mut v = vec![1, 1, 2]; - /// - /// // Reserve additional space big enough for 10 elements. - /// v.reserve(10); - /// - /// let (init, uninit) = v.split_at_spare_mut(); - /// let sum = init.iter().copied().sum::<u32>(); - /// - /// // Fill in the next 4 elements. - /// uninit[0].write(sum); - /// uninit[1].write(sum * 2); - /// uninit[2].write(sum * 3); - /// uninit[3].write(sum * 4); - /// - /// // Mark the 4 elements of the vector as being initialized. - /// unsafe { - /// let len = v.len(); - /// v.set_len(len + 4); - /// } - /// - /// assert_eq!(&v, &[1, 1, 2, 4, 8, 12, 16]); - /// ``` - #[unstable(feature = "vec_split_at_spare", issue = "81944")] - #[inline] - pub fn split_at_spare_mut(&mut self) -> (&mut [T], &mut [MaybeUninit<T>]) { - // SAFETY: - // - len is ignored and so never changed - let (init, spare, _) = unsafe { self.split_at_spare_mut_with_len() }; - (init, spare) - } - - /// Safety: changing returned .2 (&mut usize) is considered the same as calling `.set_len(_)`. - /// - /// This method provides unique access to all vec parts at once in `extend_from_within`. - unsafe fn split_at_spare_mut_with_len( - &mut self, - ) -> (&mut [T], &mut [MaybeUninit<T>], &mut usize) { - let ptr = self.as_mut_ptr(); - // SAFETY: - // - `ptr` is guaranteed to be valid for `self.len` elements - // - but the allocation extends out to `self.buf.capacity()` elements, possibly - // uninitialized - let spare_ptr = unsafe { ptr.add(self.len) }; - let spare_ptr = spare_ptr.cast::<MaybeUninit<T>>(); - let spare_len = self.buf.capacity() - self.len; - - // SAFETY: - // - `ptr` is guaranteed to be valid for `self.len` elements - // - `spare_ptr` is pointing one element past the buffer, so it doesn't overlap with `initialized` - unsafe { - let initialized = slice::from_raw_parts_mut(ptr, self.len); - let spare = slice::from_raw_parts_mut(spare_ptr, spare_len); - - (initialized, spare, &mut self.len) - } - } -} - -impl<T: Clone, A: Allocator> Vec<T, A> { - /// Resizes the `Vec` in-place so that `len` is equal to `new_len`. - /// - /// If `new_len` is greater than `len`, the `Vec` is extended by the - /// difference, with each additional slot filled with `value`. - /// If `new_len` is less than `len`, the `Vec` is simply truncated. - /// - /// This method requires `T` to implement [`Clone`], - /// in order to be able to clone the passed value. - /// If you need more flexibility (or want to rely on [`Default`] instead of - /// [`Clone`]), use [`Vec::resize_with`]. - /// If you only need to resize to a smaller size, use [`Vec::truncate`]. - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec!["hello"]; - /// vec.resize(3, "world"); - /// assert_eq!(vec, ["hello", "world", "world"]); - /// - /// let mut vec = vec![1, 2, 3, 4]; - /// vec.resize(2, 0); - /// assert_eq!(vec, [1, 2]); - /// ``` - #[cfg(not(no_global_oom_handling))] - #[stable(feature = "vec_resize", since = "1.5.0")] - pub fn resize(&mut self, new_len: usize, value: T) { - let len = self.len(); - - if new_len > len { - self.extend_with(new_len - len, ExtendElement(value)) - } else { - self.truncate(new_len); - } - } - - /// Tries to resize the `Vec` in-place so that `len` is equal to `new_len`. - /// - /// If `new_len` is greater than `len`, the `Vec` is extended by the - /// difference, with each additional slot filled with `value`. - /// If `new_len` is less than `len`, the `Vec` is simply truncated. - /// - /// This method requires `T` to implement [`Clone`], - /// in order to be able to clone the passed value. - /// If you need more flexibility (or want to rely on [`Default`] instead of - /// [`Clone`]), use [`Vec::resize_with`]. - /// If you only need to resize to a smaller size, use [`Vec::truncate`]. - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec!["hello"]; - /// vec.try_resize(3, "world").unwrap(); - /// assert_eq!(vec, ["hello", "world", "world"]); - /// - /// let mut vec = vec![1, 2, 3, 4]; - /// vec.try_resize(2, 0).unwrap(); - /// assert_eq!(vec, [1, 2]); - /// - /// let mut vec = vec![42]; - /// let result = vec.try_resize(usize::MAX, 0); - /// assert!(result.is_err()); - /// ``` - #[stable(feature = "kernel", since = "1.0.0")] - pub fn try_resize(&mut self, new_len: usize, value: T) -> Result<(), TryReserveError> { - let len = self.len(); - - if new_len > len { - self.try_extend_with(new_len - len, ExtendElement(value)) - } else { - self.truncate(new_len); - Ok(()) - } - } - - /// Clones and appends all elements in a slice to the `Vec`. - /// - /// Iterates over the slice `other`, clones each element, and then appends - /// it to this `Vec`. The `other` slice is traversed in-order. - /// - /// Note that this function is same as [`extend`] except that it is - /// specialized to work with slices instead. If and when Rust gets - /// specialization this function will likely be deprecated (but still - /// available). - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec![1]; - /// vec.extend_from_slice(&[2, 3, 4]); - /// assert_eq!(vec, [1, 2, 3, 4]); - /// ``` - /// - /// [`extend`]: Vec::extend - #[cfg(not(no_global_oom_handling))] - #[stable(feature = "vec_extend_from_slice", since = "1.6.0")] - pub fn extend_from_slice(&mut self, other: &[T]) { - self.spec_extend(other.iter()) - } - - /// Tries to clone and append all elements in a slice to the `Vec`. - /// - /// Iterates over the slice `other`, clones each element, and then appends - /// it to this `Vec`. The `other` slice is traversed in-order. - /// - /// Note that this function is same as [`extend`] except that it is - /// specialized to work with slices instead. If and when Rust gets - /// specialization this function will likely be deprecated (but still - /// available). - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec![1]; - /// vec.try_extend_from_slice(&[2, 3, 4]).unwrap(); - /// assert_eq!(vec, [1, 2, 3, 4]); - /// ``` - /// - /// [`extend`]: Vec::extend - #[stable(feature = "kernel", since = "1.0.0")] - pub fn try_extend_from_slice(&mut self, other: &[T]) -> Result<(), TryReserveError> { - self.try_spec_extend(other.iter()) - } - - /// Copies elements from `src` range to the end of the vector. - /// - /// # Panics - /// - /// Panics if the starting point is greater than the end point or if - /// the end point is greater than the length of the vector. - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec![0, 1, 2, 3, 4]; - /// - /// vec.extend_from_within(2..); - /// assert_eq!(vec, [0, 1, 2, 3, 4, 2, 3, 4]); - /// - /// vec.extend_from_within(..2); - /// assert_eq!(vec, [0, 1, 2, 3, 4, 2, 3, 4, 0, 1]); - /// - /// vec.extend_from_within(4..8); - /// assert_eq!(vec, [0, 1, 2, 3, 4, 2, 3, 4, 0, 1, 4, 2, 3, 4]); - /// ``` - #[cfg(not(no_global_oom_handling))] - #[stable(feature = "vec_extend_from_within", since = "1.53.0")] - pub fn extend_from_within<R>(&mut self, src: R) - where - R: RangeBounds<usize>, - { - let range = slice::range(src, ..self.len()); - self.reserve(range.len()); - - // SAFETY: - // - `slice::range` guarantees that the given range is valid for indexing self - unsafe { - self.spec_extend_from_within(range); - } - } -} - -impl<T, A: Allocator, const N: usize> Vec<[T; N], A> { - /// Takes a `Vec<[T; N]>` and flattens it into a `Vec<T>`. - /// - /// # Panics - /// - /// Panics if the length of the resulting vector would overflow a `usize`. - /// - /// This is only possible when flattening a vector of arrays of zero-sized - /// types, and thus tends to be irrelevant in practice. If - /// `size_of::<T>() > 0`, this will never panic. - /// - /// # Examples - /// - /// ``` - /// #![feature(slice_flatten)] - /// - /// let mut vec = vec![[1, 2, 3], [4, 5, 6], [7, 8, 9]]; - /// assert_eq!(vec.pop(), Some([7, 8, 9])); - /// - /// let mut flattened = vec.into_flattened(); - /// assert_eq!(flattened.pop(), Some(6)); - /// ``` - #[unstable(feature = "slice_flatten", issue = "95629")] - pub fn into_flattened(self) -> Vec<T, A> { - let (ptr, len, cap, alloc) = self.into_raw_parts_with_alloc(); - let (new_len, new_cap) = if T::IS_ZST { - (len.checked_mul(N).expect("vec len overflow"), usize::MAX) - } else { - // SAFETY: - // - `cap * N` cannot overflow because the allocation is already in - // the address space. - // - Each `[T; N]` has `N` valid elements, so there are `len * N` - // valid elements in the allocation. - unsafe { (len.unchecked_mul(N), cap.unchecked_mul(N)) } - }; - // SAFETY: - // - `ptr` was allocated by `self` - // - `ptr` is well-aligned because `[T; N]` has the same alignment as `T`. - // - `new_cap` refers to the same sized allocation as `cap` because - // `new_cap * size_of::<T>()` == `cap * size_of::<[T; N]>()` - // - `len` <= `cap`, so `len * N` <= `cap * N`. - unsafe { Vec::<T, A>::from_raw_parts_in(ptr.cast(), new_len, new_cap, alloc) } - } -} - -// This code generalizes `extend_with_{element,default}`. -trait ExtendWith<T> { - fn next(&mut self) -> T; - fn last(self) -> T; -} - -struct ExtendElement<T>(T); -impl<T: Clone> ExtendWith<T> for ExtendElement<T> { - fn next(&mut self) -> T { - self.0.clone() - } - fn last(self) -> T { - self.0 - } -} - -impl<T, A: Allocator> Vec<T, A> { - #[cfg(not(no_global_oom_handling))] - /// Extend the vector by `n` values, using the given generator. - fn extend_with<E: ExtendWith<T>>(&mut self, n: usize, mut value: E) { - self.reserve(n); - - unsafe { - let mut ptr = self.as_mut_ptr().add(self.len()); - // Use SetLenOnDrop to work around bug where compiler - // might not realize the store through `ptr` through self.set_len() - // don't alias. - let mut local_len = SetLenOnDrop::new(&mut self.len); - - // Write all elements except the last one - for _ in 1..n { - ptr::write(ptr, value.next()); - ptr = ptr.add(1); - // Increment the length in every step in case next() panics - local_len.increment_len(1); - } - - if n > 0 { - // We can write the last element directly without cloning needlessly - ptr::write(ptr, value.last()); - local_len.increment_len(1); - } - - // len set by scope guard - } - } - - /// Try to extend the vector by `n` values, using the given generator. - fn try_extend_with<E: ExtendWith<T>>(&mut self, n: usize, mut value: E) -> Result<(), TryReserveError> { - self.try_reserve(n)?; - - unsafe { - let mut ptr = self.as_mut_ptr().add(self.len()); - // Use SetLenOnDrop to work around bug where compiler - // might not realize the store through `ptr` through self.set_len() - // don't alias. - let mut local_len = SetLenOnDrop::new(&mut self.len); - - // Write all elements except the last one - for _ in 1..n { - ptr::write(ptr, value.next()); - ptr = ptr.add(1); - // Increment the length in every step in case next() panics - local_len.increment_len(1); - } - - if n > 0 { - // We can write the last element directly without cloning needlessly - ptr::write(ptr, value.last()); - local_len.increment_len(1); - } - - // len set by scope guard - Ok(()) - } - } -} - -impl<T: PartialEq, A: Allocator> Vec<T, A> { - /// Removes consecutive repeated elements in the vector according to the - /// [`PartialEq`] trait implementation. - /// - /// If the vector is sorted, this removes all duplicates. - /// - /// # Examples - /// - /// ``` - /// let mut vec = vec![1, 2, 2, 3, 2]; - /// - /// vec.dedup(); - /// - /// assert_eq!(vec, [1, 2, 3, 2]); - /// ``` - #[stable(feature = "rust1", since = "1.0.0")] - #[inline] - pub fn dedup(&mut self) { - self.dedup_by(|a, b| a == b) - } -} - -//////////////////////////////////////////////////////////////////////////////// -// Internal methods and functions -//////////////////////////////////////////////////////////////////////////////// - -#[doc(hidden)] -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "rust1", since = "1.0.0")] -pub fn from_elem<T: Clone>(elem: T, n: usize) -> Vec<T> { - <T as SpecFromElem>::from_elem(elem, n, Global) -} - -#[doc(hidden)] -#[cfg(not(no_global_oom_handling))] -#[unstable(feature = "allocator_api", issue = "32838")] -pub fn from_elem_in<T: Clone, A: Allocator>(elem: T, n: usize, alloc: A) -> Vec<T, A> { - <T as SpecFromElem>::from_elem(elem, n, alloc) -} - -trait ExtendFromWithinSpec { - /// # Safety - /// - /// - `src` needs to be valid index - /// - `self.capacity() - self.len()` must be `>= src.len()` - unsafe fn spec_extend_from_within(&mut self, src: Range<usize>); -} - -impl<T: Clone, A: Allocator> ExtendFromWithinSpec for Vec<T, A> { - default unsafe fn spec_extend_from_within(&mut self, src: Range<usize>) { - // SAFETY: - // - len is increased only after initializing elements - let (this, spare, len) = unsafe { self.split_at_spare_mut_with_len() }; - - // SAFETY: - // - caller guarantees that src is a valid index - let to_clone = unsafe { this.get_unchecked(src) }; - - iter::zip(to_clone, spare) - .map(|(src, dst)| dst.write(src.clone())) - // Note: - // - Element was just initialized with `MaybeUninit::write`, so it's ok to increase len - // - len is increased after each element to prevent leaks (see issue #82533) - .for_each(|_| *len += 1); - } -} - -impl<T: Copy, A: Allocator> ExtendFromWithinSpec for Vec<T, A> { - unsafe fn spec_extend_from_within(&mut self, src: Range<usize>) { - let count = src.len(); - { - let (init, spare) = self.split_at_spare_mut(); - - // SAFETY: - // - caller guarantees that `src` is a valid index - let source = unsafe { init.get_unchecked(src) }; - - // SAFETY: - // - Both pointers are created from unique slice references (`&mut [_]`) - // so they are valid and do not overlap. - // - Elements are :Copy so it's OK to copy them, without doing - // anything with the original values - // - `count` is equal to the len of `source`, so source is valid for - // `count` reads - // - `.reserve(count)` guarantees that `spare.len() >= count` so spare - // is valid for `count` writes - unsafe { ptr::copy_nonoverlapping(source.as_ptr(), spare.as_mut_ptr() as _, count) }; - } - - // SAFETY: - // - The elements were just initialized by `copy_nonoverlapping` - self.len += count; - } -} - -//////////////////////////////////////////////////////////////////////////////// -// Common trait implementations for Vec -//////////////////////////////////////////////////////////////////////////////// - -#[stable(feature = "rust1", since = "1.0.0")] -impl<T, A: Allocator> ops::Deref for Vec<T, A> { - type Target = [T]; - - #[inline] - fn deref(&self) -> &[T] { - unsafe { slice::from_raw_parts(self.as_ptr(), self.len) } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<T, A: Allocator> ops::DerefMut for Vec<T, A> { - #[inline] - fn deref_mut(&mut self) -> &mut [T] { - unsafe { slice::from_raw_parts_mut(self.as_mut_ptr(), self.len) } - } -} - -#[cfg(not(no_global_oom_handling))] -trait SpecCloneFrom { - fn clone_from(this: &mut Self, other: &Self); -} - -#[cfg(not(no_global_oom_handling))] -impl<T: Clone, A: Allocator> SpecCloneFrom for Vec<T, A> { - default fn clone_from(this: &mut Self, other: &Self) { - // drop anything that will not be overwritten - this.truncate(other.len()); - - // self.len <= other.len due to the truncate above, so the - // slices here are always in-bounds. - let (init, tail) = other.split_at(this.len()); - - // reuse the contained values' allocations/resources. - this.clone_from_slice(init); - this.extend_from_slice(tail); - } -} - -#[cfg(not(no_global_oom_handling))] -impl<T: Copy, A: Allocator> SpecCloneFrom for Vec<T, A> { - fn clone_from(this: &mut Self, other: &Self) { - this.clear(); - this.extend_from_slice(other); - } -} - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "rust1", since = "1.0.0")] -impl<T: Clone, A: Allocator + Clone> Clone for Vec<T, A> { - #[cfg(not(test))] - fn clone(&self) -> Self { - let alloc = self.allocator().clone(); - <[T]>::to_vec_in(&**self, alloc) - } - - // HACK(japaric): with cfg(test) the inherent `[T]::to_vec` method, which is - // required for this method definition, is not available. Instead use the - // `slice::to_vec` function which is only available with cfg(test) - // NB see the slice::hack module in slice.rs for more information - #[cfg(test)] - fn clone(&self) -> Self { - let alloc = self.allocator().clone(); - crate::slice::to_vec(&**self, alloc) - } - - fn clone_from(&mut self, other: &Self) { - SpecCloneFrom::clone_from(self, other) - } -} - -/// The hash of a vector is the same as that of the corresponding slice, -/// as required by the `core::borrow::Borrow` implementation. -/// -/// ``` -/// #![feature(build_hasher_simple_hash_one)] -/// use std::hash::BuildHasher; -/// -/// let b = std::collections::hash_map::RandomState::new(); -/// let v: Vec<u8> = vec![0xa8, 0x3c, 0x09]; -/// let s: &[u8] = &[0xa8, 0x3c, 0x09]; -/// assert_eq!(b.hash_one(v), b.hash_one(s)); -/// ``` -#[stable(feature = "rust1", since = "1.0.0")] -impl<T: Hash, A: Allocator> Hash for Vec<T, A> { - #[inline] - fn hash<H: Hasher>(&self, state: &mut H) { - Hash::hash(&**self, state) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -#[rustc_on_unimplemented( - message = "vector indices are of type `usize` or ranges of `usize`", - label = "vector indices are of type `usize` or ranges of `usize`" -)] -impl<T, I: SliceIndex<[T]>, A: Allocator> Index<I> for Vec<T, A> { - type Output = I::Output; - - #[inline] - fn index(&self, index: I) -> &Self::Output { - Index::index(&**self, index) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -#[rustc_on_unimplemented( - message = "vector indices are of type `usize` or ranges of `usize`", - label = "vector indices are of type `usize` or ranges of `usize`" -)] -impl<T, I: SliceIndex<[T]>, A: Allocator> IndexMut<I> for Vec<T, A> { - #[inline] - fn index_mut(&mut self, index: I) -> &mut Self::Output { - IndexMut::index_mut(&mut **self, index) - } -} - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "rust1", since = "1.0.0")] -impl<T> FromIterator<T> for Vec<T> { - #[inline] - fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> Vec<T> { - <Self as SpecFromIter<T, I::IntoIter>>::from_iter(iter.into_iter()) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<T, A: Allocator> IntoIterator for Vec<T, A> { - type Item = T; - type IntoIter = IntoIter<T, A>; - - /// Creates a consuming iterator, that is, one that moves each value out of - /// the vector (from start to end). The vector cannot be used after calling - /// this. - /// - /// # Examples - /// - /// ``` - /// let v = vec!["a".to_string(), "b".to_string()]; - /// let mut v_iter = v.into_iter(); - /// - /// let first_element: Option<String> = v_iter.next(); - /// - /// assert_eq!(first_element, Some("a".to_string())); - /// assert_eq!(v_iter.next(), Some("b".to_string())); - /// assert_eq!(v_iter.next(), None); - /// ``` - #[inline] - fn into_iter(self) -> Self::IntoIter { - unsafe { - let mut me = ManuallyDrop::new(self); - let alloc = ManuallyDrop::new(ptr::read(me.allocator())); - let begin = me.as_mut_ptr(); - let end = if T::IS_ZST { - begin.wrapping_byte_add(me.len()) - } else { - begin.add(me.len()) as *const T - }; - let cap = me.buf.capacity(); - IntoIter { - buf: NonNull::new_unchecked(begin), - phantom: PhantomData, - cap, - alloc, - ptr: begin, - end, - } - } - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a, T, A: Allocator> IntoIterator for &'a Vec<T, A> { - type Item = &'a T; - type IntoIter = slice::Iter<'a, T>; - - fn into_iter(self) -> Self::IntoIter { - self.iter() - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<'a, T, A: Allocator> IntoIterator for &'a mut Vec<T, A> { - type Item = &'a mut T; - type IntoIter = slice::IterMut<'a, T>; - - fn into_iter(self) -> Self::IntoIter { - self.iter_mut() - } -} - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "rust1", since = "1.0.0")] -impl<T, A: Allocator> Extend<T> for Vec<T, A> { - #[inline] - fn extend<I: IntoIterator<Item = T>>(&mut self, iter: I) { - <Self as SpecExtend<T, I::IntoIter>>::spec_extend(self, iter.into_iter()) - } - - #[inline] - fn extend_one(&mut self, item: T) { - self.push(item); - } - - #[inline] - fn extend_reserve(&mut self, additional: usize) { - self.reserve(additional); - } -} - -impl<T, A: Allocator> Vec<T, A> { - // leaf method to which various SpecFrom/SpecExtend implementations delegate when - // they have no further optimizations to apply - #[cfg(not(no_global_oom_handling))] - fn extend_desugared<I: Iterator<Item = T>>(&mut self, mut iterator: I) { - // This is the case for a general iterator. - // - // This function should be the moral equivalent of: - // - // for item in iterator { - // self.push(item); - // } - while let Some(element) = iterator.next() { - let len = self.len(); - if len == self.capacity() { - let (lower, _) = iterator.size_hint(); - self.reserve(lower.saturating_add(1)); - } - unsafe { - ptr::write(self.as_mut_ptr().add(len), element); - // Since next() executes user code which can panic we have to bump the length - // after each step. - // NB can't overflow since we would have had to alloc the address space - self.set_len(len + 1); - } - } - } - - // leaf method to which various SpecFrom/SpecExtend implementations delegate when - // they have no further optimizations to apply - fn try_extend_desugared<I: Iterator<Item = T>>(&mut self, mut iterator: I) -> Result<(), TryReserveError> { - // This is the case for a general iterator. - // - // This function should be the moral equivalent of: - // - // for item in iterator { - // self.push(item); - // } - while let Some(element) = iterator.next() { - let len = self.len(); - if len == self.capacity() { - let (lower, _) = iterator.size_hint(); - self.try_reserve(lower.saturating_add(1))?; - } - unsafe { - ptr::write(self.as_mut_ptr().add(len), element); - // Since next() executes user code which can panic we have to bump the length - // after each step. - // NB can't overflow since we would have had to alloc the address space - self.set_len(len + 1); - } - } - - Ok(()) - } - - // specific extend for `TrustedLen` iterators, called both by the specializations - // and internal places where resolving specialization makes compilation slower - #[cfg(not(no_global_oom_handling))] - fn extend_trusted(&mut self, iterator: impl iter::TrustedLen<Item = T>) { - let (low, high) = iterator.size_hint(); - if let Some(additional) = high { - debug_assert_eq!( - low, - additional, - "TrustedLen iterator's size hint is not exact: {:?}", - (low, high) - ); - self.reserve(additional); - unsafe { - let ptr = self.as_mut_ptr(); - let mut local_len = SetLenOnDrop::new(&mut self.len); - iterator.for_each(move |element| { - ptr::write(ptr.add(local_len.current_len()), element); - // Since the loop executes user code which can panic we have to update - // the length every step to correctly drop what we've written. - // NB can't overflow since we would have had to alloc the address space - local_len.increment_len(1); - }); - } - } else { - // Per TrustedLen contract a `None` upper bound means that the iterator length - // truly exceeds usize::MAX, which would eventually lead to a capacity overflow anyway. - // Since the other branch already panics eagerly (via `reserve()`) we do the same here. - // This avoids additional codegen for a fallback code path which would eventually - // panic anyway. - panic!("capacity overflow"); - } - } - - // specific extend for `TrustedLen` iterators, called both by the specializations - // and internal places where resolving specialization makes compilation slower - fn try_extend_trusted(&mut self, iterator: impl iter::TrustedLen<Item = T>) -> Result<(), TryReserveError> { - let (low, high) = iterator.size_hint(); - if let Some(additional) = high { - debug_assert_eq!( - low, - additional, - "TrustedLen iterator's size hint is not exact: {:?}", - (low, high) - ); - self.try_reserve(additional)?; - unsafe { - let ptr = self.as_mut_ptr(); - let mut local_len = SetLenOnDrop::new(&mut self.len); - iterator.for_each(move |element| { - ptr::write(ptr.add(local_len.current_len()), element); - // Since the loop executes user code which can panic we have to update - // the length every step to correctly drop what we've written. - // NB can't overflow since we would have had to alloc the address space - local_len.increment_len(1); - }); - } - Ok(()) - } else { - Err(TryReserveErrorKind::CapacityOverflow.into()) - } - } - - /// Creates a splicing iterator that replaces the specified range in the vector - /// with the given `replace_with` iterator and yields the removed items. - /// `replace_with` does not need to be the same length as `range`. - /// - /// `range` is removed even if the iterator is not consumed until the end. - /// - /// It is unspecified how many elements are removed from the vector - /// if the `Splice` value is leaked. - /// - /// The input iterator `replace_with` is only consumed when the `Splice` value is dropped. - /// - /// This is optimal if: - /// - /// * The tail (elements in the vector after `range`) is empty, - /// * or `replace_with` yields fewer or equal elements than `range`’s length - /// * or the lower bound of its `size_hint()` is exact. - /// - /// Otherwise, a temporary vector is allocated and the tail is moved twice. - /// - /// # Panics - /// - /// Panics if the starting point is greater than the end point or if - /// the end point is greater than the length of the vector. - /// - /// # Examples - /// - /// ``` - /// let mut v = vec![1, 2, 3, 4]; - /// let new = [7, 8, 9]; - /// let u: Vec<_> = v.splice(1..3, new).collect(); - /// assert_eq!(v, &[1, 7, 8, 9, 4]); - /// assert_eq!(u, &[2, 3]); - /// ``` - #[cfg(not(no_global_oom_handling))] - #[inline] - #[stable(feature = "vec_splice", since = "1.21.0")] - pub fn splice<R, I>(&mut self, range: R, replace_with: I) -> Splice<'_, I::IntoIter, A> - where - R: RangeBounds<usize>, - I: IntoIterator<Item = T>, - { - Splice { drain: self.drain(range), replace_with: replace_with.into_iter() } - } - - /// Creates an iterator which uses a closure to determine if an element should be removed. - /// - /// If the closure returns true, then the element is removed and yielded. - /// If the closure returns false, the element will remain in the vector and will not be yielded - /// by the iterator. - /// - /// Using this method is equivalent to the following code: - /// - /// ``` - /// # let some_predicate = |x: &mut i32| { *x == 2 || *x == 3 || *x == 6 }; - /// # let mut vec = vec![1, 2, 3, 4, 5, 6]; - /// let mut i = 0; - /// while i < vec.len() { - /// if some_predicate(&mut vec[i]) { - /// let val = vec.remove(i); - /// // your code here - /// } else { - /// i += 1; - /// } - /// } - /// - /// # assert_eq!(vec, vec![1, 4, 5]); - /// ``` - /// - /// But `drain_filter` is easier to use. `drain_filter` is also more efficient, - /// because it can backshift the elements of the array in bulk. - /// - /// Note that `drain_filter` also lets you mutate every element in the filter closure, - /// regardless of whether you choose to keep or remove it. - /// - /// # Examples - /// - /// Splitting an array into evens and odds, reusing the original allocation: - /// - /// ``` - /// #![feature(drain_filter)] - /// let mut numbers = vec![1, 2, 3, 4, 5, 6, 8, 9, 11, 13, 14, 15]; - /// - /// let evens = numbers.drain_filter(|x| *x % 2 == 0).collect::<Vec<_>>(); - /// let odds = numbers; - /// - /// assert_eq!(evens, vec![2, 4, 6, 8, 14]); - /// assert_eq!(odds, vec![1, 3, 5, 9, 11, 13, 15]); - /// ``` - #[unstable(feature = "drain_filter", reason = "recently added", issue = "43244")] - pub fn drain_filter<F>(&mut self, filter: F) -> DrainFilter<'_, T, F, A> - where - F: FnMut(&mut T) -> bool, - { - let old_len = self.len(); - - // Guard against us getting leaked (leak amplification) - unsafe { - self.set_len(0); - } - - DrainFilter { vec: self, idx: 0, del: 0, old_len, pred: filter, panic_flag: false } - } -} - -/// Extend implementation that copies elements out of references before pushing them onto the Vec. -/// -/// This implementation is specialized for slice iterators, where it uses [`copy_from_slice`] to -/// append the entire slice at once. -/// -/// [`copy_from_slice`]: slice::copy_from_slice -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "extend_ref", since = "1.2.0")] -impl<'a, T: Copy + 'a, A: Allocator + 'a> Extend<&'a T> for Vec<T, A> { - fn extend<I: IntoIterator<Item = &'a T>>(&mut self, iter: I) { - self.spec_extend(iter.into_iter()) - } - - #[inline] - fn extend_one(&mut self, &item: &'a T) { - self.push(item); - } - - #[inline] - fn extend_reserve(&mut self, additional: usize) { - self.reserve(additional); - } -} - -/// Implements comparison of vectors, [lexicographically](core::cmp::Ord#lexicographical-comparison). -#[stable(feature = "rust1", since = "1.0.0")] -impl<T: PartialOrd, A: Allocator> PartialOrd for Vec<T, A> { - #[inline] - fn partial_cmp(&self, other: &Self) -> Option<Ordering> { - PartialOrd::partial_cmp(&**self, &**other) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<T: Eq, A: Allocator> Eq for Vec<T, A> {} - -/// Implements ordering of vectors, [lexicographically](core::cmp::Ord#lexicographical-comparison). -#[stable(feature = "rust1", since = "1.0.0")] -impl<T: Ord, A: Allocator> Ord for Vec<T, A> { - #[inline] - fn cmp(&self, other: &Self) -> Ordering { - Ord::cmp(&**self, &**other) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -unsafe impl<#[may_dangle] T, A: Allocator> Drop for Vec<T, A> { - fn drop(&mut self) { - unsafe { - // use drop for [T] - // use a raw slice to refer to the elements of the vector as weakest necessary type; - // could avoid questions of validity in certain cases - ptr::drop_in_place(ptr::slice_from_raw_parts_mut(self.as_mut_ptr(), self.len)) - } - // RawVec handles deallocation - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -#[rustc_const_unstable(feature = "const_default_impls", issue = "87864")] -impl<T> const Default for Vec<T> { - /// Creates an empty `Vec<T>`. - /// - /// The vector will not allocate until elements are pushed onto it. - fn default() -> Vec<T> { - Vec::new() - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<T: fmt::Debug, A: Allocator> fmt::Debug for Vec<T, A> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Debug::fmt(&**self, f) - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<T, A: Allocator> AsRef<Vec<T, A>> for Vec<T, A> { - fn as_ref(&self) -> &Vec<T, A> { - self - } -} - -#[stable(feature = "vec_as_mut", since = "1.5.0")] -impl<T, A: Allocator> AsMut<Vec<T, A>> for Vec<T, A> { - fn as_mut(&mut self) -> &mut Vec<T, A> { - self - } -} - -#[stable(feature = "rust1", since = "1.0.0")] -impl<T, A: Allocator> AsRef<[T]> for Vec<T, A> { - fn as_ref(&self) -> &[T] { - self - } -} - -#[stable(feature = "vec_as_mut", since = "1.5.0")] -impl<T, A: Allocator> AsMut<[T]> for Vec<T, A> { - fn as_mut(&mut self) -> &mut [T] { - self - } -} - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "rust1", since = "1.0.0")] -impl<T: Clone> From<&[T]> for Vec<T> { - /// Allocate a `Vec<T>` and fill it by cloning `s`'s items. - /// - /// # Examples - /// - /// ``` - /// assert_eq!(Vec::from(&[1, 2, 3][..]), vec![1, 2, 3]); - /// ``` - #[cfg(not(test))] - fn from(s: &[T]) -> Vec<T> { - s.to_vec() - } - #[cfg(test)] - fn from(s: &[T]) -> Vec<T> { - crate::slice::to_vec(s, Global) - } -} - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "vec_from_mut", since = "1.19.0")] -impl<T: Clone> From<&mut [T]> for Vec<T> { - /// Allocate a `Vec<T>` and fill it by cloning `s`'s items. - /// - /// # Examples - /// - /// ``` - /// assert_eq!(Vec::from(&mut [1, 2, 3][..]), vec![1, 2, 3]); - /// ``` - #[cfg(not(test))] - fn from(s: &mut [T]) -> Vec<T> { - s.to_vec() - } - #[cfg(test)] - fn from(s: &mut [T]) -> Vec<T> { - crate::slice::to_vec(s, Global) - } -} - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "vec_from_array", since = "1.44.0")] -impl<T, const N: usize> From<[T; N]> for Vec<T> { - /// Allocate a `Vec<T>` and move `s`'s items into it. - /// - /// # Examples - /// - /// ``` - /// assert_eq!(Vec::from([1, 2, 3]), vec![1, 2, 3]); - /// ``` - #[cfg(not(test))] - fn from(s: [T; N]) -> Vec<T> { - <[T]>::into_vec( - #[rustc_box] - Box::new(s), - ) - } - - #[cfg(test)] - fn from(s: [T; N]) -> Vec<T> { - crate::slice::into_vec(Box::new(s)) - } -} - -#[cfg(not(no_borrow))] -#[stable(feature = "vec_from_cow_slice", since = "1.14.0")] -impl<'a, T> From<Cow<'a, [T]>> for Vec<T> -where - [T]: ToOwned<Owned = Vec<T>>, -{ - /// Convert a clone-on-write slice into a vector. - /// - /// If `s` already owns a `Vec<T>`, it will be returned directly. - /// If `s` is borrowing a slice, a new `Vec<T>` will be allocated and - /// filled by cloning `s`'s items into it. - /// - /// # Examples - /// - /// ``` - /// # use std::borrow::Cow; - /// let o: Cow<[i32]> = Cow::Owned(vec![1, 2, 3]); - /// let b: Cow<[i32]> = Cow::Borrowed(&[1, 2, 3]); - /// assert_eq!(Vec::from(o), Vec::from(b)); - /// ``` - fn from(s: Cow<'a, [T]>) -> Vec<T> { - s.into_owned() - } -} - -// note: test pulls in std, which causes errors here -#[cfg(not(test))] -#[stable(feature = "vec_from_box", since = "1.18.0")] -impl<T, A: Allocator> From<Box<[T], A>> for Vec<T, A> { - /// Convert a boxed slice into a vector by transferring ownership of - /// the existing heap allocation. - /// - /// # Examples - /// - /// ``` - /// let b: Box<[i32]> = vec![1, 2, 3].into_boxed_slice(); - /// assert_eq!(Vec::from(b), vec![1, 2, 3]); - /// ``` - fn from(s: Box<[T], A>) -> Self { - s.into_vec() - } -} - -// note: test pulls in std, which causes errors here -#[cfg(not(no_global_oom_handling))] -#[cfg(not(test))] -#[stable(feature = "box_from_vec", since = "1.20.0")] -impl<T, A: Allocator> From<Vec<T, A>> for Box<[T], A> { - /// Convert a vector into a boxed slice. - /// - /// If `v` has excess capacity, its items will be moved into a - /// newly-allocated buffer with exactly the right capacity. - /// - /// # Examples - /// - /// ``` - /// assert_eq!(Box::from(vec![1, 2, 3]), vec![1, 2, 3].into_boxed_slice()); - /// ``` - /// - /// Any excess capacity is removed: - /// ``` - /// let mut vec = Vec::with_capacity(10); - /// vec.extend([1, 2, 3]); - /// - /// assert_eq!(Box::from(vec), vec![1, 2, 3].into_boxed_slice()); - /// ``` - fn from(v: Vec<T, A>) -> Self { - v.into_boxed_slice() - } -} - -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "rust1", since = "1.0.0")] -impl From<&str> for Vec<u8> { - /// Allocate a `Vec<u8>` and fill it with a UTF-8 string. - /// - /// # Examples - /// - /// ``` - /// assert_eq!(Vec::from("123"), vec![b'1', b'2', b'3']); - /// ``` - fn from(s: &str) -> Vec<u8> { - From::from(s.as_bytes()) - } -} - -#[stable(feature = "array_try_from_vec", since = "1.48.0")] -impl<T, A: Allocator, const N: usize> TryFrom<Vec<T, A>> for [T; N] { - type Error = Vec<T, A>; - - /// Gets the entire contents of the `Vec<T>` as an array, - /// if its size exactly matches that of the requested array. - /// - /// # Examples - /// - /// ``` - /// assert_eq!(vec![1, 2, 3].try_into(), Ok([1, 2, 3])); - /// assert_eq!(<Vec<i32>>::new().try_into(), Ok([])); - /// ``` - /// - /// If the length doesn't match, the input comes back in `Err`: - /// ``` - /// let r: Result<[i32; 4], _> = (0..10).collect::<Vec<_>>().try_into(); - /// assert_eq!(r, Err(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9])); - /// ``` - /// - /// If you're fine with just getting a prefix of the `Vec<T>`, - /// you can call [`.truncate(N)`](Vec::truncate) first. - /// ``` - /// let mut v = String::from("hello world").into_bytes(); - /// v.sort(); - /// v.truncate(2); - /// let [a, b]: [_; 2] = v.try_into().unwrap(); - /// assert_eq!(a, b' '); - /// assert_eq!(b, b'd'); - /// ``` - fn try_from(mut vec: Vec<T, A>) -> Result<[T; N], Vec<T, A>> { - if vec.len() != N { - return Err(vec); - } - - // SAFETY: `.set_len(0)` is always sound. - unsafe { vec.set_len(0) }; - - // SAFETY: A `Vec`'s pointer is always aligned properly, and - // the alignment the array needs is the same as the items. - // We checked earlier that we have sufficient items. - // The items will not double-drop as the `set_len` - // tells the `Vec` not to also drop them. - let array = unsafe { ptr::read(vec.as_ptr() as *const [T; N]) }; - Ok(array) - } -} diff --git a/rust/alloc/vec/partial_eq.rs b/rust/alloc/vec/partial_eq.rs deleted file mode 100644 index 10ad4e492287..000000000000 --- a/rust/alloc/vec/partial_eq.rs +++ /dev/null @@ -1,49 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 OR MIT - -use crate::alloc::Allocator; -#[cfg(not(no_global_oom_handling))] -use crate::borrow::Cow; - -use super::Vec; - -macro_rules! __impl_slice_eq1 { - ([$($vars:tt)*] $lhs:ty, $rhs:ty $(where $ty:ty: $bound:ident)?, #[$stability:meta]) => { - #[$stability] - impl<T, U, $($vars)*> PartialEq<$rhs> for $lhs - where - T: PartialEq<U>, - $($ty: $bound)? - { - #[inline] - fn eq(&self, other: &$rhs) -> bool { self[..] == other[..] } - #[inline] - fn ne(&self, other: &$rhs) -> bool { self[..] != other[..] } - } - } -} - -__impl_slice_eq1! { [A1: Allocator, A2: Allocator] Vec<T, A1>, Vec<U, A2>, #[stable(feature = "rust1", since = "1.0.0")] } -__impl_slice_eq1! { [A: Allocator] Vec<T, A>, &[U], #[stable(feature = "rust1", since = "1.0.0")] } -__impl_slice_eq1! { [A: Allocator] Vec<T, A>, &mut [U], #[stable(feature = "rust1", since = "1.0.0")] } -__impl_slice_eq1! { [A: Allocator] &[T], Vec<U, A>, #[stable(feature = "partialeq_vec_for_ref_slice", since = "1.46.0")] } -__impl_slice_eq1! { [A: Allocator] &mut [T], Vec<U, A>, #[stable(feature = "partialeq_vec_for_ref_slice", since = "1.46.0")] } -__impl_slice_eq1! { [A: Allocator] Vec<T, A>, [U], #[stable(feature = "partialeq_vec_for_slice", since = "1.48.0")] } -__impl_slice_eq1! { [A: Allocator] [T], Vec<U, A>, #[stable(feature = "partialeq_vec_for_slice", since = "1.48.0")] } -#[cfg(not(no_global_oom_handling))] -__impl_slice_eq1! { [A: Allocator] Cow<'_, [T]>, Vec<U, A> where T: Clone, #[stable(feature = "rust1", since = "1.0.0")] } -#[cfg(not(no_global_oom_handling))] -__impl_slice_eq1! { [] Cow<'_, [T]>, &[U] where T: Clone, #[stable(feature = "rust1", since = "1.0.0")] } -#[cfg(not(no_global_oom_handling))] -__impl_slice_eq1! { [] Cow<'_, [T]>, &mut [U] where T: Clone, #[stable(feature = "rust1", since = "1.0.0")] } -__impl_slice_eq1! { [A: Allocator, const N: usize] Vec<T, A>, [U; N], #[stable(feature = "rust1", since = "1.0.0")] } -__impl_slice_eq1! { [A: Allocator, const N: usize] Vec<T, A>, &[U; N], #[stable(feature = "rust1", since = "1.0.0")] } - -// NOTE: some less important impls are omitted to reduce code bloat -// FIXME(Centril): Reconsider this? -//__impl_slice_eq1! { [const N: usize] Vec<A>, &mut [B; N], } -//__impl_slice_eq1! { [const N: usize] [A; N], Vec<B>, } -//__impl_slice_eq1! { [const N: usize] &[A; N], Vec<B>, } -//__impl_slice_eq1! { [const N: usize] &mut [A; N], Vec<B>, } -//__impl_slice_eq1! { [const N: usize] Cow<'a, [A]>, [B; N], } -//__impl_slice_eq1! { [const N: usize] Cow<'a, [A]>, &[B; N], } -//__impl_slice_eq1! { [const N: usize] Cow<'a, [A]>, &mut [B; N], } diff --git a/rust/alloc/vec/set_len_on_drop.rs b/rust/alloc/vec/set_len_on_drop.rs deleted file mode 100644 index d3c7297b80ec..000000000000 --- a/rust/alloc/vec/set_len_on_drop.rs +++ /dev/null @@ -1,35 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 OR MIT - -// Set the length of the vec when the `SetLenOnDrop` value goes out of scope. -// -// The idea is: The length field in SetLenOnDrop is a local variable -// that the optimizer will see does not alias with any stores through the Vec's data -// pointer. This is a workaround for alias analysis issue #32155 -pub(super) struct SetLenOnDrop<'a> { - len: &'a mut usize, - local_len: usize, -} - -impl<'a> SetLenOnDrop<'a> { - #[inline] - pub(super) fn new(len: &'a mut usize) -> Self { - SetLenOnDrop { local_len: *len, len } - } - - #[inline] - pub(super) fn increment_len(&mut self, increment: usize) { - self.local_len += increment; - } - - #[inline] - pub(super) fn current_len(&self) -> usize { - self.local_len - } -} - -impl Drop for SetLenOnDrop<'_> { - #[inline] - fn drop(&mut self) { - *self.len = self.local_len; - } -} diff --git a/rust/alloc/vec/spec_extend.rs b/rust/alloc/vec/spec_extend.rs deleted file mode 100644 index a6a735201e59..000000000000 --- a/rust/alloc/vec/spec_extend.rs +++ /dev/null @@ -1,119 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 OR MIT - -use crate::alloc::Allocator; -use crate::collections::TryReserveError; -use core::iter::TrustedLen; -use core::slice::{self}; - -use super::{IntoIter, Vec}; - -// Specialization trait used for Vec::extend -#[cfg(not(no_global_oom_handling))] -pub(super) trait SpecExtend<T, I> { - fn spec_extend(&mut self, iter: I); -} - -// Specialization trait used for Vec::try_extend -pub(super) trait TrySpecExtend<T, I> { - fn try_spec_extend(&mut self, iter: I) -> Result<(), TryReserveError>; -} - -#[cfg(not(no_global_oom_handling))] -impl<T, I, A: Allocator> SpecExtend<T, I> for Vec<T, A> -where - I: Iterator<Item = T>, -{ - default fn spec_extend(&mut self, iter: I) { - self.extend_desugared(iter) - } -} - -impl<T, I, A: Allocator> TrySpecExtend<T, I> for Vec<T, A> -where - I: Iterator<Item = T>, -{ - default fn try_spec_extend(&mut self, iter: I) -> Result<(), TryReserveError> { - self.try_extend_desugared(iter) - } -} - -#[cfg(not(no_global_oom_handling))] -impl<T, I, A: Allocator> SpecExtend<T, I> for Vec<T, A> -where - I: TrustedLen<Item = T>, -{ - default fn spec_extend(&mut self, iterator: I) { - self.extend_trusted(iterator) - } -} - -impl<T, I, A: Allocator> TrySpecExtend<T, I> for Vec<T, A> -where - I: TrustedLen<Item = T>, -{ - default fn try_spec_extend(&mut self, iterator: I) -> Result<(), TryReserveError> { - self.try_extend_trusted(iterator) - } -} - -#[cfg(not(no_global_oom_handling))] -impl<T, A: Allocator> SpecExtend<T, IntoIter<T>> for Vec<T, A> { - fn spec_extend(&mut self, mut iterator: IntoIter<T>) { - unsafe { - self.append_elements(iterator.as_slice() as _); - } - iterator.forget_remaining_elements(); - } -} - -impl<T, A: Allocator> TrySpecExtend<T, IntoIter<T>> for Vec<T, A> { - fn try_spec_extend(&mut self, mut iterator: IntoIter<T>) -> Result<(), TryReserveError> { - unsafe { - self.try_append_elements(iterator.as_slice() as _)?; - } - iterator.forget_remaining_elements(); - Ok(()) - } -} - -#[cfg(not(no_global_oom_handling))] -impl<'a, T: 'a, I, A: Allocator + 'a> SpecExtend<&'a T, I> for Vec<T, A> -where - I: Iterator<Item = &'a T>, - T: Clone, -{ - default fn spec_extend(&mut self, iterator: I) { - self.spec_extend(iterator.cloned()) - } -} - -impl<'a, T: 'a, I, A: Allocator + 'a> TrySpecExtend<&'a T, I> for Vec<T, A> -where - I: Iterator<Item = &'a T>, - T: Clone, -{ - default fn try_spec_extend(&mut self, iterator: I) -> Result<(), TryReserveError> { - self.try_spec_extend(iterator.cloned()) - } -} - -#[cfg(not(no_global_oom_handling))] -impl<'a, T: 'a, A: Allocator + 'a> SpecExtend<&'a T, slice::Iter<'a, T>> for Vec<T, A> -where - T: Copy, -{ - fn spec_extend(&mut self, iterator: slice::Iter<'a, T>) { - let slice = iterator.as_slice(); - unsafe { self.append_elements(slice) }; - } -} - -impl<'a, T: 'a, A: Allocator + 'a> TrySpecExtend<&'a T, slice::Iter<'a, T>> for Vec<T, A> -where - T: Copy, -{ - fn try_spec_extend(&mut self, iterator: slice::Iter<'a, T>) -> Result<(), TryReserveError> { - let slice = iterator.as_slice(); - unsafe { self.try_append_elements(slice) } - } -} diff --git a/rust/bindgen_parameters b/rust/bindgen_parameters index 552d9a85925b..a721d466bee4 100644 --- a/rust/bindgen_parameters +++ b/rust/bindgen_parameters @@ -20,3 +20,7 @@ # `seccomp`'s comment gets understood as a doctest --no-doc-comments + +# These functions use the `__preserve_most` calling convention, which neither bindgen +# nor Rust currently understand, and which Clang currently declares to be unstable. +--blocklist-function __list_.*_report diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 058954961bfc..ddb5644d4fd9 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -6,13 +6,22 @@ * Sorted alphabetically. */ +#include <kunit/test.h> #include <linux/errname.h> -#include <linux/slab.h> +#include <linux/ethtool.h> +#include <linux/jiffies.h> +#include <linux/mdio.h> +#include <linux/phy.h> #include <linux/refcount.h> -#include <linux/wait.h> #include <linux/sched.h> +#include <linux/slab.h> +#include <linux/wait.h> +#include <linux/workqueue.h> /* `bindgen` gets confused at certain things. */ -const size_t BINDINGS_ARCH_SLAB_MINALIGN = ARCH_SLAB_MINALIGN; -const gfp_t BINDINGS_GFP_KERNEL = GFP_KERNEL; -const gfp_t BINDINGS___GFP_ZERO = __GFP_ZERO; +const size_t RUST_CONST_HELPER_ARCH_SLAB_MINALIGN = ARCH_SLAB_MINALIGN; +const gfp_t RUST_CONST_HELPER_GFP_ATOMIC = GFP_ATOMIC; +const gfp_t RUST_CONST_HELPER_GFP_KERNEL = GFP_KERNEL; +const gfp_t RUST_CONST_HELPER_GFP_KERNEL_ACCOUNT = GFP_KERNEL_ACCOUNT; +const gfp_t RUST_CONST_HELPER_GFP_NOWAIT = GFP_NOWAIT; +const gfp_t RUST_CONST_HELPER___GFP_ZERO = __GFP_ZERO; diff --git a/rust/bindings/lib.rs b/rust/bindings/lib.rs index 9bcbea04dac3..40ddaee50d8b 100644 --- a/rust/bindings/lib.rs +++ b/rust/bindings/lib.rs @@ -48,6 +48,3 @@ mod bindings_helper { } pub use bindings_raw::*; - -pub const GFP_KERNEL: gfp_t = BINDINGS_GFP_KERNEL; -pub const __GFP_ZERO: gfp_t = BINDINGS___GFP_ZERO; diff --git a/rust/compiler_builtins.rs b/rust/compiler_builtins.rs index 43378357ece9..bba2922c6ef7 100644 --- a/rust/compiler_builtins.rs +++ b/rust/compiler_builtins.rs @@ -19,6 +19,7 @@ //! [`compiler_builtins`]: https://github.com/rust-lang/compiler-builtins //! [`compiler-rt`]: https://compiler-rt.llvm.org/ +#![allow(internal_features)] #![feature(compiler_builtins)] #![compiler_builtins] #![no_builtins] @@ -37,14 +38,21 @@ macro_rules! define_panicking_intrinsics( ); define_panicking_intrinsics!("`f32` should not be used", { + __addsf3, __eqsf2, __gesf2, __lesf2, + __ltsf2, + __mulsf3, __nesf2, __unordsf2, }); define_panicking_intrinsics!("`f64` should not be used", { + __adddf3, + __ledf2, + __ltdf2, + __muldf3, __unorddf2, }); diff --git a/rust/exports.c b/rust/exports.c index 83e2a7070cae..3803c21d1403 100644 --- a/rust/exports.c +++ b/rust/exports.c @@ -11,7 +11,7 @@ * accidentally exposed. */ -#include <linux/module.h> +#include <linux/export.h> #define EXPORT_SYMBOL_RUST_GPL(sym) extern int sym; EXPORT_SYMBOL_GPL(sym) diff --git a/rust/helpers.c b/rust/helpers.c index bb594da56137..2c37a0f5d7a8 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -4,7 +4,7 @@ * cannot be called either. This file explicitly creates functions ("helpers") * that wrap those so that they can be called from Rust. * - * Even though Rust kernel modules should never use directly the bindings, some + * Even though Rust kernel modules should never use the bindings directly, some * of these helpers need to be exported because Rust generics and inlined * functions may not get their code generated in the crate where they are * defined. Other helpers, called from non-inline functions, may not be @@ -16,17 +16,22 @@ * * All symbols are exported as GPL-only to guarantee no GPL-only feature is * accidentally exposed. + * + * Sorted alphabetically. */ +#include <kunit/test-bug.h> #include <linux/bug.h> #include <linux/build_bug.h> #include <linux/err.h> #include <linux/errname.h> -#include <linux/refcount.h> #include <linux/mutex.h> -#include <linux/spinlock.h> +#include <linux/refcount.h> #include <linux/sched/signal.h> +#include <linux/slab.h> +#include <linux/spinlock.h> #include <linux/wait.h> +#include <linux/workqueue.h> __noreturn void rust_helper_BUG(void) { @@ -135,20 +140,44 @@ void rust_helper_put_task_struct(struct task_struct *t) } EXPORT_SYMBOL_GPL(rust_helper_put_task_struct); +struct kunit *rust_helper_kunit_get_current_test(void) +{ + return kunit_get_current_test(); +} +EXPORT_SYMBOL_GPL(rust_helper_kunit_get_current_test); + +void rust_helper_init_work_with_key(struct work_struct *work, work_func_t func, + bool onstack, const char *name, + struct lock_class_key *key) +{ + __init_work(work, onstack); + work->data = (atomic_long_t)WORK_DATA_INIT(); + lockdep_init_map(&work->lockdep_map, name, key, 0); + INIT_LIST_HEAD(&work->entry); + work->func = func; +} +EXPORT_SYMBOL_GPL(rust_helper_init_work_with_key); + +void * __must_check __realloc_size(2) +rust_helper_krealloc(const void *objp, size_t new_size, gfp_t flags) +{ + return krealloc(objp, new_size, flags); +} +EXPORT_SYMBOL_GPL(rust_helper_krealloc); + /* - * We use `bindgen`'s `--size_t-is-usize` option to bind the C `size_t` type - * as the Rust `usize` type, so we can use it in contexts where Rust - * expects a `usize` like slice (array) indices. `usize` is defined to be - * the same as C's `uintptr_t` type (can hold any pointer) but not - * necessarily the same as `size_t` (can hold the size of any single - * object). Most modern platforms use the same concrete integer type for + * `bindgen` binds the C `size_t` type as the Rust `usize` type, so we can + * use it in contexts where Rust expects a `usize` like slice (array) indices. + * `usize` is defined to be the same as C's `uintptr_t` type (can hold any + * pointer) but not necessarily the same as `size_t` (can hold the size of any + * single object). Most modern platforms use the same concrete integer type for * both of them, but in case we find ourselves on a platform where * that's not true, fail early instead of risking ABI or * integer-overflow issues. * * If your platform fails this assertion, it means that you are in - * danger of integer-overflow bugs (even if you attempt to remove - * `--size_t-is-usize`). It may be easiest to change the kernel ABI on + * danger of integer-overflow bugs (even if you attempt to add + * `--no-size_t-is-usize`). It may be easiest to change the kernel ABI on * your platform such that `size_t` matches `uintptr_t` (i.e., to increase * `size_t`, because `uintptr_t` has to be at least as big as `size_t`). */ diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs new file mode 100644 index 000000000000..531b5e471cb1 --- /dev/null +++ b/rust/kernel/alloc.rs @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Extensions to the [`alloc`] crate. + +#[cfg(not(test))] +#[cfg(not(testlib))] +mod allocator; +pub mod box_ext; +pub mod vec_ext; + +/// Indicates an allocation error. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub struct AllocError; + +/// Flags to be used when allocating memory. +/// +/// They can be combined with the operators `|`, `&`, and `!`. +/// +/// Values can be used from the [`flags`] module. +#[derive(Clone, Copy)] +pub struct Flags(u32); + +impl core::ops::BitOr for Flags { + type Output = Self; + fn bitor(self, rhs: Self) -> Self::Output { + Self(self.0 | rhs.0) + } +} + +impl core::ops::BitAnd for Flags { + type Output = Self; + fn bitand(self, rhs: Self) -> Self::Output { + Self(self.0 & rhs.0) + } +} + +impl core::ops::Not for Flags { + type Output = Self; + fn not(self) -> Self::Output { + Self(!self.0) + } +} + +/// Allocation flags. +/// +/// These are meant to be used in functions that can allocate memory. +pub mod flags { + use super::Flags; + + /// Zeroes out the allocated memory. + /// + /// This is normally or'd with other flags. + pub const __GFP_ZERO: Flags = Flags(bindings::__GFP_ZERO); + + /// Users can not sleep and need the allocation to succeed. + /// + /// A lower watermark is applied to allow access to "atomic reserves". The current + /// implementation doesn't support NMI and few other strict non-preemptive contexts (e.g. + /// raw_spin_lock). The same applies to [`GFP_NOWAIT`]. + pub const GFP_ATOMIC: Flags = Flags(bindings::GFP_ATOMIC); + + /// Typical for kernel-internal allocations. The caller requires ZONE_NORMAL or a lower zone + /// for direct access but can direct reclaim. + pub const GFP_KERNEL: Flags = Flags(bindings::GFP_KERNEL); + + /// The same as [`GFP_KERNEL`], except the allocation is accounted to kmemcg. + pub const GFP_KERNEL_ACCOUNT: Flags = Flags(bindings::GFP_KERNEL_ACCOUNT); + + /// Ror kernel allocations that should not stall for direct reclaim, start physical IO or + /// use any filesystem callback. It is very likely to fail to allocate memory, even for very + /// small allocations. + pub const GFP_NOWAIT: Flags = Flags(bindings::GFP_NOWAIT); +} diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs new file mode 100644 index 000000000000..229642960cd1 --- /dev/null +++ b/rust/kernel/alloc/allocator.rs @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Allocator support. + +use super::{flags::*, Flags}; +use core::alloc::{GlobalAlloc, Layout}; +use core::ptr; + +struct KernelAllocator; + +/// Calls `krealloc` with a proper size to alloc a new object aligned to `new_layout`'s alignment. +/// +/// # Safety +/// +/// - `ptr` can be either null or a pointer which has been allocated by this allocator. +/// - `new_layout` must have a non-zero size. +pub(crate) unsafe fn krealloc_aligned(ptr: *mut u8, new_layout: Layout, flags: Flags) -> *mut u8 { + // Customized layouts from `Layout::from_size_align()` can have size < align, so pad first. + let layout = new_layout.pad_to_align(); + + let mut size = layout.size(); + + if layout.align() > bindings::ARCH_SLAB_MINALIGN { + // The alignment requirement exceeds the slab guarantee, thus try to enlarge the size + // to use the "power-of-two" size/alignment guarantee (see comments in `kmalloc()` for + // more information). + // + // Note that `layout.size()` (after padding) is guaranteed to be a multiple of + // `layout.align()`, so `next_power_of_two` gives enough alignment guarantee. + size = size.next_power_of_two(); + } + + // SAFETY: + // - `ptr` is either null or a pointer returned from a previous `k{re}alloc()` by the + // function safety requirement. + // - `size` is greater than 0 since it's either a `layout.size()` (which cannot be zero + // according to the function safety requirement) or a result from `next_power_of_two()`. + unsafe { bindings::krealloc(ptr as *const core::ffi::c_void, size, flags.0) as *mut u8 } +} + +unsafe impl GlobalAlloc for KernelAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + // SAFETY: `ptr::null_mut()` is null and `layout` has a non-zero size by the function safety + // requirement. + unsafe { krealloc_aligned(ptr::null_mut(), layout, GFP_KERNEL) } + } + + unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) { + unsafe { + bindings::kfree(ptr as *const core::ffi::c_void); + } + } + + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + // SAFETY: + // - `new_size`, when rounded up to the nearest multiple of `layout.align()`, will not + // overflow `isize` by the function safety requirement. + // - `layout.align()` is a proper alignment (i.e. not zero and must be a power of two). + let layout = unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) }; + + // SAFETY: + // - `ptr` is either null or a pointer allocated by this allocator by the function safety + // requirement. + // - the size of `layout` is not zero because `new_size` is not zero by the function safety + // requirement. + unsafe { krealloc_aligned(ptr, layout, GFP_KERNEL) } + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + // SAFETY: `ptr::null_mut()` is null and `layout` has a non-zero size by the function safety + // requirement. + unsafe { krealloc_aligned(ptr::null_mut(), layout, GFP_KERNEL | __GFP_ZERO) } + } +} + +#[global_allocator] +static ALLOCATOR: KernelAllocator = KernelAllocator; + +// See <https://github.com/rust-lang/rust/pull/86844>. +#[no_mangle] +static __rust_no_alloc_shim_is_unstable: u8 = 0; diff --git a/rust/kernel/alloc/box_ext.rs b/rust/kernel/alloc/box_ext.rs new file mode 100644 index 000000000000..829cb1c1cf9e --- /dev/null +++ b/rust/kernel/alloc/box_ext.rs @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Extensions to [`Box`] for fallible allocations. + +use super::{AllocError, Flags}; +use alloc::boxed::Box; +use core::mem::MaybeUninit; + +/// Extensions to [`Box`]. +pub trait BoxExt<T>: Sized { + /// Allocates a new box. + /// + /// The allocation may fail, in which case an error is returned. + fn new(x: T, flags: Flags) -> Result<Self, AllocError>; + + /// Allocates a new uninitialised box. + /// + /// The allocation may fail, in which case an error is returned. + fn new_uninit(flags: Flags) -> Result<Box<MaybeUninit<T>>, AllocError>; +} + +impl<T> BoxExt<T> for Box<T> { + fn new(x: T, flags: Flags) -> Result<Self, AllocError> { + let b = <Self as BoxExt<_>>::new_uninit(flags)?; + Ok(Box::write(b, x)) + } + + #[cfg(any(test, testlib))] + fn new_uninit(_flags: Flags) -> Result<Box<MaybeUninit<T>>, AllocError> { + Ok(Box::new_uninit()) + } + + #[cfg(not(any(test, testlib)))] + fn new_uninit(flags: Flags) -> Result<Box<MaybeUninit<T>>, AllocError> { + let ptr = if core::mem::size_of::<MaybeUninit<T>>() == 0 { + core::ptr::NonNull::<_>::dangling().as_ptr() + } else { + let layout = core::alloc::Layout::new::<MaybeUninit<T>>(); + + // SAFETY: Memory is being allocated (first arg is null). The only other source of + // safety issues is sleeping on atomic context, which is addressed by klint. Lastly, + // the type is not a SZT (checked above). + let ptr = + unsafe { super::allocator::krealloc_aligned(core::ptr::null_mut(), layout, flags) }; + if ptr.is_null() { + return Err(AllocError); + } + + ptr.cast::<MaybeUninit<T>>() + }; + + // SAFETY: For non-zero-sized types, we allocate above using the global allocator. For + // zero-sized types, we use `NonNull::dangling`. + Ok(unsafe { Box::from_raw(ptr) }) + } +} diff --git a/rust/kernel/alloc/vec_ext.rs b/rust/kernel/alloc/vec_ext.rs new file mode 100644 index 000000000000..1297a4be32e8 --- /dev/null +++ b/rust/kernel/alloc/vec_ext.rs @@ -0,0 +1,185 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Extensions to [`Vec`] for fallible allocations. + +use super::{AllocError, Flags}; +use alloc::vec::Vec; + +/// Extensions to [`Vec`]. +pub trait VecExt<T>: Sized { + /// Creates a new [`Vec`] instance with at least the given capacity. + /// + /// # Examples + /// + /// ``` + /// let v = Vec::<u32>::with_capacity(20, GFP_KERNEL)?; + /// + /// assert!(v.capacity() >= 20); + /// # Ok::<(), Error>(()) + /// ``` + fn with_capacity(capacity: usize, flags: Flags) -> Result<Self, AllocError>; + + /// Appends an element to the back of the [`Vec`] instance. + /// + /// # Examples + /// + /// ``` + /// let mut v = Vec::new(); + /// v.push(1, GFP_KERNEL)?; + /// assert_eq!(&v, &[1]); + /// + /// v.push(2, GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 2]); + /// # Ok::<(), Error>(()) + /// ``` + fn push(&mut self, v: T, flags: Flags) -> Result<(), AllocError>; + + /// Pushes clones of the elements of slice into the [`Vec`] instance. + /// + /// # Examples + /// + /// ``` + /// let mut v = Vec::new(); + /// v.push(1, GFP_KERNEL)?; + /// + /// v.extend_from_slice(&[20, 30, 40], GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 20, 30, 40]); + /// + /// v.extend_from_slice(&[50, 60], GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 20, 30, 40, 50, 60]); + /// # Ok::<(), Error>(()) + /// ``` + fn extend_from_slice(&mut self, other: &[T], flags: Flags) -> Result<(), AllocError> + where + T: Clone; + + /// Ensures that the capacity exceeds the length by at least `additional` elements. + /// + /// # Examples + /// + /// ``` + /// let mut v = Vec::new(); + /// v.push(1, GFP_KERNEL)?; + /// + /// v.reserve(10, GFP_KERNEL)?; + /// let cap = v.capacity(); + /// assert!(cap >= 10); + /// + /// v.reserve(10, GFP_KERNEL)?; + /// let new_cap = v.capacity(); + /// assert_eq!(new_cap, cap); + /// + /// # Ok::<(), Error>(()) + /// ``` + fn reserve(&mut self, additional: usize, flags: Flags) -> Result<(), AllocError>; +} + +impl<T> VecExt<T> for Vec<T> { + fn with_capacity(capacity: usize, flags: Flags) -> Result<Self, AllocError> { + let mut v = Vec::new(); + <Self as VecExt<_>>::reserve(&mut v, capacity, flags)?; + Ok(v) + } + + fn push(&mut self, v: T, flags: Flags) -> Result<(), AllocError> { + <Self as VecExt<_>>::reserve(self, 1, flags)?; + let s = self.spare_capacity_mut(); + s[0].write(v); + + // SAFETY: We just initialised the first spare entry, so it is safe to increase the length + // by 1. We also know that the new length is <= capacity because of the previous call to + // `reserve` above. + unsafe { self.set_len(self.len() + 1) }; + Ok(()) + } + + fn extend_from_slice(&mut self, other: &[T], flags: Flags) -> Result<(), AllocError> + where + T: Clone, + { + <Self as VecExt<_>>::reserve(self, other.len(), flags)?; + for (slot, item) in core::iter::zip(self.spare_capacity_mut(), other) { + slot.write(item.clone()); + } + + // SAFETY: We just initialised the `other.len()` spare entries, so it is safe to increase + // the length by the same amount. We also know that the new length is <= capacity because + // of the previous call to `reserve` above. + unsafe { self.set_len(self.len() + other.len()) }; + Ok(()) + } + + #[cfg(any(test, testlib))] + fn reserve(&mut self, additional: usize, _flags: Flags) -> Result<(), AllocError> { + Vec::reserve(self, additional); + Ok(()) + } + + #[cfg(not(any(test, testlib)))] + fn reserve(&mut self, additional: usize, flags: Flags) -> Result<(), AllocError> { + let len = self.len(); + let cap = self.capacity(); + + if cap - len >= additional { + return Ok(()); + } + + if core::mem::size_of::<T>() == 0 { + // The capacity is already `usize::MAX` for SZTs, we can't go higher. + return Err(AllocError); + } + + // We know cap is <= `isize::MAX` because `Layout::array` fails if the resulting byte size + // is greater than `isize::MAX`. So the multiplication by two won't overflow. + let new_cap = core::cmp::max(cap * 2, len.checked_add(additional).ok_or(AllocError)?); + let layout = core::alloc::Layout::array::<T>(new_cap).map_err(|_| AllocError)?; + + let (old_ptr, len, cap) = destructure(self); + + // We need to make sure that `ptr` is either NULL or comes from a previous call to + // `krealloc_aligned`. A `Vec<T>`'s `ptr` value is not guaranteed to be NULL and might be + // dangling after being created with `Vec::new`. Instead, we can rely on `Vec<T>`'s capacity + // to be zero if no memory has been allocated yet. + let ptr = if cap == 0 { + core::ptr::null_mut() + } else { + old_ptr + }; + + // SAFETY: `ptr` is valid because it's either NULL or comes from a previous call to + // `krealloc_aligned`. We also verified that the type is not a ZST. + let new_ptr = unsafe { super::allocator::krealloc_aligned(ptr.cast(), layout, flags) }; + if new_ptr.is_null() { + // SAFETY: We are just rebuilding the existing `Vec` with no changes. + unsafe { rebuild(self, old_ptr, len, cap) }; + Err(AllocError) + } else { + // SAFETY: `ptr` has been reallocated with the layout for `new_cap` elements. New cap + // is greater than `cap`, so it continues to be >= `len`. + unsafe { rebuild(self, new_ptr.cast::<T>(), len, new_cap) }; + Ok(()) + } + } +} + +#[cfg(not(any(test, testlib)))] +fn destructure<T>(v: &mut Vec<T>) -> (*mut T, usize, usize) { + let mut tmp = Vec::new(); + core::mem::swap(&mut tmp, v); + let mut tmp = core::mem::ManuallyDrop::new(tmp); + let len = tmp.len(); + let cap = tmp.capacity(); + (tmp.as_mut_ptr(), len, cap) +} + +/// Rebuilds a `Vec` from a pointer, length, and capacity. +/// +/// # Safety +/// +/// The same as [`Vec::from_raw_parts`]. +#[cfg(not(any(test, testlib)))] +unsafe fn rebuild<T>(v: &mut Vec<T>, ptr: *mut T, len: usize, cap: usize) { + // SAFETY: The safety requirements from this function satisfy those of `from_raw_parts`. + let mut tmp = unsafe { Vec::from_raw_parts(ptr, len, cap) }; + core::mem::swap(&mut tmp, v); +} diff --git a/rust/kernel/allocator.rs b/rust/kernel/allocator.rs deleted file mode 100644 index 9363b527be66..000000000000 --- a/rust/kernel/allocator.rs +++ /dev/null @@ -1,108 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -//! Allocator support. - -use core::alloc::{GlobalAlloc, Layout}; -use core::ptr; - -use crate::bindings; - -struct KernelAllocator; - -/// Calls `krealloc` with a proper size to alloc a new object aligned to `new_layout`'s alignment. -/// -/// # Safety -/// -/// - `ptr` can be either null or a pointer which has been allocated by this allocator. -/// - `new_layout` must have a non-zero size. -unsafe fn krealloc_aligned(ptr: *mut u8, new_layout: Layout, flags: bindings::gfp_t) -> *mut u8 { - // Customized layouts from `Layout::from_size_align()` can have size < align, so pad first. - let layout = new_layout.pad_to_align(); - - let mut size = layout.size(); - - if layout.align() > bindings::BINDINGS_ARCH_SLAB_MINALIGN { - // The alignment requirement exceeds the slab guarantee, thus try to enlarge the size - // to use the "power-of-two" size/alignment guarantee (see comments in `kmalloc()` for - // more information). - // - // Note that `layout.size()` (after padding) is guaranteed to be a multiple of - // `layout.align()`, so `next_power_of_two` gives enough alignment guarantee. - size = size.next_power_of_two(); - } - - // SAFETY: - // - `ptr` is either null or a pointer returned from a previous `k{re}alloc()` by the - // function safety requirement. - // - `size` is greater than 0 since it's either a `layout.size()` (which cannot be zero - // according to the function safety requirement) or a result from `next_power_of_two()`. - unsafe { bindings::krealloc(ptr as *const core::ffi::c_void, size, flags) as *mut u8 } -} - -unsafe impl GlobalAlloc for KernelAllocator { - unsafe fn alloc(&self, layout: Layout) -> *mut u8 { - // `krealloc()` is used instead of `kmalloc()` because the latter is - // an inline function and cannot be bound to as a result. - unsafe { bindings::krealloc(ptr::null(), layout.size(), bindings::GFP_KERNEL) as *mut u8 } - } - - unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) { - unsafe { - bindings::kfree(ptr as *const core::ffi::c_void); - } - } -} - -#[global_allocator] -static ALLOCATOR: KernelAllocator = KernelAllocator; - -// `rustc` only generates these for some crate types. Even then, we would need -// to extract the object file that has them from the archive. For the moment, -// let's generate them ourselves instead. -// -// Note: Although these are *safe* functions, they are called by the compiler -// with parameters that obey the same `GlobalAlloc` function safety -// requirements: size and align should form a valid layout, and size is -// greater than 0. -// -// Note that `#[no_mangle]` implies exported too, nowadays. -#[no_mangle] -fn __rust_alloc(size: usize, align: usize) -> *mut u8 { - // SAFETY: See assumption above. - let layout = unsafe { Layout::from_size_align_unchecked(size, align) }; - - // SAFETY: `ptr::null_mut()` is null, per assumption above the size of `layout` is greater - // than 0. - unsafe { krealloc_aligned(ptr::null_mut(), layout, bindings::GFP_KERNEL) } -} - -#[no_mangle] -fn __rust_dealloc(ptr: *mut u8, _size: usize, _align: usize) { - unsafe { bindings::kfree(ptr as *const core::ffi::c_void) }; -} - -#[no_mangle] -fn __rust_realloc(ptr: *mut u8, _old_size: usize, align: usize, new_size: usize) -> *mut u8 { - // SAFETY: See assumption above. - let new_layout = unsafe { Layout::from_size_align_unchecked(new_size, align) }; - - // SAFETY: Per assumption above, `ptr` is allocated by `__rust_*` before, and the size of - // `new_layout` is greater than 0. - unsafe { krealloc_aligned(ptr, new_layout, bindings::GFP_KERNEL) } -} - -#[no_mangle] -fn __rust_alloc_zeroed(size: usize, align: usize) -> *mut u8 { - // SAFETY: See assumption above. - let layout = unsafe { Layout::from_size_align_unchecked(size, align) }; - - // SAFETY: `ptr::null_mut()` is null, per assumption above the size of `layout` is greater - // than 0. - unsafe { - krealloc_aligned( - ptr::null_mut(), - layout, - bindings::GFP_KERNEL | bindings::__GFP_ZERO, - ) - } -} diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index 05fcab6abfe6..55280ae9fe40 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -2,16 +2,12 @@ //! Kernel errors. //! -//! C header: [`include/uapi/asm-generic/errno-base.h`](../../../include/uapi/asm-generic/errno-base.h) +//! C header: [`include/uapi/asm-generic/errno-base.h`](srctree/include/uapi/asm-generic/errno-base.h) -use crate::str::CStr; +use crate::{alloc::AllocError, str::CStr}; -use alloc::{ - alloc::{AllocError, LayoutError}, - collections::TryReserveError, -}; +use alloc::alloc::LayoutError; -use core::convert::From; use core::fmt; use core::num::TryFromIntError; use core::str::Utf8Error; @@ -37,7 +33,7 @@ pub mod code { declare_err!(E2BIG, "Argument list too long."); declare_err!(ENOEXEC, "Exec format error."); declare_err!(EBADF, "Bad file number."); - declare_err!(ECHILD, "Exec format error."); + declare_err!(ECHILD, "No child processes."); declare_err!(EAGAIN, "Try again."); declare_err!(ENOMEM, "Out of memory."); declare_err!(EACCES, "Permission denied."); @@ -133,7 +129,7 @@ impl Error { /// Returns the error encoded as a pointer. #[allow(dead_code)] pub(crate) fn to_ptr<T>(self) -> *mut T { - // SAFETY: self.0 is a valid error due to its invariant. + // SAFETY: `self.0` is a valid error due to its invariant. unsafe { bindings::ERR_PTR(self.0.into()) as *mut _ } } @@ -192,12 +188,6 @@ impl From<Utf8Error> for Error { } } -impl From<TryReserveError> for Error { - fn from(_: TryReserveError) -> Error { - code::ENOMEM - } -} - impl From<LayoutError> for Error { fn from(_: LayoutError) -> Error { code::ENOMEM @@ -264,13 +254,9 @@ pub fn to_result(err: core::ffi::c_int) -> Result { /// pdev: &mut PlatformDevice, /// index: u32, /// ) -> Result<*mut core::ffi::c_void> { -/// // SAFETY: FFI call. -/// unsafe { -/// from_err_ptr(bindings::devm_platform_ioremap_resource( -/// pdev.to_ptr(), -/// index, -/// )) -/// } +/// // SAFETY: `pdev` points to a valid platform device. There are no safety requirements +/// // on `index`. +/// from_err_ptr(unsafe { bindings::devm_platform_ioremap_resource(pdev.to_ptr(), index) }) /// } /// ``` // TODO: Remove `dead_code` marker once an in-kernel client is available. @@ -335,3 +321,7 @@ where Err(e) => T::from(e.to_errno() as i16), } } + +/// Error message for calling a default function of a [`#[vtable]`](macros::vtable) trait. +pub const VTABLE_DEFAULT_ERROR: &str = + "This function must not be called, see the #[vtable] documentation."; diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index b4332a4ec1f4..68605b633e73 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -35,8 +35,8 @@ //! that you need to write `<-` instead of `:` for fields that you want to initialize in-place. //! //! ```rust -//! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] -//! use kernel::{prelude::*, sync::Mutex, new_mutex}; +//! # #![allow(clippy::disallowed_names)] +//! use kernel::sync::{new_mutex, Mutex}; //! # use core::pin::Pin; //! #[pin_data] //! struct Foo { @@ -55,8 +55,8 @@ //! (or just the stack) to actually initialize a `Foo`: //! //! ```rust -//! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] -//! # use kernel::{prelude::*, sync::Mutex, new_mutex}; +//! # #![allow(clippy::disallowed_names)] +//! # use kernel::sync::{new_mutex, Mutex}; //! # use core::pin::Pin; //! # #[pin_data] //! # struct Foo { @@ -68,7 +68,7 @@ //! # a <- new_mutex!(42, "Foo::a"), //! # b: 24, //! # }); -//! let foo: Result<Pin<Box<Foo>>> = Box::pin_init(foo); +//! let foo: Result<Pin<Box<Foo>>> = Box::pin_init(foo, GFP_KERNEL); //! ``` //! //! For more information see the [`pin_init!`] macro. @@ -79,15 +79,16 @@ //! above method only works for types where you can access the fields. //! //! ```rust -//! # use kernel::{new_mutex, sync::{Arc, Mutex}}; -//! let mtx: Result<Arc<Mutex<usize>>> = Arc::pin_init(new_mutex!(42, "example::mtx")); +//! # use kernel::sync::{new_mutex, Arc, Mutex}; +//! let mtx: Result<Arc<Mutex<usize>>> = +//! Arc::pin_init(new_mutex!(42, "example::mtx"), GFP_KERNEL); //! ``` //! //! To declare an init macro/function you just return an [`impl PinInit<T, E>`]: //! //! ```rust -//! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] -//! # use kernel::{sync::Mutex, prelude::*, new_mutex, init::PinInit, try_pin_init}; +//! # #![allow(clippy::disallowed_names)] +//! # use kernel::{sync::Mutex, new_mutex, init::PinInit, try_pin_init}; //! #[pin_data] //! struct DriverData { //! #[pin] @@ -99,7 +100,7 @@ //! fn new() -> impl PinInit<Self, Error> { //! try_pin_init!(Self { //! status <- new_mutex!(0, "DriverData::status"), -//! buffer: Box::init(kernel::init::zeroed())?, +//! buffer: Box::init(kernel::init::zeroed(), GFP_KERNEL)?, //! }) //! } //! } @@ -120,14 +121,24 @@ //! `slot` gets called. //! //! ```rust -//! use kernel::{prelude::*, init}; +//! # #![allow(unreachable_pub, clippy::disallowed_names)] +//! use kernel::{init, types::Opaque}; //! use core::{ptr::addr_of_mut, marker::PhantomPinned, pin::Pin}; //! # mod bindings { +//! # #![allow(non_camel_case_types)] //! # pub struct foo; //! # pub unsafe fn init_foo(_ptr: *mut foo) {} //! # pub unsafe fn destroy_foo(_ptr: *mut foo) {} //! # pub unsafe fn enable_foo(_ptr: *mut foo, _flags: u32) -> i32 { 0 } //! # } +//! # // `Error::from_errno` is `pub(crate)` in the `kernel` crate, thus provide a workaround. +//! # trait FromErrno { +//! # fn from_errno(errno: core::ffi::c_int) -> Error { +//! # // Dummy error that can be constructed outside the `kernel` crate. +//! # Error::from(core::fmt::Error) +//! # } +//! # } +//! # impl FromErrno for Error {} //! /// # Invariants //! /// //! /// `foo` is always initialized @@ -158,7 +169,7 @@ //! if err != 0 { //! // Enabling has failed, first clean up the foo and then return the error. //! bindings::destroy_foo(Opaque::raw_get(foo)); -//! return Err(Error::from_kernel_errno(err)); +//! return Err(Error::from_errno(err)); //! } //! //! // All fields of `RawFoo` have been initialized, since `_p` is a ZST. @@ -200,13 +211,14 @@ //! [`pin_init!`]: crate::pin_init! use crate::{ + alloc::{box_ext::BoxExt, AllocError, Flags}, error::{self, Error}, sync::UniqueArc, + types::{Opaque, ScopeGuard}, }; use alloc::boxed::Box; use core::{ - alloc::AllocError, - cell::Cell, + cell::UnsafeCell, convert::Infallible, marker::PhantomData, mem::MaybeUninit, @@ -225,9 +237,8 @@ pub mod macros; /// # Examples /// /// ```rust -/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] -/// # use kernel::{init, pin_init, stack_pin_init, init::*, sync::Mutex, new_mutex}; -/// # use macros::pin_data; +/// # #![allow(clippy::disallowed_names)] +/// # use kernel::{init, macros::pin_data, pin_init, stack_pin_init, init::*, sync::Mutex, new_mutex}; /// # use core::pin::Pin; /// #[pin_data] /// struct Foo { @@ -277,8 +288,8 @@ macro_rules! stack_pin_init { /// /// # Examples /// -/// ```rust -/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] +/// ```rust,ignore +/// # #![allow(clippy::disallowed_names)] /// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex}; /// # use macros::pin_data; /// # use core::{alloc::AllocError, pin::Pin}; @@ -295,16 +306,16 @@ macro_rules! stack_pin_init { /// /// stack_try_pin_init!(let foo: Result<Pin<&mut Foo>, AllocError> = pin_init!(Foo { /// a <- new_mutex!(42), -/// b: Box::try_new(Bar { +/// b: Box::new(Bar { /// x: 64, -/// })?, +/// }, GFP_KERNEL)?, /// })); /// let foo = foo.unwrap(); /// pr_info!("a: {}", &*foo.a.lock()); /// ``` /// -/// ```rust -/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] +/// ```rust,ignore +/// # #![allow(clippy::disallowed_names)] /// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex}; /// # use macros::pin_data; /// # use core::{alloc::AllocError, pin::Pin}; @@ -321,9 +332,9 @@ macro_rules! stack_pin_init { /// /// stack_try_pin_init!(let foo: Pin<&mut Foo> =? pin_init!(Foo { /// a <- new_mutex!(42), -/// b: Box::try_new(Bar { +/// b: Box::new(Bar { /// x: 64, -/// })?, +/// }, GFP_KERNEL)?, /// })); /// pr_info!("a: {}", &*foo.a.lock()); /// # Ok::<_, AllocError>(()) @@ -356,7 +367,7 @@ macro_rules! stack_try_pin_init { /// The syntax is almost identical to that of a normal `struct` initializer: /// /// ```rust -/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] +/// # #![allow(clippy::disallowed_names)] /// # use kernel::{init, pin_init, macros::pin_data, init::*}; /// # use core::pin::Pin; /// #[pin_data] @@ -380,7 +391,7 @@ macro_rules! stack_try_pin_init { /// }, /// }); /// # initializer } -/// # Box::pin_init(demo()).unwrap(); +/// # Box::pin_init(demo(), GFP_KERNEL).unwrap(); /// ``` /// /// Arbitrary Rust expressions can be used to set the value of a variable. @@ -401,8 +412,8 @@ macro_rules! stack_try_pin_init { /// To create an initializer function, simply declare it like this: /// /// ```rust -/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] -/// # use kernel::{init, pin_init, prelude::*, init::*}; +/// # #![allow(clippy::disallowed_names)] +/// # use kernel::{init, pin_init, init::*}; /// # use core::pin::Pin; /// # #[pin_data] /// # struct Foo { @@ -428,7 +439,7 @@ macro_rules! stack_try_pin_init { /// Users of `Foo` can now create it like this: /// /// ```rust -/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] +/// # #![allow(clippy::disallowed_names)] /// # use kernel::{init, pin_init, macros::pin_data, init::*}; /// # use core::pin::Pin; /// # #[pin_data] @@ -450,13 +461,13 @@ macro_rules! stack_try_pin_init { /// # }) /// # } /// # } -/// let foo = Box::pin_init(Foo::new()); +/// let foo = Box::pin_init(Foo::new(), GFP_KERNEL); /// ``` /// /// They can also easily embed it into their own `struct`s: /// /// ```rust -/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] +/// # #![allow(clippy::disallowed_names)] /// # use kernel::{init, pin_init, macros::pin_data, init::*}; /// # use core::pin::Pin; /// # #[pin_data] @@ -509,14 +520,17 @@ macro_rules! stack_try_pin_init { /// - Fields that you want to initialize in-place have to use `<-` instead of `:`. /// - In front of the initializer you can write `&this in` to have access to a [`NonNull<Self>`] /// pointer named `this` inside of the initializer. +/// - Using struct update syntax one can place `..Zeroable::zeroed()` at the very end of the +/// struct, this initializes every field with 0 and then runs all initializers specified in the +/// body. This can only be done if [`Zeroable`] is implemented for the struct. /// /// For instance: /// /// ```rust -/// # use kernel::pin_init; -/// # use macros::pin_data; +/// # use kernel::{macros::{Zeroable, pin_data}, pin_init}; /// # use core::{ptr::addr_of_mut, marker::PhantomPinned}; /// #[pin_data] +/// #[derive(Zeroable)] /// struct Buf { /// // `ptr` points into `buf`. /// ptr: *mut u8, @@ -529,6 +543,10 @@ macro_rules! stack_try_pin_init { /// ptr: unsafe { addr_of_mut!((*this.as_ptr()).buf).cast() }, /// pin: PhantomPinned, /// }); +/// pin_init!(Buf { +/// buf: [1; 64], +/// ..Zeroable::zeroed() +/// }); /// ``` /// /// [`try_pin_init!`]: kernel::try_pin_init @@ -540,11 +558,15 @@ macro_rules! pin_init { ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }) => { - $crate::try_pin_init!( + $crate::__init_internal!( @this($($this)?), @typ($t $(::<$($generics),*>)?), @fields($($fields)*), @error(::core::convert::Infallible), + @data(PinData, use_data), + @has_data(HasPinData, __pin_data), + @construct_closure(pin_init_from_closure), + @munch_fields($($fields)*), ) }; } @@ -579,7 +601,7 @@ macro_rules! pin_init { /// impl BigBuf { /// fn new() -> impl PinInit<Self, Error> { /// try_pin_init!(Self { -/// big: Box::init(init::zeroed())?, +/// big: Box::init(init::zeroed(), GFP_KERNEL)?, /// small: [0; 1024 * 1024], /// ptr: core::ptr::null_mut(), /// }? Error) @@ -593,205 +615,31 @@ macro_rules! try_pin_init { ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }) => { - $crate::try_pin_init!( + $crate::__init_internal!( @this($($this)?), @typ($t $(::<$($generics),*>)? ), @fields($($fields)*), @error($crate::error::Error), + @data(PinData, use_data), + @has_data(HasPinData, __pin_data), + @construct_closure(pin_init_from_closure), + @munch_fields($($fields)*), ) }; ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }? $err:ty) => { - $crate::try_pin_init!( + $crate::__init_internal!( @this($($this)?), @typ($t $(::<$($generics),*>)? ), @fields($($fields)*), @error($err), + @data(PinData, use_data), + @has_data(HasPinData, __pin_data), + @construct_closure(pin_init_from_closure), + @munch_fields($($fields)*), ) }; - ( - @this($($this:ident)?), - @typ($t:ident $(::<$($generics:ty),*>)?), - @fields($($fields:tt)*), - @error($err:ty), - ) => {{ - // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return - // type and shadow it later when we insert the arbitrary user code. That way there will be - // no possibility of returning without `unsafe`. - struct __InitOk; - // Get the pin data from the supplied type. - let data = unsafe { - use $crate::init::__internal::HasPinData; - $t$(::<$($generics),*>)?::__pin_data() - }; - // Ensure that `data` really is of type `PinData` and help with type inference: - let init = $crate::init::__internal::PinData::make_closure::<_, __InitOk, $err>( - data, - move |slot| { - { - // Shadow the structure so it cannot be used to return early. - struct __InitOk; - // Create the `this` so it can be referenced by the user inside of the - // expressions creating the individual fields. - $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)? - // Initialize every field. - $crate::try_pin_init!(init_slot: - @data(data), - @slot(slot), - @munch_fields($($fields)*,), - ); - // We use unreachable code to ensure that all fields have been mentioned exactly - // once, this struct initializer will still be type-checked and complain with a - // very natural error message if a field is forgotten/mentioned more than once. - #[allow(unreachable_code, clippy::diverging_sub_expression)] - if false { - $crate::try_pin_init!(make_initializer: - @slot(slot), - @type_name($t), - @munch_fields($($fields)*,), - @acc(), - ); - } - // Forget all guards, since initialization was a success. - $crate::try_pin_init!(forget_guards: - @munch_fields($($fields)*,), - ); - } - Ok(__InitOk) - } - ); - let init = move |slot| -> ::core::result::Result<(), $err> { - init(slot).map(|__InitOk| ()) - }; - let init = unsafe { $crate::init::pin_init_from_closure::<_, $err>(init) }; - init - }}; - (init_slot: - @data($data:ident), - @slot($slot:ident), - @munch_fields($(,)?), - ) => { - // Endpoint of munching, no fields are left. - }; - (init_slot: - @data($data:ident), - @slot($slot:ident), - // In-place initialization syntax. - @munch_fields($field:ident <- $val:expr, $($rest:tt)*), - ) => { - let $field = $val; - // Call the initializer. - // - // SAFETY: `slot` is valid, because we are inside of an initializer closure, we - // return when an error/panic occurs. - // We also use the `data` to require the correct trait (`Init` or `PinInit`) for `$field`. - unsafe { $data.$field(::core::ptr::addr_of_mut!((*$slot).$field), $field)? }; - // Create the drop guard. - // - // We only give access to `&DropGuard`, so it cannot be forgotten via safe code. - // - // SAFETY: We forget the guard later when initialization has succeeded. - let $field = &unsafe { - $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) - }; - - $crate::try_pin_init!(init_slot: - @data($data), - @slot($slot), - @munch_fields($($rest)*), - ); - }; - (init_slot: - @data($data:ident), - @slot($slot:ident), - // Direct value init, this is safe for every field. - @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), - ) => { - $(let $field = $val;)? - // Initialize the field. - // - // SAFETY: The memory at `slot` is uninitialized. - unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) }; - // Create the drop guard: - // - // We only give access to `&DropGuard`, so it cannot be accidentally forgotten. - // - // SAFETY: We forget the guard later when initialization has succeeded. - let $field = &unsafe { - $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) - }; - - $crate::try_pin_init!(init_slot: - @data($data), - @slot($slot), - @munch_fields($($rest)*), - ); - }; - (make_initializer: - @slot($slot:ident), - @type_name($t:ident), - @munch_fields($(,)?), - @acc($($acc:tt)*), - ) => { - // Endpoint, nothing more to munch, create the initializer. - // Since we are in the `if false` branch, this will never get executed. We abuse `slot` to - // get the correct type inference here: - unsafe { - ::core::ptr::write($slot, $t { - $($acc)* - }); - } - }; - (make_initializer: - @slot($slot:ident), - @type_name($t:ident), - @munch_fields($field:ident <- $val:expr, $($rest:tt)*), - @acc($($acc:tt)*), - ) => { - $crate::try_pin_init!(make_initializer: - @slot($slot), - @type_name($t), - @munch_fields($($rest)*), - @acc($($acc)* $field: ::core::panic!(),), - ); - }; - (make_initializer: - @slot($slot:ident), - @type_name($t:ident), - @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), - @acc($($acc:tt)*), - ) => { - $crate::try_pin_init!(make_initializer: - @slot($slot), - @type_name($t), - @munch_fields($($rest)*), - @acc($($acc)* $field: ::core::panic!(),), - ); - }; - (forget_guards: - @munch_fields($(,)?), - ) => { - // Munching finished. - }; - (forget_guards: - @munch_fields($field:ident <- $val:expr, $($rest:tt)*), - ) => { - unsafe { $crate::init::__internal::DropGuard::forget($field) }; - - $crate::try_pin_init!(forget_guards: - @munch_fields($($rest)*), - ); - }; - (forget_guards: - @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), - ) => { - unsafe { $crate::init::__internal::DropGuard::forget($field) }; - - $crate::try_pin_init!(forget_guards: - @munch_fields($($rest)*), - ); - }; } /// Construct an in-place initializer for `struct`s. @@ -816,11 +664,15 @@ macro_rules! init { ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }) => { - $crate::try_init!( + $crate::__init_internal!( @this($($this)?), @typ($t $(::<$($generics),*>)?), @fields($($fields)*), @error(::core::convert::Infallible), + @data(InitData, /*no use_data*/), + @has_data(HasInitData, __init_data), + @construct_closure(init_from_closure), + @munch_fields($($fields)*), ) } } @@ -841,7 +693,7 @@ macro_rules! init { /// # Examples /// /// ```rust -/// use kernel::{init::PinInit, error::Error, InPlaceInit}; +/// use kernel::{init::{PinInit, zeroed}, error::Error}; /// struct BigBuf { /// big: Box<[u8; 1024 * 1024 * 1024]>, /// small: [u8; 1024 * 1024], @@ -850,7 +702,7 @@ macro_rules! init { /// impl BigBuf { /// fn new() -> impl Init<Self, Error> { /// try_init!(Self { -/// big: Box::init(zeroed())?, +/// big: Box::init(zeroed(), GFP_KERNEL)?, /// small: [0; 1024 * 1024], /// }? Error) /// } @@ -863,199 +715,31 @@ macro_rules! try_init { ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }) => { - $crate::try_init!( + $crate::__init_internal!( @this($($this)?), @typ($t $(::<$($generics),*>)?), @fields($($fields)*), @error($crate::error::Error), + @data(InitData, /*no use_data*/), + @has_data(HasInitData, __init_data), + @construct_closure(init_from_closure), + @munch_fields($($fields)*), ) }; ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }? $err:ty) => { - $crate::try_init!( + $crate::__init_internal!( @this($($this)?), @typ($t $(::<$($generics),*>)?), @fields($($fields)*), @error($err), + @data(InitData, /*no use_data*/), + @has_data(HasInitData, __init_data), + @construct_closure(init_from_closure), + @munch_fields($($fields)*), ) }; - ( - @this($($this:ident)?), - @typ($t:ident $(::<$($generics:ty),*>)?), - @fields($($fields:tt)*), - @error($err:ty), - ) => {{ - // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return - // type and shadow it later when we insert the arbitrary user code. That way there will be - // no possibility of returning without `unsafe`. - struct __InitOk; - // Get the init data from the supplied type. - let data = unsafe { - use $crate::init::__internal::HasInitData; - $t$(::<$($generics),*>)?::__init_data() - }; - // Ensure that `data` really is of type `InitData` and help with type inference: - let init = $crate::init::__internal::InitData::make_closure::<_, __InitOk, $err>( - data, - move |slot| { - { - // Shadow the structure so it cannot be used to return early. - struct __InitOk; - // Create the `this` so it can be referenced by the user inside of the - // expressions creating the individual fields. - $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)? - // Initialize every field. - $crate::try_init!(init_slot: - @slot(slot), - @munch_fields($($fields)*,), - ); - // We use unreachable code to ensure that all fields have been mentioned exactly - // once, this struct initializer will still be type-checked and complain with a - // very natural error message if a field is forgotten/mentioned more than once. - #[allow(unreachable_code, clippy::diverging_sub_expression)] - if false { - $crate::try_init!(make_initializer: - @slot(slot), - @type_name($t), - @munch_fields($($fields)*,), - @acc(), - ); - } - // Forget all guards, since initialization was a success. - $crate::try_init!(forget_guards: - @munch_fields($($fields)*,), - ); - } - Ok(__InitOk) - } - ); - let init = move |slot| -> ::core::result::Result<(), $err> { - init(slot).map(|__InitOk| ()) - }; - let init = unsafe { $crate::init::init_from_closure::<_, $err>(init) }; - init - }}; - (init_slot: - @slot($slot:ident), - @munch_fields( $(,)?), - ) => { - // Endpoint of munching, no fields are left. - }; - (init_slot: - @slot($slot:ident), - @munch_fields($field:ident <- $val:expr, $($rest:tt)*), - ) => { - let $field = $val; - // Call the initializer. - // - // SAFETY: `slot` is valid, because we are inside of an initializer closure, we - // return when an error/panic occurs. - unsafe { - $crate::init::Init::__init($field, ::core::ptr::addr_of_mut!((*$slot).$field))?; - } - // Create the drop guard. - // - // We only give access to `&DropGuard`, so it cannot be accidentally forgotten. - // - // SAFETY: We forget the guard later when initialization has succeeded. - let $field = &unsafe { - $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) - }; - - $crate::try_init!(init_slot: - @slot($slot), - @munch_fields($($rest)*), - ); - }; - (init_slot: - @slot($slot:ident), - // Direct value init. - @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), - ) => { - $(let $field = $val;)? - // Call the initializer. - // - // SAFETY: The memory at `slot` is uninitialized. - unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) }; - // Create the drop guard. - // - // We only give access to `&DropGuard`, so it cannot be accidentally forgotten. - // - // SAFETY: We forget the guard later when initialization has succeeded. - let $field = &unsafe { - $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) - }; - - $crate::try_init!(init_slot: - @slot($slot), - @munch_fields($($rest)*), - ); - }; - (make_initializer: - @slot($slot:ident), - @type_name($t:ident), - @munch_fields( $(,)?), - @acc($($acc:tt)*), - ) => { - // Endpoint, nothing more to munch, create the initializer. - // Since we are in the `if false` branch, this will never get executed. We abuse `slot` to - // get the correct type inference here: - unsafe { - ::core::ptr::write($slot, $t { - $($acc)* - }); - } - }; - (make_initializer: - @slot($slot:ident), - @type_name($t:ident), - @munch_fields($field:ident <- $val:expr, $($rest:tt)*), - @acc($($acc:tt)*), - ) => { - $crate::try_init!(make_initializer: - @slot($slot), - @type_name($t), - @munch_fields($($rest)*), - @acc($($acc)*$field: ::core::panic!(),), - ); - }; - (make_initializer: - @slot($slot:ident), - @type_name($t:ident), - @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), - @acc($($acc:tt)*), - ) => { - $crate::try_init!(make_initializer: - @slot($slot), - @type_name($t), - @munch_fields($($rest)*), - @acc($($acc)*$field: ::core::panic!(),), - ); - }; - (forget_guards: - @munch_fields($(,)?), - ) => { - // Munching finished. - }; - (forget_guards: - @munch_fields($field:ident <- $val:expr, $($rest:tt)*), - ) => { - unsafe { $crate::init::__internal::DropGuard::forget($field) }; - - $crate::try_init!(forget_guards: - @munch_fields($($rest)*), - ); - }; - (forget_guards: - @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), - ) => { - unsafe { $crate::init::__internal::DropGuard::forget($field) }; - - $crate::try_init!(forget_guards: - @munch_fields($($rest)*), - ); - }; } /// A pin-initializer for the type `T`. @@ -1068,10 +752,10 @@ macro_rules! try_init { /// /// # Safety /// -/// When implementing this type you will need to take great care. Also there are probably very few +/// When implementing this trait you will need to take great care. Also there are probably very few /// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible. /// -/// The [`PinInit::__pinned_init`] function +/// The [`PinInit::__pinned_init`] function: /// - returns `Ok(())` if it initialized every field of `slot`, /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: /// - `slot` can be deallocated without UB occurring, @@ -1092,6 +776,79 @@ pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized { /// deallocate. /// - `slot` will not move until it is dropped, i.e. it will be pinned. unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E>; + + /// First initializes the value using `self` then calls the function `f` with the initialized + /// value. + /// + /// If `f` returns an error the value is dropped and the initializer will forward the error. + /// + /// # Examples + /// + /// ```rust + /// # #![allow(clippy::disallowed_names)] + /// use kernel::{types::Opaque, init::pin_init_from_closure}; + /// #[repr(C)] + /// struct RawFoo([u8; 16]); + /// extern { + /// fn init_foo(_: *mut RawFoo); + /// } + /// + /// #[pin_data] + /// struct Foo { + /// #[pin] + /// raw: Opaque<RawFoo>, + /// } + /// + /// impl Foo { + /// fn setup(self: Pin<&mut Self>) { + /// pr_info!("Setting up foo"); + /// } + /// } + /// + /// let foo = pin_init!(Foo { + /// raw <- unsafe { + /// Opaque::ffi_init(|s| { + /// init_foo(s); + /// }) + /// }, + /// }).pin_chain(|foo| { + /// foo.setup(); + /// Ok(()) + /// }); + /// ``` + fn pin_chain<F>(self, f: F) -> ChainPinInit<Self, F, T, E> + where + F: FnOnce(Pin<&mut T>) -> Result<(), E>, + { + ChainPinInit(self, f, PhantomData) + } +} + +/// An initializer returned by [`PinInit::pin_chain`]. +pub struct ChainPinInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, Box<T>)>); + +// SAFETY: The `__pinned_init` function is implemented such that it +// - returns `Ok(())` on successful initialization, +// - returns `Err(err)` on error and in this case `slot` will be dropped. +// - considers `slot` pinned. +unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainPinInit<I, F, T, E> +where + I: PinInit<T, E>, + F: FnOnce(Pin<&mut T>) -> Result<(), E>, +{ + unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: All requirements fulfilled since this function is `__pinned_init`. + unsafe { self.0.__pinned_init(slot)? }; + // SAFETY: The above call initialized `slot` and we still have unique access. + let val = unsafe { &mut *slot }; + // SAFETY: `slot` is considered pinned. + let val = unsafe { Pin::new_unchecked(val) }; + (self.1)(val).map_err(|e| { + // SAFETY: `slot` was initialized above. + unsafe { core::ptr::drop_in_place(slot) }; + e + }) + } } /// An initializer for `T`. @@ -1105,10 +862,10 @@ pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized { /// /// # Safety /// -/// When implementing this type you will need to take great care. Also there are probably very few +/// When implementing this trait you will need to take great care. Also there are probably very few /// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible. /// -/// The [`Init::__init`] function +/// The [`Init::__init`] function: /// - returns `Ok(())` if it initialized every field of `slot`, /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: /// - `slot` can be deallocated without UB occurring, @@ -1124,7 +881,7 @@ pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized { /// /// [`Arc<T>`]: crate::sync::Arc #[must_use = "An initializer must be used in order to create its value."] -pub unsafe trait Init<T: ?Sized, E = Infallible>: Sized { +pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> { /// Initializes `slot`. /// /// # Safety @@ -1133,16 +890,73 @@ pub unsafe trait Init<T: ?Sized, E = Infallible>: Sized { /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to /// deallocate. unsafe fn __init(self, slot: *mut T) -> Result<(), E>; + + /// First initializes the value using `self` then calls the function `f` with the initialized + /// value. + /// + /// If `f` returns an error the value is dropped and the initializer will forward the error. + /// + /// # Examples + /// + /// ```rust + /// # #![allow(clippy::disallowed_names)] + /// use kernel::{types::Opaque, init::{self, init_from_closure}}; + /// struct Foo { + /// buf: [u8; 1_000_000], + /// } + /// + /// impl Foo { + /// fn setup(&mut self) { + /// pr_info!("Setting up foo"); + /// } + /// } + /// + /// let foo = init!(Foo { + /// buf <- init::zeroed() + /// }).chain(|foo| { + /// foo.setup(); + /// Ok(()) + /// }); + /// ``` + fn chain<F>(self, f: F) -> ChainInit<Self, F, T, E> + where + F: FnOnce(&mut T) -> Result<(), E>, + { + ChainInit(self, f, PhantomData) + } +} + +/// An initializer returned by [`Init::chain`]. +pub struct ChainInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, Box<T>)>); + +// SAFETY: The `__init` function is implemented such that it +// - returns `Ok(())` on successful initialization, +// - returns `Err(err)` on error and in this case `slot` will be dropped. +unsafe impl<T: ?Sized, E, I, F> Init<T, E> for ChainInit<I, F, T, E> +where + I: Init<T, E>, + F: FnOnce(&mut T) -> Result<(), E>, +{ + unsafe fn __init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: All requirements fulfilled since this function is `__init`. + unsafe { self.0.__pinned_init(slot)? }; + // SAFETY: The above call initialized `slot` and we still have unique access. + (self.1)(unsafe { &mut *slot }).map_err(|e| { + // SAFETY: `slot` was initialized above. + unsafe { core::ptr::drop_in_place(slot) }; + e + }) + } } -// SAFETY: Every in-place initializer can also be used as a pin-initializer. -unsafe impl<T: ?Sized, E, I> PinInit<T, E> for I +// SAFETY: `__pinned_init` behaves exactly the same as `__init`. +unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainInit<I, F, T, E> where I: Init<T, E>, + F: FnOnce(&mut T) -> Result<(), E>, { unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { - // SAFETY: `__init` meets the same requirements as `__pinned_init`, except that it does not - // require `slot` to not move after init. + // SAFETY: `__init` has less strict requirements compared to `__pinned_init`. unsafe { self.__init(slot) } } } @@ -1194,6 +1008,93 @@ pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> { unsafe { init_from_closure(|_| Ok(())) } } +/// Initializes an array by initializing each element via the provided initializer. +/// +/// # Examples +/// +/// ```rust +/// use kernel::{error::Error, init::init_array_from_fn}; +/// let array: Box<[usize; 1_000]> = Box::init::<Error>(init_array_from_fn(|i| i), GFP_KERNEL).unwrap(); +/// assert_eq!(array.len(), 1_000); +/// ``` +pub fn init_array_from_fn<I, const N: usize, T, E>( + mut make_init: impl FnMut(usize) -> I, +) -> impl Init<[T; N], E> +where + I: Init<T, E>, +{ + let init = move |slot: *mut [T; N]| { + let slot = slot.cast::<T>(); + // Counts the number of initialized elements and when dropped drops that many elements from + // `slot`. + let mut init_count = ScopeGuard::new_with_data(0, |i| { + // We now free every element that has been initialized before. + // SAFETY: The loop initialized exactly the values from 0..i and since we + // return `Err` below, the caller will consider the memory at `slot` as + // uninitialized. + unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) }; + }); + for i in 0..N { + let init = make_init(i); + // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`. + let ptr = unsafe { slot.add(i) }; + // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init` + // requirements. + unsafe { init.__init(ptr) }?; + *init_count += 1; + } + init_count.dismiss(); + Ok(()) + }; + // SAFETY: The initializer above initializes every element of the array. On failure it drops + // any initialized elements and returns `Err`. + unsafe { init_from_closure(init) } +} + +/// Initializes an array by initializing each element via the provided initializer. +/// +/// # Examples +/// +/// ```rust +/// use kernel::{sync::{Arc, Mutex}, init::pin_init_array_from_fn, new_mutex}; +/// let array: Arc<[Mutex<usize>; 1_000]> = +/// Arc::pin_init(pin_init_array_from_fn(|i| new_mutex!(i)), GFP_KERNEL).unwrap(); +/// assert_eq!(array.len(), 1_000); +/// ``` +pub fn pin_init_array_from_fn<I, const N: usize, T, E>( + mut make_init: impl FnMut(usize) -> I, +) -> impl PinInit<[T; N], E> +where + I: PinInit<T, E>, +{ + let init = move |slot: *mut [T; N]| { + let slot = slot.cast::<T>(); + // Counts the number of initialized elements and when dropped drops that many elements from + // `slot`. + let mut init_count = ScopeGuard::new_with_data(0, |i| { + // We now free every element that has been initialized before. + // SAFETY: The loop initialized exactly the values from 0..i and since we + // return `Err` below, the caller will consider the memory at `slot` as + // uninitialized. + unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) }; + }); + for i in 0..N { + let init = make_init(i); + // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`. + let ptr = unsafe { slot.add(i) }; + // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init` + // requirements. + unsafe { init.__pinned_init(ptr) }?; + *init_count += 1; + } + init_count.dismiss(); + Ok(()) + }; + // SAFETY: The initializer above initializes every element of the array. On failure it drops + // any initialized elements and returns `Err`. + unsafe { pin_init_from_closure(init) } +} + // SAFETY: Every type can be initialized by-value. unsafe impl<T, E> Init<T, E> for T { unsafe fn __init(self, slot: *mut T) -> Result<(), E> { @@ -1202,13 +1103,20 @@ unsafe impl<T, E> Init<T, E> for T { } } +// SAFETY: Every type can be initialized by-value. `__pinned_init` calls `__init`. +unsafe impl<T, E> PinInit<T, E> for T { + unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { + unsafe { self.__init(slot) } + } +} + /// Smart pointer that can initialize memory in-place. pub trait InPlaceInit<T>: Sized { /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this /// type. /// /// If `T: !Unpin` it will not be able to move afterwards. - fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E> + fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Pin<Self>, E> where E: From<AllocError>; @@ -1216,7 +1124,7 @@ pub trait InPlaceInit<T>: Sized { /// type. /// /// If `T: !Unpin` it will not be able to move afterwards. - fn pin_init<E>(init: impl PinInit<T, E>) -> error::Result<Pin<Self>> + fn pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> error::Result<Pin<Self>> where Error: From<E>, { @@ -1224,16 +1132,16 @@ pub trait InPlaceInit<T>: Sized { let init = unsafe { pin_init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e))) }; - Self::try_pin_init(init) + Self::try_pin_init(init, flags) } /// Use the given initializer to in-place initialize a `T`. - fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E> + fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E> where E: From<AllocError>; /// Use the given initializer to in-place initialize a `T`. - fn init<E>(init: impl Init<T, E>) -> error::Result<Self> + fn init<E>(init: impl Init<T, E>, flags: Flags) -> error::Result<Self> where Error: From<E>, { @@ -1241,17 +1149,17 @@ pub trait InPlaceInit<T>: Sized { let init = unsafe { init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e))) }; - Self::try_init(init) + Self::try_init(init, flags) } } impl<T> InPlaceInit<T> for Box<T> { #[inline] - fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E> + fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Pin<Self>, E> where E: From<AllocError>, { - let mut this = Box::try_new_uninit()?; + let mut this = <Box<_> as BoxExt<_>>::new_uninit(flags)?; let slot = this.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, // slot is valid and will not be moved, because we pin it later. @@ -1261,11 +1169,11 @@ impl<T> InPlaceInit<T> for Box<T> { } #[inline] - fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E> + fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E> where E: From<AllocError>, { - let mut this = Box::try_new_uninit()?; + let mut this = <Box<_> as BoxExt<_>>::new_uninit(flags)?; let slot = this.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, // slot is valid. @@ -1277,11 +1185,11 @@ impl<T> InPlaceInit<T> for Box<T> { impl<T> InPlaceInit<T> for UniqueArc<T> { #[inline] - fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E> + fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Pin<Self>, E> where E: From<AllocError>, { - let mut this = UniqueArc::try_new_uninit()?; + let mut this = UniqueArc::new_uninit(flags)?; let slot = this.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, // slot is valid and will not be moved, because we pin it later. @@ -1291,11 +1199,11 @@ impl<T> InPlaceInit<T> for UniqueArc<T> { } #[inline] - fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E> + fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E> where E: From<AllocError>, { - let mut this = UniqueArc::try_new_uninit()?; + let mut this = UniqueArc::new_uninit(flags)?; let slot = this.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, // slot is valid. @@ -1385,11 +1293,23 @@ impl_zeroable! { i8, i16, i32, i64, i128, isize, f32, f64, - // SAFETY: These are ZSTs, there is nothing to zero. - {<T: ?Sized>} PhantomData<T>, core::marker::PhantomPinned, Infallible, (), + // Note: do not add uninhabited types (such as `!` or `core::convert::Infallible`) to this list; + // creating an instance of an uninhabited type is immediate undefined behavior. For more on + // uninhabited/empty types, consult The Rustonomicon: + // <https://doc.rust-lang.org/stable/nomicon/exotic-sizes.html#empty-types>. The Rust Reference + // also has information on undefined behavior: + // <https://doc.rust-lang.org/stable/reference/behavior-considered-undefined.html>. + // + // SAFETY: These are inhabited ZSTs; there is nothing to zero and a valid value exists. + {<T: ?Sized>} PhantomData<T>, core::marker::PhantomPinned, (), // SAFETY: Type is allowed to take any value, including all zeros. {<T>} MaybeUninit<T>, + // SAFETY: Type is allowed to take any value, including all zeros. + {<T>} Opaque<T>, + + // SAFETY: `T: Zeroable` and `UnsafeCell` is `repr(transparent)`. + {<T: ?Sized + Zeroable>} UnsafeCell<T>, // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee). Option<NonZeroU8>, Option<NonZeroU16>, Option<NonZeroU32>, Option<NonZeroU64>, diff --git a/rust/kernel/init/__internal.rs b/rust/kernel/init/__internal.rs index 44751fb62b51..db3372619ecd 100644 --- a/rust/kernel/init/__internal.rs +++ b/rust/kernel/init/__internal.rs @@ -13,7 +13,7 @@ use super::*; /// /// [nomicon]: https://doc.rust-lang.org/nomicon/subtyping.html /// [this table]: https://doc.rust-lang.org/nomicon/phantom-data.html#table-of-phantomdata-patterns -type Invariant<T> = PhantomData<fn(*mut T) -> *mut T>; +pub(super) type Invariant<T> = PhantomData<fn(*mut T) -> *mut T>; /// This is the module-internal type implementing `PinInit` and `Init`. It is unsafe to create this /// type, since the closure needs to fulfill the same safety requirement as the @@ -32,6 +32,18 @@ where } } +// SAFETY: While constructing the `InitClosure`, the user promised that it upholds the +// `__pinned_init` invariants. +unsafe impl<T: ?Sized, F, E> PinInit<T, E> for InitClosure<F, T, E> +where + F: FnOnce(*mut T) -> Result<(), E>, +{ + #[inline] + unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { + (self.0)(slot) + } +} + /// This trait is only implemented via the `#[pin_data]` proc-macro. It is used to facilitate /// the pin projections within the initializers. /// @@ -174,7 +186,6 @@ impl<T> StackInit<T> { /// Can be forgotten to prevent the drop. pub struct DropGuard<T: ?Sized> { ptr: *mut T, - do_drop: Cell<bool>, } impl<T: ?Sized> DropGuard<T> { @@ -190,32 +201,16 @@ impl<T: ?Sized> DropGuard<T> { /// - will not be dropped by any other means. #[inline] pub unsafe fn new(ptr: *mut T) -> Self { - Self { - ptr, - do_drop: Cell::new(true), - } - } - - /// Prevents this guard from dropping the supplied pointer. - /// - /// # Safety - /// - /// This function is unsafe in order to prevent safe code from forgetting this guard. It should - /// only be called by the macros in this module. - #[inline] - pub unsafe fn forget(&self) { - self.do_drop.set(false); + Self { ptr } } } impl<T: ?Sized> Drop for DropGuard<T> { #[inline] fn drop(&mut self) { - if self.do_drop.get() { - // SAFETY: A `DropGuard` can only be constructed using the unsafe `new` function - // ensuring that this operation is safe. - unsafe { ptr::drop_in_place(self.ptr) } - } + // SAFETY: A `DropGuard` can only be constructed using the unsafe `new` function + // ensuring that this operation is safe. + unsafe { ptr::drop_in_place(self.ptr) } } } diff --git a/rust/kernel/init/macros.rs b/rust/kernel/init/macros.rs index 00aa4e956c0a..02ecedc4ae7a 100644 --- a/rust/kernel/init/macros.rs +++ b/rust/kernel/init/macros.rs @@ -1,10 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT //! This module provides the macros that actually implement the proc-macros `pin_data` and -//! `pinned_drop`. +//! `pinned_drop`. It also contains `__init_internal` the implementation of the `{try_}{pin_}init!` +//! macros. //! //! These macros should never be called directly, since they expect their input to be -//! in a certain format which is internal. Use the proc-macros instead. +//! in a certain format which is internal. If used incorrectly, these macros can lead to UB even in +//! safe code! Use the public facing macros instead. //! //! This architecture has been chosen because the kernel does not yet have access to `syn` which //! would make matters a lot easier for implementing these as proc-macros. @@ -43,7 +45,7 @@ //! #[pinned_drop] //! impl PinnedDrop for Foo { //! fn drop(self: Pin<&mut Self>) { -//! println!("{self:p} is getting dropped."); +//! pr_info!("{self:p} is getting dropped."); //! } //! } //! @@ -168,8 +170,10 @@ //! t: T, //! } //! #[doc(hidden)] -//! impl<'__pin, T> -//! ::core::marker::Unpin for Bar<T> where __Unpin<'__pin, T>: ::core::marker::Unpin {} +//! impl<'__pin, T> ::core::marker::Unpin for Bar<T> +//! where +//! __Unpin<'__pin, T>: ::core::marker::Unpin, +//! {} //! // Now we need to ensure that `Bar` does not implement `Drop`, since that would give users //! // access to `&mut self` inside of `drop` even if the struct was pinned. This could lead to //! // UB with only safe code, so we disallow this by giving a trait implementation error using @@ -186,8 +190,9 @@ //! // for safety, but a good sanity check, since no normal code calls `PinnedDrop::drop`. //! #[allow(non_camel_case_types)] //! trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {} -//! impl<T: ::kernel::init::PinnedDrop> -//! UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {} +//! impl< +//! T: ::kernel::init::PinnedDrop, +//! > UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {} //! impl<T> UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for Bar<T> {} //! }; //! ``` @@ -217,7 +222,7 @@ //! // return type and shadow it later when we insert the arbitrary user code. That way //! // there will be no possibility of returning without `unsafe`. //! struct __InitOk; -//! // Get the pin-data type from the initialized type. +//! // Get the data about fields from the supplied type. //! // - the function is unsafe, hence the unsafe block //! // - we `use` the `HasPinData` trait in the block, it is only available in that //! // scope. @@ -225,8 +230,7 @@ //! use ::kernel::init::__internal::HasPinData; //! Self::__pin_data() //! }; -//! // Use `data` to help with type inference, the closure supplied will have the type -//! // `FnOnce(*mut Self) -> Result<__InitOk, Infallible>`. +//! // Ensure that `data` really is of type `PinData` and help with type inference: //! let init = ::kernel::init::__internal::PinData::make_closure::< //! _, //! __InitOk, @@ -234,71 +238,75 @@ //! >(data, move |slot| { //! { //! // Shadow the structure so it cannot be used to return early. If a user -//! // tries to write `return Ok(__InitOk)`, then they get a type error, since -//! // that will refer to this struct instead of the one defined above. +//! // tries to write `return Ok(__InitOk)`, then they get a type error, +//! // since that will refer to this struct instead of the one defined +//! // above. //! struct __InitOk; //! // This is the expansion of `t,`, which is syntactic sugar for `t: t,`. -//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).t), t) }; -//! // Since initialization could fail later (not in this case, since the error -//! // type is `Infallible`) we will need to drop this field if there is an -//! // error later. This `DropGuard` will drop the field when it gets dropped -//! // and has not yet been forgotten. We make a reference to it, so users -//! // cannot `mem::forget` it from the initializer, since the name is the same -//! // as the field (including hygiene). -//! let t = &unsafe { -//! ::kernel::init::__internal::DropGuard::new( -//! ::core::addr_of_mut!((*slot).t), -//! ) +//! { +//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).t), t) }; +//! } +//! // Since initialization could fail later (not in this case, since the +//! // error type is `Infallible`) we will need to drop this field if there +//! // is an error later. This `DropGuard` will drop the field when it gets +//! // dropped and has not yet been forgotten. +//! let __t_guard = unsafe { +//! ::pinned_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).t)) //! }; //! // Expansion of `x: 0,`: -//! // Since this can be an arbitrary expression we cannot place it inside of -//! // the `unsafe` block, so we bind it here. -//! let x = 0; -//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).x), x) }; +//! // Since this can be an arbitrary expression we cannot place it inside +//! // of the `unsafe` block, so we bind it here. +//! { +//! let x = 0; +//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).x), x) }; +//! } //! // We again create a `DropGuard`. -//! let x = &unsafe { -//! ::kernel::init::__internal::DropGuard::new( -//! ::core::addr_of_mut!((*slot).x), -//! ) +//! let __x_guard = unsafe { +//! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).x)) //! }; -//! +//! // Since initialization has successfully completed, we can now forget +//! // the guards. This is not `mem::forget`, since we only have +//! // `&DropGuard`. +//! ::core::mem::forget(__x_guard); +//! ::core::mem::forget(__t_guard); //! // Here we use the type checker to ensure that every field has been //! // initialized exactly once, since this is `if false` it will never get //! // executed, but still type-checked. -//! // Additionally we abuse `slot` to automatically infer the correct type for -//! // the struct. This is also another check that every field is accessible -//! // from this scope. +//! // Additionally we abuse `slot` to automatically infer the correct type +//! // for the struct. This is also another check that every field is +//! // accessible from this scope. //! #[allow(unreachable_code, clippy::diverging_sub_expression)] -//! if false { +//! let _ = || { //! unsafe { //! ::core::ptr::write( //! slot, //! Self { -//! // We only care about typecheck finding every field here, -//! // the expression does not matter, just conjure one using -//! // `panic!()`: +//! // We only care about typecheck finding every field +//! // here, the expression does not matter, just conjure +//! // one using `panic!()`: //! t: ::core::panic!(), //! x: ::core::panic!(), //! }, //! ); //! }; -//! } -//! // Since initialization has successfully completed, we can now forget the -//! // guards. This is not `mem::forget`, since we only have `&DropGuard`. -//! unsafe { ::kernel::init::__internal::DropGuard::forget(t) }; -//! unsafe { ::kernel::init::__internal::DropGuard::forget(x) }; +//! }; //! } //! // We leave the scope above and gain access to the previously shadowed //! // `__InitOk` that we need to return. //! Ok(__InitOk) //! }); //! // Change the return type from `__InitOk` to `()`. -//! let init = move |slot| -> ::core::result::Result<(), ::core::convert::Infallible> { +//! let init = move | +//! slot, +//! | -> ::core::result::Result<(), ::core::convert::Infallible> { //! init(slot).map(|__InitOk| ()) //! }; //! // Construct the initializer. //! let init = unsafe { -//! ::kernel::init::pin_init_from_closure::<_, ::core::convert::Infallible>(init) +//! ::kernel::init::pin_init_from_closure::< +//! _, +//! ::core::convert::Infallible, +//! >(init) //! }; //! init //! } @@ -372,7 +380,10 @@ //! b: Bar<u32>, //! } //! #[doc(hidden)] -//! impl<'__pin> ::core::marker::Unpin for Foo where __Unpin<'__pin>: ::core::marker::Unpin {} +//! impl<'__pin> ::core::marker::Unpin for Foo +//! where +//! __Unpin<'__pin>: ::core::marker::Unpin, +//! {} //! // Since we specified `PinnedDrop` as the argument to `#[pin_data]`, we expect `Foo` to //! // implement `PinnedDrop`. Thus we do not need to prevent `Drop` implementations like //! // before, instead we implement `Drop` here and delegate to `PinnedDrop`. @@ -401,7 +412,7 @@ //! #[pinned_drop] //! impl PinnedDrop for Foo { //! fn drop(self: Pin<&mut Self>) { -//! println!("{self:p} is getting dropped."); +//! pr_info!("{self:p} is getting dropped."); //! } //! } //! ``` @@ -412,7 +423,7 @@ //! // `unsafe`, full path and the token parameter are added, everything else stays the same. //! unsafe impl ::kernel::init::PinnedDrop for Foo { //! fn drop(self: Pin<&mut Self>, _: ::kernel::init::__internal::OnlyCallFromDrop) { -//! println!("{self:p} is getting dropped."); +//! pr_info!("{self:p} is getting dropped."); //! } //! } //! ``` @@ -447,18 +458,21 @@ //! >(data, move |slot| { //! { //! struct __InitOk; -//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).a), a) }; -//! let a = &unsafe { +//! { +//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).a), a) }; +//! } +//! let __a_guard = unsafe { //! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).a)) //! }; -//! let b = Bar::new(36); +//! let init = Bar::new(36); //! unsafe { data.b(::core::addr_of_mut!((*slot).b), b)? }; -//! let b = &unsafe { +//! let __b_guard = unsafe { //! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).b)) //! }; -//! +//! ::core::mem::forget(__b_guard); +//! ::core::mem::forget(__a_guard); //! #[allow(unreachable_code, clippy::diverging_sub_expression)] -//! if false { +//! let _ = || { //! unsafe { //! ::core::ptr::write( //! slot, @@ -468,13 +482,13 @@ //! }, //! ); //! }; -//! } -//! unsafe { ::kernel::init::__internal::DropGuard::forget(a) }; -//! unsafe { ::kernel::init::__internal::DropGuard::forget(b) }; +//! }; //! } //! Ok(__InitOk) //! }); -//! let init = move |slot| -> ::core::result::Result<(), ::core::convert::Infallible> { +//! let init = move | +//! slot, +//! | -> ::core::result::Result<(), ::core::convert::Infallible> { //! init(slot).map(|__InitOk| ()) //! }; //! let init = unsafe { @@ -524,6 +538,7 @@ macro_rules! __pin_data { ), @impl_generics($($impl_generics:tt)*), @ty_generics($($ty_generics:tt)*), + @decl_generics($($decl_generics:tt)*), @body({ $($fields:tt)* }), ) => { // We now use token munching to iterate through all of the fields. While doing this we @@ -546,6 +561,9 @@ macro_rules! __pin_data { @impl_generics($($impl_generics)*), // The 'ty generics', the generics that will need to be specified on the impl blocks. @ty_generics($($ty_generics)*), + // The 'decl generics', the generics that need to be specified on the struct + // definition. + @decl_generics($($decl_generics)*), // The where clause of any impl block and the declaration. @where($($($whr)*)?), // The remaining fields tokens that need to be processed. @@ -571,6 +589,7 @@ macro_rules! __pin_data { @name($name:ident), @impl_generics($($impl_generics:tt)*), @ty_generics($($ty_generics:tt)*), + @decl_generics($($decl_generics:tt)*), @where($($whr:tt)*), // We found a PhantomPinned field, this should generally be pinned! @fields_munch($field:ident : $($($(::)?core::)?marker::)?PhantomPinned, $($rest:tt)*), @@ -593,6 +612,7 @@ macro_rules! __pin_data { @name($name), @impl_generics($($impl_generics)*), @ty_generics($($ty_generics)*), + @decl_generics($($decl_generics)*), @where($($whr)*), @fields_munch($($rest)*), @pinned($($pinned)* $($accum)* $field: ::core::marker::PhantomPinned,), @@ -609,6 +629,7 @@ macro_rules! __pin_data { @name($name:ident), @impl_generics($($impl_generics:tt)*), @ty_generics($($ty_generics:tt)*), + @decl_generics($($decl_generics:tt)*), @where($($whr:tt)*), // We reached the field declaration. @fields_munch($field:ident : $type:ty, $($rest:tt)*), @@ -626,6 +647,7 @@ macro_rules! __pin_data { @name($name), @impl_generics($($impl_generics)*), @ty_generics($($ty_generics)*), + @decl_generics($($decl_generics)*), @where($($whr)*), @fields_munch($($rest)*), @pinned($($pinned)* $($accum)* $field: $type,), @@ -642,6 +664,7 @@ macro_rules! __pin_data { @name($name:ident), @impl_generics($($impl_generics:tt)*), @ty_generics($($ty_generics:tt)*), + @decl_generics($($decl_generics:tt)*), @where($($whr:tt)*), // We reached the field declaration. @fields_munch($field:ident : $type:ty, $($rest:tt)*), @@ -659,6 +682,7 @@ macro_rules! __pin_data { @name($name), @impl_generics($($impl_generics)*), @ty_generics($($ty_generics)*), + @decl_generics($($decl_generics)*), @where($($whr)*), @fields_munch($($rest)*), @pinned($($pinned)*), @@ -675,6 +699,7 @@ macro_rules! __pin_data { @name($name:ident), @impl_generics($($impl_generics:tt)*), @ty_generics($($ty_generics:tt)*), + @decl_generics($($decl_generics:tt)*), @where($($whr:tt)*), // We found the `#[pin]` attr. @fields_munch(#[pin] $($rest:tt)*), @@ -691,6 +716,7 @@ macro_rules! __pin_data { @name($name), @impl_generics($($impl_generics)*), @ty_generics($($ty_generics)*), + @decl_generics($($decl_generics)*), @where($($whr)*), @fields_munch($($rest)*), // We do not include `#[pin]` in the list of attributes, since it is not actually an @@ -710,6 +736,7 @@ macro_rules! __pin_data { @name($name:ident), @impl_generics($($impl_generics:tt)*), @ty_generics($($ty_generics:tt)*), + @decl_generics($($decl_generics:tt)*), @where($($whr:tt)*), // We reached the field declaration with visibility, for simplicity we only munch the // visibility and put it into `$accum`. @@ -727,6 +754,7 @@ macro_rules! __pin_data { @name($name), @impl_generics($($impl_generics)*), @ty_generics($($ty_generics)*), + @decl_generics($($decl_generics)*), @where($($whr)*), @fields_munch($field $($rest)*), @pinned($($pinned)*), @@ -743,6 +771,7 @@ macro_rules! __pin_data { @name($name:ident), @impl_generics($($impl_generics:tt)*), @ty_generics($($ty_generics:tt)*), + @decl_generics($($decl_generics:tt)*), @where($($whr:tt)*), // Some other attribute, just put it into `$accum`. @fields_munch(#[$($attr:tt)*] $($rest:tt)*), @@ -759,6 +788,7 @@ macro_rules! __pin_data { @name($name), @impl_generics($($impl_generics)*), @ty_generics($($ty_generics)*), + @decl_generics($($decl_generics)*), @where($($whr)*), @fields_munch($($rest)*), @pinned($($pinned)*), @@ -775,6 +805,7 @@ macro_rules! __pin_data { @name($name:ident), @impl_generics($($impl_generics:tt)*), @ty_generics($($ty_generics:tt)*), + @decl_generics($($decl_generics:tt)*), @where($($whr:tt)*), // We reached the end of the fields, plus an optional additional comma, since we added one // before and the user is also allowed to put a trailing comma. @@ -788,7 +819,7 @@ macro_rules! __pin_data { ) => { // Declare the struct with all fields in the correct order. $($struct_attrs)* - $vis struct $name <$($impl_generics)*> + $vis struct $name <$($decl_generics)*> where $($whr)* { $($fields)* @@ -960,6 +991,7 @@ macro_rules! __pin_data { where $($whr)* { $( + $(#[$($p_attr)*])* $pvis unsafe fn $p_field<E>( self, slot: *mut $p_type, @@ -969,6 +1001,7 @@ macro_rules! __pin_data { } )* $( + $(#[$($attr)*])* $fvis unsafe fn $field<E>( self, slot: *mut $type, @@ -980,3 +1013,388 @@ macro_rules! __pin_data { } }; } + +/// The internal init macro. Do not call manually! +/// +/// This is called by the `{try_}{pin_}init!` macros with various inputs. +/// +/// This macro has multiple internal call configurations, these are always the very first ident: +/// - nothing: this is the base case and called by the `{try_}{pin_}init!` macros. +/// - `with_update_parsed`: when the `..Zeroable::zeroed()` syntax has been handled. +/// - `init_slot`: recursively creates the code that initializes all fields in `slot`. +/// - `make_initializer`: recursively create the struct initializer that guarantees that every +/// field has been initialized exactly once. +#[doc(hidden)] +#[macro_export] +macro_rules! __init_internal { + ( + @this($($this:ident)?), + @typ($t:path), + @fields($($fields:tt)*), + @error($err:ty), + // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData` + // case. + @data($data:ident, $($use_data:ident)?), + // `HasPinData` or `HasInitData`. + @has_data($has_data:ident, $get_data:ident), + // `pin_init_from_closure` or `init_from_closure`. + @construct_closure($construct_closure:ident), + @munch_fields(), + ) => { + $crate::__init_internal!(with_update_parsed: + @this($($this)?), + @typ($t), + @fields($($fields)*), + @error($err), + @data($data, $($use_data)?), + @has_data($has_data, $get_data), + @construct_closure($construct_closure), + @zeroed(), // Nothing means default behavior. + ) + }; + ( + @this($($this:ident)?), + @typ($t:path), + @fields($($fields:tt)*), + @error($err:ty), + // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData` + // case. + @data($data:ident, $($use_data:ident)?), + // `HasPinData` or `HasInitData`. + @has_data($has_data:ident, $get_data:ident), + // `pin_init_from_closure` or `init_from_closure`. + @construct_closure($construct_closure:ident), + @munch_fields(..Zeroable::zeroed()), + ) => { + $crate::__init_internal!(with_update_parsed: + @this($($this)?), + @typ($t), + @fields($($fields)*), + @error($err), + @data($data, $($use_data)?), + @has_data($has_data, $get_data), + @construct_closure($construct_closure), + @zeroed(()), // `()` means zero all fields not mentioned. + ) + }; + ( + @this($($this:ident)?), + @typ($t:path), + @fields($($fields:tt)*), + @error($err:ty), + // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData` + // case. + @data($data:ident, $($use_data:ident)?), + // `HasPinData` or `HasInitData`. + @has_data($has_data:ident, $get_data:ident), + // `pin_init_from_closure` or `init_from_closure`. + @construct_closure($construct_closure:ident), + @munch_fields($ignore:tt $($rest:tt)*), + ) => { + $crate::__init_internal!( + @this($($this)?), + @typ($t), + @fields($($fields)*), + @error($err), + @data($data, $($use_data)?), + @has_data($has_data, $get_data), + @construct_closure($construct_closure), + @munch_fields($($rest)*), + ) + }; + (with_update_parsed: + @this($($this:ident)?), + @typ($t:path), + @fields($($fields:tt)*), + @error($err:ty), + // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData` + // case. + @data($data:ident, $($use_data:ident)?), + // `HasPinData` or `HasInitData`. + @has_data($has_data:ident, $get_data:ident), + // `pin_init_from_closure` or `init_from_closure`. + @construct_closure($construct_closure:ident), + @zeroed($($init_zeroed:expr)?), + ) => {{ + // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return + // type and shadow it later when we insert the arbitrary user code. That way there will be + // no possibility of returning without `unsafe`. + struct __InitOk; + // Get the data about fields from the supplied type. + let data = unsafe { + use $crate::init::__internal::$has_data; + // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal + // information that is associated to already parsed fragments, so a path fragment + // cannot be used in this position. Doing the retokenization results in valid rust + // code. + ::kernel::macros::paste!($t::$get_data()) + }; + // Ensure that `data` really is of type `$data` and help with type inference: + let init = $crate::init::__internal::$data::make_closure::<_, __InitOk, $err>( + data, + move |slot| { + { + // Shadow the structure so it cannot be used to return early. + struct __InitOk; + // If `$init_zeroed` is present we should zero the slot now and not emit an + // error when fields are missing (since they will be zeroed). We also have to + // check that the type actually implements `Zeroable`. + $({ + fn assert_zeroable<T: $crate::init::Zeroable>(_: *mut T) {} + // Ensure that the struct is indeed `Zeroable`. + assert_zeroable(slot); + // SAFETY: The type implements `Zeroable` by the check above. + unsafe { ::core::ptr::write_bytes(slot, 0, 1) }; + $init_zeroed // This will be `()` if set. + })? + // Create the `this` so it can be referenced by the user inside of the + // expressions creating the individual fields. + $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)? + // Initialize every field. + $crate::__init_internal!(init_slot($($use_data)?): + @data(data), + @slot(slot), + @guards(), + @munch_fields($($fields)*,), + ); + // We use unreachable code to ensure that all fields have been mentioned exactly + // once, this struct initializer will still be type-checked and complain with a + // very natural error message if a field is forgotten/mentioned more than once. + #[allow(unreachable_code, clippy::diverging_sub_expression)] + let _ = || { + $crate::__init_internal!(make_initializer: + @slot(slot), + @type_name($t), + @munch_fields($($fields)*,), + @acc(), + ); + }; + } + Ok(__InitOk) + } + ); + let init = move |slot| -> ::core::result::Result<(), $err> { + init(slot).map(|__InitOk| ()) + }; + let init = unsafe { $crate::init::$construct_closure::<_, $err>(init) }; + init + }}; + (init_slot($($use_data:ident)?): + @data($data:ident), + @slot($slot:ident), + @guards($($guards:ident,)*), + @munch_fields($(..Zeroable::zeroed())? $(,)?), + ) => { + // Endpoint of munching, no fields are left. If execution reaches this point, all fields + // have been initialized. Therefore we can now dismiss the guards by forgetting them. + $(::core::mem::forget($guards);)* + }; + (init_slot($use_data:ident): // `use_data` is present, so we use the `data` to init fields. + @data($data:ident), + @slot($slot:ident), + @guards($($guards:ident,)*), + // In-place initialization syntax. + @munch_fields($field:ident <- $val:expr, $($rest:tt)*), + ) => { + let init = $val; + // Call the initializer. + // + // SAFETY: `slot` is valid, because we are inside of an initializer closure, we + // return when an error/panic occurs. + // We also use the `data` to require the correct trait (`Init` or `PinInit`) for `$field`. + unsafe { $data.$field(::core::ptr::addr_of_mut!((*$slot).$field), init)? }; + // Create the drop guard: + // + // We rely on macro hygiene to make it impossible for users to access this local variable. + // We use `paste!` to create new hygiene for `$field`. + ::kernel::macros::paste! { + // SAFETY: We forget the guard later when initialization has succeeded. + let [< __ $field _guard >] = unsafe { + $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) + }; + + $crate::__init_internal!(init_slot($use_data): + @data($data), + @slot($slot), + @guards([< __ $field _guard >], $($guards,)*), + @munch_fields($($rest)*), + ); + } + }; + (init_slot(): // No `use_data`, so we use `Init::__init` directly. + @data($data:ident), + @slot($slot:ident), + @guards($($guards:ident,)*), + // In-place initialization syntax. + @munch_fields($field:ident <- $val:expr, $($rest:tt)*), + ) => { + let init = $val; + // Call the initializer. + // + // SAFETY: `slot` is valid, because we are inside of an initializer closure, we + // return when an error/panic occurs. + unsafe { $crate::init::Init::__init(init, ::core::ptr::addr_of_mut!((*$slot).$field))? }; + // Create the drop guard: + // + // We rely on macro hygiene to make it impossible for users to access this local variable. + // We use `paste!` to create new hygiene for `$field`. + ::kernel::macros::paste! { + // SAFETY: We forget the guard later when initialization has succeeded. + let [< __ $field _guard >] = unsafe { + $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) + }; + + $crate::__init_internal!(init_slot(): + @data($data), + @slot($slot), + @guards([< __ $field _guard >], $($guards,)*), + @munch_fields($($rest)*), + ); + } + }; + (init_slot($($use_data:ident)?): + @data($data:ident), + @slot($slot:ident), + @guards($($guards:ident,)*), + // Init by-value. + @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), + ) => { + { + $(let $field = $val;)? + // Initialize the field. + // + // SAFETY: The memory at `slot` is uninitialized. + unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) }; + } + // Create the drop guard: + // + // We rely on macro hygiene to make it impossible for users to access this local variable. + // We use `paste!` to create new hygiene for `$field`. + ::kernel::macros::paste! { + // SAFETY: We forget the guard later when initialization has succeeded. + let [< __ $field _guard >] = unsafe { + $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) + }; + + $crate::__init_internal!(init_slot($($use_data)?): + @data($data), + @slot($slot), + @guards([< __ $field _guard >], $($guards,)*), + @munch_fields($($rest)*), + ); + } + }; + (make_initializer: + @slot($slot:ident), + @type_name($t:path), + @munch_fields(..Zeroable::zeroed() $(,)?), + @acc($($acc:tt)*), + ) => { + // Endpoint, nothing more to munch, create the initializer. Since the users specified + // `..Zeroable::zeroed()`, the slot will already have been zeroed and all field that have + // not been overwritten are thus zero and initialized. We still check that all fields are + // actually accessible by using the struct update syntax ourselves. + // We are inside of a closure that is never executed and thus we can abuse `slot` to + // get the correct type inference here: + #[allow(unused_assignments)] + unsafe { + let mut zeroed = ::core::mem::zeroed(); + // We have to use type inference here to make zeroed have the correct type. This does + // not get executed, so it has no effect. + ::core::ptr::write($slot, zeroed); + zeroed = ::core::mem::zeroed(); + // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal + // information that is associated to already parsed fragments, so a path fragment + // cannot be used in this position. Doing the retokenization results in valid rust + // code. + ::kernel::macros::paste!( + ::core::ptr::write($slot, $t { + $($acc)* + ..zeroed + }); + ); + } + }; + (make_initializer: + @slot($slot:ident), + @type_name($t:path), + @munch_fields($(,)?), + @acc($($acc:tt)*), + ) => { + // Endpoint, nothing more to munch, create the initializer. + // Since we are in the closure that is never called, this will never get executed. + // We abuse `slot` to get the correct type inference here: + unsafe { + // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal + // information that is associated to already parsed fragments, so a path fragment + // cannot be used in this position. Doing the retokenization results in valid rust + // code. + ::kernel::macros::paste!( + ::core::ptr::write($slot, $t { + $($acc)* + }); + ); + } + }; + (make_initializer: + @slot($slot:ident), + @type_name($t:path), + @munch_fields($field:ident <- $val:expr, $($rest:tt)*), + @acc($($acc:tt)*), + ) => { + $crate::__init_internal!(make_initializer: + @slot($slot), + @type_name($t), + @munch_fields($($rest)*), + @acc($($acc)* $field: ::core::panic!(),), + ); + }; + (make_initializer: + @slot($slot:ident), + @type_name($t:path), + @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), + @acc($($acc:tt)*), + ) => { + $crate::__init_internal!(make_initializer: + @slot($slot), + @type_name($t), + @munch_fields($($rest)*), + @acc($($acc)* $field: ::core::panic!(),), + ); + }; +} + +#[doc(hidden)] +#[macro_export] +macro_rules! __derive_zeroable { + (parse_input: + @sig( + $(#[$($struct_attr:tt)*])* + $vis:vis struct $name:ident + $(where $($whr:tt)*)? + ), + @impl_generics($($impl_generics:tt)*), + @ty_generics($($ty_generics:tt)*), + @body({ + $( + $(#[$($field_attr:tt)*])* + $field:ident : $field_ty:ty + ),* $(,)? + }), + ) => { + // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero. + #[automatically_derived] + unsafe impl<$($impl_generics)*> $crate::init::Zeroable for $name<$($ty_generics)*> + where + $($($whr)*)? + {} + const _: () = { + fn assert_zeroable<T: ?::core::marker::Sized + $crate::init::Zeroable>() {} + fn ensure_zeroable<$($impl_generics)*>() + where $($($whr)*)? + { + $(assert_zeroable::<$field_ty>();)* + } + }; + }; +} diff --git a/rust/kernel/ioctl.rs b/rust/kernel/ioctl.rs index c49e1a8d3fd0..cfa7d080b531 100644 --- a/rust/kernel/ioctl.rs +++ b/rust/kernel/ioctl.rs @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 -//! ioctl() number definitions +//! `ioctl()` number definitions. //! -//! C header: [`include/asm-generic/ioctl.h`](../../../../include/asm-generic/ioctl.h) +//! C header: [`include/asm-generic/ioctl.h`](srctree/include/asm-generic/ioctl.h) #![allow(non_snake_case)] @@ -28,13 +28,13 @@ pub const fn _IO(ty: u32, nr: u32) -> u32 { _IOC(uapi::_IOC_NONE, ty, nr, 0) } -/// Build an ioctl number for an read-only ioctl. +/// Build an ioctl number for a read-only ioctl. #[inline(always)] pub const fn _IOR<T>(ty: u32, nr: u32) -> u32 { _IOC(uapi::_IOC_READ, ty, nr, core::mem::size_of::<T>()) } -/// Build an ioctl number for an write-only ioctl. +/// Build an ioctl number for a write-only ioctl. #[inline(always)] pub const fn _IOW<T>(ty: u32, nr: u32) -> u32 { _IOC(uapi::_IOC_WRITE, ty, nr, core::mem::size_of::<T>()) diff --git a/rust/kernel/kunit.rs b/rust/kernel/kunit.rs new file mode 100644 index 000000000000..0ba77276ae7e --- /dev/null +++ b/rust/kernel/kunit.rs @@ -0,0 +1,163 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! KUnit-based macros for Rust unit tests. +//! +//! C header: [`include/kunit/test.h`](srctree/include/kunit/test.h) +//! +//! Reference: <https://docs.kernel.org/dev-tools/kunit/index.html> + +use core::{ffi::c_void, fmt}; + +/// Prints a KUnit error-level message. +/// +/// Public but hidden since it should only be used from KUnit generated code. +#[doc(hidden)] +pub fn err(args: fmt::Arguments<'_>) { + // SAFETY: The format string is null-terminated and the `%pA` specifier matches the argument we + // are passing. + #[cfg(CONFIG_PRINTK)] + unsafe { + bindings::_printk( + b"\x013%pA\0".as_ptr() as _, + &args as *const _ as *const c_void, + ); + } +} + +/// Prints a KUnit info-level message. +/// +/// Public but hidden since it should only be used from KUnit generated code. +#[doc(hidden)] +pub fn info(args: fmt::Arguments<'_>) { + // SAFETY: The format string is null-terminated and the `%pA` specifier matches the argument we + // are passing. + #[cfg(CONFIG_PRINTK)] + unsafe { + bindings::_printk( + b"\x016%pA\0".as_ptr() as _, + &args as *const _ as *const c_void, + ); + } +} + +/// Asserts that a boolean expression is `true` at runtime. +/// +/// Public but hidden since it should only be used from generated tests. +/// +/// Unlike the one in `core`, this one does not panic; instead, it is mapped to the KUnit +/// facilities. See [`assert!`] for more details. +#[doc(hidden)] +#[macro_export] +macro_rules! kunit_assert { + ($name:literal, $file:literal, $diff:expr, $condition:expr $(,)?) => { + 'out: { + // Do nothing if the condition is `true`. + if $condition { + break 'out; + } + + static FILE: &'static $crate::str::CStr = $crate::c_str!($file); + static LINE: i32 = core::line!() as i32 - $diff; + static CONDITION: &'static $crate::str::CStr = $crate::c_str!(stringify!($condition)); + + // SAFETY: FFI call without safety requirements. + let kunit_test = unsafe { $crate::bindings::kunit_get_current_test() }; + if kunit_test.is_null() { + // The assertion failed but this task is not running a KUnit test, so we cannot call + // KUnit, but at least print an error to the kernel log. This may happen if this + // macro is called from an spawned thread in a test (see + // `scripts/rustdoc_test_gen.rs`) or if some non-test code calls this macro by + // mistake (it is hidden to prevent that). + // + // This mimics KUnit's failed assertion format. + $crate::kunit::err(format_args!( + " # {}: ASSERTION FAILED at {FILE}:{LINE}\n", + $name + )); + $crate::kunit::err(format_args!( + " Expected {CONDITION} to be true, but is false\n" + )); + $crate::kunit::err(format_args!( + " Failure not reported to KUnit since this is a non-KUnit task\n" + )); + break 'out; + } + + #[repr(transparent)] + struct Location($crate::bindings::kunit_loc); + + #[repr(transparent)] + struct UnaryAssert($crate::bindings::kunit_unary_assert); + + // SAFETY: There is only a static instance and in that one the pointer field points to + // an immutable C string. + unsafe impl Sync for Location {} + + // SAFETY: There is only a static instance and in that one the pointer field points to + // an immutable C string. + unsafe impl Sync for UnaryAssert {} + + static LOCATION: Location = Location($crate::bindings::kunit_loc { + file: FILE.as_char_ptr(), + line: LINE, + }); + static ASSERTION: UnaryAssert = UnaryAssert($crate::bindings::kunit_unary_assert { + assert: $crate::bindings::kunit_assert {}, + condition: CONDITION.as_char_ptr(), + expected_true: true, + }); + + // SAFETY: + // - FFI call. + // - The `kunit_test` pointer is valid because we got it from + // `kunit_get_current_test()` and it was not null. This means we are in a KUnit + // test, and that the pointer can be passed to KUnit functions and assertions. + // - The string pointers (`file` and `condition` above) point to null-terminated + // strings since they are `CStr`s. + // - The function pointer (`format`) points to the proper function. + // - The pointers passed will remain valid since they point to `static`s. + // - The format string is allowed to be null. + // - There are, however, problems with this: first of all, this will end up stopping + // the thread, without running destructors. While that is problematic in itself, + // it is considered UB to have what is effectively a forced foreign unwind + // with `extern "C"` ABI. One could observe the stack that is now gone from + // another thread. We should avoid pinning stack variables to prevent library UB, + // too. For the moment, given that test failures are reported immediately before the + // next test runs, that test failures should be fixed and that KUnit is explicitly + // documented as not suitable for production environments, we feel it is reasonable. + unsafe { + $crate::bindings::__kunit_do_failed_assertion( + kunit_test, + core::ptr::addr_of!(LOCATION.0), + $crate::bindings::kunit_assert_type_KUNIT_ASSERTION, + core::ptr::addr_of!(ASSERTION.0.assert), + Some($crate::bindings::kunit_unary_assert_format), + core::ptr::null(), + ); + } + + // SAFETY: FFI call; the `test` pointer is valid because this hidden macro should only + // be called by the generated documentation tests which forward the test pointer given + // by KUnit. + unsafe { + $crate::bindings::__kunit_abort(kunit_test); + } + } + }; +} + +/// Asserts that two expressions are equal to each other (using [`PartialEq`]). +/// +/// Public but hidden since it should only be used from generated tests. +/// +/// Unlike the one in `core`, this one does not panic; instead, it is mapped to the KUnit +/// facilities. See [`assert!`] for more details. +#[doc(hidden)] +#[macro_export] +macro_rules! kunit_assert_eq { + ($name:literal, $file:literal, $diff:expr, $left:expr, $right:expr $(,)?) => {{ + // For the moment, we just forward to the expression assert because, for binary asserts, + // KUnit supports only a few types (e.g. integers). + $crate::kunit_assert!($name, $file, $diff, $left == $right); + }}; +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 85b261209977..fbd91a48ff8b 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -12,7 +12,6 @@ //! do so first instead of bypassing this crate. #![no_std] -#![feature(allocator_api)] #![feature(coerce_unsized)] #![feature(dispatch_from_dyn)] #![feature(new_uninit)] @@ -27,13 +26,15 @@ compile_error!("Missing kernel configuration for conditional compilation"); // Allow proc-macros to refer to `::kernel` inside the `kernel` crate (this crate). extern crate self as kernel; -#[cfg(not(test))] -#[cfg(not(testlib))] -mod allocator; +pub mod alloc; mod build_assert; pub mod error; pub mod init; pub mod ioctl; +#[cfg(CONFIG_KUNIT)] +pub mod kunit; +#[cfg(CONFIG_NET)] +pub mod net; pub mod prelude; pub mod print; mod static_assert; @@ -42,7 +43,9 @@ pub mod std_vendor; pub mod str; pub mod sync; pub mod task; +pub mod time; pub mod types; +pub mod workqueue; #[doc(hidden)] pub use bindings; @@ -58,7 +61,7 @@ const __LOG_PREFIX: &[u8] = b"rust_kernel\0"; /// The top level entrypoint to implementing a kernel module. /// /// For any teardown or cleanup operations, your type may implement [`Drop`]. -pub trait Module: Sized + Sync { +pub trait Module: Sized + Sync + Send { /// Called at module initialization time. /// /// Use this method to perform whatever setup or registration your module @@ -70,7 +73,7 @@ pub trait Module: Sized + Sync { /// Equivalent to `THIS_MODULE` in the C API. /// -/// C header: `include/linux/export.h` +/// C header: [`include/linux/export.h`](srctree/include/linux/export.h) pub struct ThisModule(*mut bindings::module); // SAFETY: `THIS_MODULE` may be used from all threads within a module. @@ -85,6 +88,13 @@ impl ThisModule { pub const unsafe fn from_ptr(ptr: *mut bindings::module) -> ThisModule { ThisModule(ptr) } + + /// Access the raw pointer for this module. + /// + /// It is up to the user to use it correctly. + pub const fn as_ptr(&self) -> *mut bindings::module { + self.0 + } } #[cfg(not(any(testlib, test)))] @@ -93,7 +103,36 @@ fn panic(info: &core::panic::PanicInfo<'_>) -> ! { pr_emerg!("{}\n", info); // SAFETY: FFI call. unsafe { bindings::BUG() }; - // Bindgen currently does not recognize `__noreturn` so `BUG` returns `()` - // instead of `!`. See <https://github.com/rust-lang/rust-bindgen/issues/2094>. - loop {} +} + +/// Produces a pointer to an object from a pointer to one of its fields. +/// +/// # Safety +/// +/// The pointer passed to this macro, and the pointer returned by this macro, must both be in +/// bounds of the same allocation. +/// +/// # Examples +/// +/// ``` +/// # use kernel::container_of; +/// struct Test { +/// a: u64, +/// b: u32, +/// } +/// +/// let test = Test { a: 10, b: 20 }; +/// let b_ptr = &test.b; +/// // SAFETY: The pointer points at the `b` field of a `Test`, so the resulting pointer will be +/// // in-bounds of the same allocation as `b_ptr`. +/// let test_alias = unsafe { container_of!(b_ptr, Test, b) }; +/// assert!(core::ptr::eq(&test, test_alias)); +/// ``` +#[macro_export] +macro_rules! container_of { + ($ptr:expr, $type:ty, $($f:tt)*) => {{ + let ptr = $ptr as *const _ as *const u8; + let offset: usize = ::core::mem::offset_of!($type, $($f)*); + ptr.sub(offset) as *const $type + }} } diff --git a/rust/kernel/net.rs b/rust/kernel/net.rs new file mode 100644 index 000000000000..fe415cb369d3 --- /dev/null +++ b/rust/kernel/net.rs @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Networking. + +#[cfg(CONFIG_RUST_PHYLIB_ABSTRACTIONS)] +pub mod phy; diff --git a/rust/kernel/net/phy.rs b/rust/kernel/net/phy.rs new file mode 100644 index 000000000000..fd40b703d224 --- /dev/null +++ b/rust/kernel/net/phy.rs @@ -0,0 +1,905 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2023 FUJITA Tomonori <fujita.tomonori@gmail.com> + +//! Network PHY device. +//! +//! C headers: [`include/linux/phy.h`](srctree/include/linux/phy.h). + +use crate::{error::*, prelude::*, types::Opaque}; + +use core::marker::PhantomData; + +/// PHY state machine states. +/// +/// Corresponds to the kernel's [`enum phy_state`]. +/// +/// Some of PHY drivers access to the state of PHY's software state machine. +/// +/// [`enum phy_state`]: srctree/include/linux/phy.h +#[derive(PartialEq, Eq)] +pub enum DeviceState { + /// PHY device and driver are not ready for anything. + Down, + /// PHY is ready to send and receive packets. + Ready, + /// PHY is up, but no polling or interrupts are done. + Halted, + /// PHY is up, but is in an error state. + Error, + /// PHY and attached device are ready to do work. + Up, + /// PHY is currently running. + Running, + /// PHY is up, but not currently plugged in. + NoLink, + /// PHY is performing a cable test. + CableTest, +} + +/// A mode of Ethernet communication. +/// +/// PHY drivers get duplex information from hardware and update the current state. +pub enum DuplexMode { + /// PHY is in full-duplex mode. + Full, + /// PHY is in half-duplex mode. + Half, + /// PHY is in unknown duplex mode. + Unknown, +} + +/// An instance of a PHY device. +/// +/// Wraps the kernel's [`struct phy_device`]. +/// +/// A [`Device`] instance is created when a callback in [`Driver`] is executed. A PHY driver +/// executes [`Driver`]'s methods during the callback. +/// +/// # Invariants +/// +/// Referencing a `phy_device` using this struct asserts that you are in +/// a context where all methods defined on this struct are safe to call. +/// +/// [`struct phy_device`]: srctree/include/linux/phy.h +// During the calls to most functions in [`Driver`], the C side (`PHYLIB`) holds a lock that is +// unique for every instance of [`Device`]. `PHYLIB` uses a different serialization technique for +// [`Driver::resume`] and [`Driver::suspend`]: `PHYLIB` updates `phy_device`'s state with +// the lock held, thus guaranteeing that [`Driver::resume`] has exclusive access to the instance. +// [`Driver::resume`] and [`Driver::suspend`] also are called where only one thread can access +// to the instance. +#[repr(transparent)] +pub struct Device(Opaque<bindings::phy_device>); + +impl Device { + /// Creates a new [`Device`] instance from a raw pointer. + /// + /// # Safety + /// + /// For the duration of 'a, the pointer must point at a valid `phy_device`, + /// and the caller must be in a context where all methods defined on this struct + /// are safe to call. + unsafe fn from_raw<'a>(ptr: *mut bindings::phy_device) -> &'a mut Self { + // CAST: `Self` is a `repr(transparent)` wrapper around `bindings::phy_device`. + let ptr = ptr.cast::<Self>(); + // SAFETY: by the function requirements the pointer is valid and we have unique access for + // the duration of `'a`. + unsafe { &mut *ptr } + } + + /// Gets the id of the PHY. + pub fn phy_id(&self) -> u32 { + let phydev = self.0.get(); + // SAFETY: The struct invariant ensures that we may access + // this field without additional synchronization. + unsafe { (*phydev).phy_id } + } + + /// Gets the state of PHY state machine states. + pub fn state(&self) -> DeviceState { + let phydev = self.0.get(); + // SAFETY: The struct invariant ensures that we may access + // this field without additional synchronization. + let state = unsafe { (*phydev).state }; + // TODO: this conversion code will be replaced with automatically generated code by bindgen + // when it becomes possible. + match state { + bindings::phy_state_PHY_DOWN => DeviceState::Down, + bindings::phy_state_PHY_READY => DeviceState::Ready, + bindings::phy_state_PHY_HALTED => DeviceState::Halted, + bindings::phy_state_PHY_ERROR => DeviceState::Error, + bindings::phy_state_PHY_UP => DeviceState::Up, + bindings::phy_state_PHY_RUNNING => DeviceState::Running, + bindings::phy_state_PHY_NOLINK => DeviceState::NoLink, + bindings::phy_state_PHY_CABLETEST => DeviceState::CableTest, + _ => DeviceState::Error, + } + } + + /// Gets the current link state. + /// + /// It returns true if the link is up. + pub fn is_link_up(&self) -> bool { + const LINK_IS_UP: u64 = 1; + // TODO: the code to access to the bit field will be replaced with automatically + // generated code by bindgen when it becomes possible. + // SAFETY: The struct invariant ensures that we may access + // this field without additional synchronization. + let bit_field = unsafe { &(*self.0.get())._bitfield_1 }; + bit_field.get(14, 1) == LINK_IS_UP + } + + /// Gets the current auto-negotiation configuration. + /// + /// It returns true if auto-negotiation is enabled. + pub fn is_autoneg_enabled(&self) -> bool { + // TODO: the code to access to the bit field will be replaced with automatically + // generated code by bindgen when it becomes possible. + // SAFETY: The struct invariant ensures that we may access + // this field without additional synchronization. + let bit_field = unsafe { &(*self.0.get())._bitfield_1 }; + bit_field.get(13, 1) == bindings::AUTONEG_ENABLE as u64 + } + + /// Gets the current auto-negotiation state. + /// + /// It returns true if auto-negotiation is completed. + pub fn is_autoneg_completed(&self) -> bool { + const AUTONEG_COMPLETED: u64 = 1; + // TODO: the code to access to the bit field will be replaced with automatically + // generated code by bindgen when it becomes possible. + // SAFETY: The struct invariant ensures that we may access + // this field without additional synchronization. + let bit_field = unsafe { &(*self.0.get())._bitfield_1 }; + bit_field.get(15, 1) == AUTONEG_COMPLETED + } + + /// Sets the speed of the PHY. + pub fn set_speed(&mut self, speed: u32) { + let phydev = self.0.get(); + // SAFETY: The struct invariant ensures that we may access + // this field without additional synchronization. + unsafe { (*phydev).speed = speed as i32 }; + } + + /// Sets duplex mode. + pub fn set_duplex(&mut self, mode: DuplexMode) { + let phydev = self.0.get(); + let v = match mode { + DuplexMode::Full => bindings::DUPLEX_FULL as i32, + DuplexMode::Half => bindings::DUPLEX_HALF as i32, + DuplexMode::Unknown => bindings::DUPLEX_UNKNOWN as i32, + }; + // SAFETY: The struct invariant ensures that we may access + // this field without additional synchronization. + unsafe { (*phydev).duplex = v }; + } + + /// Reads a given C22 PHY register. + // This function reads a hardware register and updates the stats so takes `&mut self`. + pub fn read(&mut self, regnum: u16) -> Result<u16> { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call, open code of `phy_read()` with a valid `phy_device` pointer + // `phydev`. + let ret = unsafe { + bindings::mdiobus_read((*phydev).mdio.bus, (*phydev).mdio.addr, regnum.into()) + }; + if ret < 0 { + Err(Error::from_errno(ret)) + } else { + Ok(ret as u16) + } + } + + /// Writes a given C22 PHY register. + pub fn write(&mut self, regnum: u16, val: u16) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call, open code of `phy_write()` with a valid `phy_device` pointer + // `phydev`. + to_result(unsafe { + bindings::mdiobus_write((*phydev).mdio.bus, (*phydev).mdio.addr, regnum.into(), val) + }) + } + + /// Reads a paged register. + pub fn read_paged(&mut self, page: u16, regnum: u16) -> Result<u16> { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + let ret = unsafe { bindings::phy_read_paged(phydev, page.into(), regnum.into()) }; + if ret < 0 { + Err(Error::from_errno(ret)) + } else { + Ok(ret as u16) + } + } + + /// Resolves the advertisements into PHY settings. + pub fn resolve_aneg_linkmode(&mut self) { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + unsafe { bindings::phy_resolve_aneg_linkmode(phydev) }; + } + + /// Executes software reset the PHY via `BMCR_RESET` bit. + pub fn genphy_soft_reset(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::genphy_soft_reset(phydev) }) + } + + /// Initializes the PHY. + pub fn init_hw(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::phy_init_hw(phydev) }) + } + + /// Starts auto-negotiation. + pub fn start_aneg(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::_phy_start_aneg(phydev) }) + } + + /// Resumes the PHY via `BMCR_PDOWN` bit. + pub fn genphy_resume(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::genphy_resume(phydev) }) + } + + /// Suspends the PHY via `BMCR_PDOWN` bit. + pub fn genphy_suspend(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::genphy_suspend(phydev) }) + } + + /// Checks the link status and updates current link state. + pub fn genphy_read_status(&mut self) -> Result<u16> { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + let ret = unsafe { bindings::genphy_read_status(phydev) }; + if ret < 0 { + Err(Error::from_errno(ret)) + } else { + Ok(ret as u16) + } + } + + /// Updates the link status. + pub fn genphy_update_link(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::genphy_update_link(phydev) }) + } + + /// Reads link partner ability. + pub fn genphy_read_lpa(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::genphy_read_lpa(phydev) }) + } + + /// Reads PHY abilities. + pub fn genphy_read_abilities(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::genphy_read_abilities(phydev) }) + } +} + +/// Defines certain other features this PHY supports (like interrupts). +/// +/// These flag values are used in [`Driver::FLAGS`]. +pub mod flags { + /// PHY is internal. + pub const IS_INTERNAL: u32 = bindings::PHY_IS_INTERNAL; + /// PHY needs to be reset after the refclk is enabled. + pub const RST_AFTER_CLK_EN: u32 = bindings::PHY_RST_AFTER_CLK_EN; + /// Polling is used to detect PHY status changes. + pub const POLL_CABLE_TEST: u32 = bindings::PHY_POLL_CABLE_TEST; + /// Don't suspend. + pub const ALWAYS_CALL_SUSPEND: u32 = bindings::PHY_ALWAYS_CALL_SUSPEND; +} + +/// An adapter for the registration of a PHY driver. +struct Adapter<T: Driver> { + _p: PhantomData<T>, +} + +impl<T: Driver> Adapter<T> { + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn soft_reset_callback( + phydev: *mut bindings::phy_device, + ) -> core::ffi::c_int { + from_result(|| { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::soft_reset(dev)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn get_features_callback( + phydev: *mut bindings::phy_device, + ) -> core::ffi::c_int { + from_result(|| { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::get_features(dev)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn suspend_callback(phydev: *mut bindings::phy_device) -> core::ffi::c_int { + from_result(|| { + // SAFETY: The C core code ensures that the accessors on + // `Device` are okay to call even though `phy_device->lock` + // might not be held. + let dev = unsafe { Device::from_raw(phydev) }; + T::suspend(dev)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn resume_callback(phydev: *mut bindings::phy_device) -> core::ffi::c_int { + from_result(|| { + // SAFETY: The C core code ensures that the accessors on + // `Device` are okay to call even though `phy_device->lock` + // might not be held. + let dev = unsafe { Device::from_raw(phydev) }; + T::resume(dev)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn config_aneg_callback( + phydev: *mut bindings::phy_device, + ) -> core::ffi::c_int { + from_result(|| { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::config_aneg(dev)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn read_status_callback( + phydev: *mut bindings::phy_device, + ) -> core::ffi::c_int { + from_result(|| { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::read_status(dev)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn match_phy_device_callback( + phydev: *mut bindings::phy_device, + ) -> core::ffi::c_int { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::match_phy_device(dev) as i32 + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn read_mmd_callback( + phydev: *mut bindings::phy_device, + devnum: i32, + regnum: u16, + ) -> i32 { + from_result(|| { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + // CAST: the C side verifies devnum < 32. + let ret = T::read_mmd(dev, devnum as u8, regnum)?; + Ok(ret.into()) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn write_mmd_callback( + phydev: *mut bindings::phy_device, + devnum: i32, + regnum: u16, + val: u16, + ) -> i32 { + from_result(|| { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::write_mmd(dev, devnum as u8, regnum, val)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn link_change_notify_callback(phydev: *mut bindings::phy_device) { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::link_change_notify(dev); + } +} + +/// Driver structure for a particular PHY type. +/// +/// Wraps the kernel's [`struct phy_driver`]. +/// This is used to register a driver for a particular PHY type with the kernel. +/// +/// # Invariants +/// +/// `self.0` is always in a valid state. +/// +/// [`struct phy_driver`]: srctree/include/linux/phy.h +#[repr(transparent)] +pub struct DriverVTable(Opaque<bindings::phy_driver>); + +// SAFETY: `DriverVTable` doesn't expose any &self method to access internal data, so it's safe to +// share `&DriverVTable` across execution context boundries. +unsafe impl Sync for DriverVTable {} + +/// Creates a [`DriverVTable`] instance from [`Driver`]. +/// +/// This is used by [`module_phy_driver`] macro to create a static array of `phy_driver`. +/// +/// [`module_phy_driver`]: crate::module_phy_driver +pub const fn create_phy_driver<T: Driver>() -> DriverVTable { + // INVARIANT: All the fields of `struct phy_driver` are initialized properly. + DriverVTable(Opaque::new(bindings::phy_driver { + name: T::NAME.as_char_ptr().cast_mut(), + flags: T::FLAGS, + phy_id: T::PHY_DEVICE_ID.id, + phy_id_mask: T::PHY_DEVICE_ID.mask_as_int(), + soft_reset: if T::HAS_SOFT_RESET { + Some(Adapter::<T>::soft_reset_callback) + } else { + None + }, + get_features: if T::HAS_GET_FEATURES { + Some(Adapter::<T>::get_features_callback) + } else { + None + }, + match_phy_device: if T::HAS_MATCH_PHY_DEVICE { + Some(Adapter::<T>::match_phy_device_callback) + } else { + None + }, + suspend: if T::HAS_SUSPEND { + Some(Adapter::<T>::suspend_callback) + } else { + None + }, + resume: if T::HAS_RESUME { + Some(Adapter::<T>::resume_callback) + } else { + None + }, + config_aneg: if T::HAS_CONFIG_ANEG { + Some(Adapter::<T>::config_aneg_callback) + } else { + None + }, + read_status: if T::HAS_READ_STATUS { + Some(Adapter::<T>::read_status_callback) + } else { + None + }, + read_mmd: if T::HAS_READ_MMD { + Some(Adapter::<T>::read_mmd_callback) + } else { + None + }, + write_mmd: if T::HAS_WRITE_MMD { + Some(Adapter::<T>::write_mmd_callback) + } else { + None + }, + link_change_notify: if T::HAS_LINK_CHANGE_NOTIFY { + Some(Adapter::<T>::link_change_notify_callback) + } else { + None + }, + // SAFETY: The rest is zeroed out to initialize `struct phy_driver`, + // sets `Option<&F>` to be `None`. + ..unsafe { core::mem::MaybeUninit::<bindings::phy_driver>::zeroed().assume_init() } + })) +} + +/// Driver implementation for a particular PHY type. +/// +/// This trait is used to create a [`DriverVTable`]. +#[vtable] +pub trait Driver { + /// Defines certain other features this PHY supports. + /// It is a combination of the flags in the [`flags`] module. + const FLAGS: u32 = 0; + + /// The friendly name of this PHY type. + const NAME: &'static CStr; + + /// This driver only works for PHYs with IDs which match this field. + /// The default id and mask are zero. + const PHY_DEVICE_ID: DeviceId = DeviceId::new_with_custom_mask(0, 0); + + /// Issues a PHY software reset. + fn soft_reset(_dev: &mut Device) -> Result { + kernel::build_error(VTABLE_DEFAULT_ERROR) + } + + /// Probes the hardware to determine what abilities it has. + fn get_features(_dev: &mut Device) -> Result { + kernel::build_error(VTABLE_DEFAULT_ERROR) + } + + /// Returns true if this is a suitable driver for the given phydev. + /// If not implemented, matching is based on [`Driver::PHY_DEVICE_ID`]. + fn match_phy_device(_dev: &Device) -> bool { + false + } + + /// Configures the advertisement and resets auto-negotiation + /// if auto-negotiation is enabled. + fn config_aneg(_dev: &mut Device) -> Result { + kernel::build_error(VTABLE_DEFAULT_ERROR) + } + + /// Determines the negotiated speed and duplex. + fn read_status(_dev: &mut Device) -> Result<u16> { + kernel::build_error(VTABLE_DEFAULT_ERROR) + } + + /// Suspends the hardware, saving state if needed. + fn suspend(_dev: &mut Device) -> Result { + kernel::build_error(VTABLE_DEFAULT_ERROR) + } + + /// Resumes the hardware, restoring state if needed. + fn resume(_dev: &mut Device) -> Result { + kernel::build_error(VTABLE_DEFAULT_ERROR) + } + + /// Overrides the default MMD read function for reading a MMD register. + fn read_mmd(_dev: &mut Device, _devnum: u8, _regnum: u16) -> Result<u16> { + kernel::build_error(VTABLE_DEFAULT_ERROR) + } + + /// Overrides the default MMD write function for writing a MMD register. + fn write_mmd(_dev: &mut Device, _devnum: u8, _regnum: u16, _val: u16) -> Result { + kernel::build_error(VTABLE_DEFAULT_ERROR) + } + + /// Callback for notification of link change. + fn link_change_notify(_dev: &mut Device) {} +} + +/// Registration structure for PHY drivers. +/// +/// Registers [`DriverVTable`] instances with the kernel. They will be unregistered when dropped. +/// +/// # Invariants +/// +/// The `drivers` slice are currently registered to the kernel via `phy_drivers_register`. +pub struct Registration { + drivers: Pin<&'static mut [DriverVTable]>, +} + +// SAFETY: The only action allowed in a `Registration` instance is dropping it, which is safe to do +// from any thread because `phy_drivers_unregister` can be called from any thread context. +unsafe impl Send for Registration {} + +impl Registration { + /// Registers a PHY driver. + pub fn register( + module: &'static crate::ThisModule, + drivers: Pin<&'static mut [DriverVTable]>, + ) -> Result<Self> { + if drivers.is_empty() { + return Err(code::EINVAL); + } + // SAFETY: The type invariants of [`DriverVTable`] ensure that all elements of + // the `drivers` slice are initialized properly. `drivers` will not be moved. + // So it's just an FFI call. + to_result(unsafe { + bindings::phy_drivers_register(drivers[0].0.get(), drivers.len().try_into()?, module.0) + })?; + // INVARIANT: The `drivers` slice is successfully registered to the kernel via `phy_drivers_register`. + Ok(Registration { drivers }) + } +} + +impl Drop for Registration { + fn drop(&mut self) { + // SAFETY: The type invariants guarantee that `self.drivers` is valid. + // So it's just an FFI call. + unsafe { + bindings::phy_drivers_unregister(self.drivers[0].0.get(), self.drivers.len() as i32) + }; + } +} + +/// An identifier for PHY devices on an MDIO/MII bus. +/// +/// Represents the kernel's `struct mdio_device_id`. This is used to find an appropriate +/// PHY driver. +pub struct DeviceId { + id: u32, + mask: DeviceMask, +} + +impl DeviceId { + /// Creates a new instance with the exact match mask. + pub const fn new_with_exact_mask(id: u32) -> Self { + DeviceId { + id, + mask: DeviceMask::Exact, + } + } + + /// Creates a new instance with the model match mask. + pub const fn new_with_model_mask(id: u32) -> Self { + DeviceId { + id, + mask: DeviceMask::Model, + } + } + + /// Creates a new instance with the vendor match mask. + pub const fn new_with_vendor_mask(id: u32) -> Self { + DeviceId { + id, + mask: DeviceMask::Vendor, + } + } + + /// Creates a new instance with a custom match mask. + pub const fn new_with_custom_mask(id: u32, mask: u32) -> Self { + DeviceId { + id, + mask: DeviceMask::Custom(mask), + } + } + + /// Creates a new instance from [`Driver`]. + pub const fn new_with_driver<T: Driver>() -> Self { + T::PHY_DEVICE_ID + } + + /// Get a `mask` as u32. + pub const fn mask_as_int(&self) -> u32 { + self.mask.as_int() + } + + // macro use only + #[doc(hidden)] + pub const fn mdio_device_id(&self) -> bindings::mdio_device_id { + bindings::mdio_device_id { + phy_id: self.id, + phy_id_mask: self.mask.as_int(), + } + } +} + +enum DeviceMask { + Exact, + Model, + Vendor, + Custom(u32), +} + +impl DeviceMask { + const MASK_EXACT: u32 = !0; + const MASK_MODEL: u32 = !0 << 4; + const MASK_VENDOR: u32 = !0 << 10; + + const fn as_int(&self) -> u32 { + match self { + DeviceMask::Exact => Self::MASK_EXACT, + DeviceMask::Model => Self::MASK_MODEL, + DeviceMask::Vendor => Self::MASK_VENDOR, + DeviceMask::Custom(mask) => *mask, + } + } +} + +/// Declares a kernel module for PHYs drivers. +/// +/// This creates a static array of kernel's `struct phy_driver` and registers it. +/// This also corresponds to the kernel's `MODULE_DEVICE_TABLE` macro, which embeds the information +/// for module loading into the module binary file. Every driver needs an entry in `device_table`. +/// +/// # Examples +/// +/// ``` +/// # mod module_phy_driver_sample { +/// use kernel::c_str; +/// use kernel::net::phy::{self, DeviceId}; +/// use kernel::prelude::*; +/// +/// kernel::module_phy_driver! { +/// drivers: [PhySample], +/// device_table: [ +/// DeviceId::new_with_driver::<PhySample>() +/// ], +/// name: "rust_sample_phy", +/// author: "Rust for Linux Contributors", +/// description: "Rust sample PHYs driver", +/// license: "GPL", +/// } +/// +/// struct PhySample; +/// +/// #[vtable] +/// impl phy::Driver for PhySample { +/// const NAME: &'static CStr = c_str!("PhySample"); +/// const PHY_DEVICE_ID: phy::DeviceId = phy::DeviceId::new_with_exact_mask(0x00000001); +/// } +/// # } +/// ``` +/// +/// This expands to the following code: +/// +/// ```ignore +/// use kernel::c_str; +/// use kernel::net::phy::{self, DeviceId}; +/// use kernel::prelude::*; +/// +/// struct Module { +/// _reg: ::kernel::net::phy::Registration, +/// } +/// +/// module! { +/// type: Module, +/// name: "rust_sample_phy", +/// author: "Rust for Linux Contributors", +/// description: "Rust sample PHYs driver", +/// license: "GPL", +/// } +/// +/// struct PhySample; +/// +/// #[vtable] +/// impl phy::Driver for PhySample { +/// const NAME: &'static CStr = c_str!("PhySample"); +/// const PHY_DEVICE_ID: phy::DeviceId = phy::DeviceId::new_with_exact_mask(0x00000001); +/// } +/// +/// const _: () = { +/// static mut DRIVERS: [::kernel::net::phy::DriverVTable; 1] = +/// [::kernel::net::phy::create_phy_driver::<PhySample>()]; +/// +/// impl ::kernel::Module for Module { +/// fn init(module: &'static ThisModule) -> Result<Self> { +/// let drivers = unsafe { &mut DRIVERS }; +/// let mut reg = ::kernel::net::phy::Registration::register( +/// module, +/// ::core::pin::Pin::static_mut(drivers), +/// )?; +/// Ok(Module { _reg: reg }) +/// } +/// } +/// }; +/// +/// #[cfg(MODULE)] +/// #[no_mangle] +/// static __mod_mdio__phydev_device_table: [::kernel::bindings::mdio_device_id; 2] = [ +/// ::kernel::bindings::mdio_device_id { +/// phy_id: 0x00000001, +/// phy_id_mask: 0xffffffff, +/// }, +/// ::kernel::bindings::mdio_device_id { +/// phy_id: 0, +/// phy_id_mask: 0, +/// }, +/// ]; +/// ``` +#[macro_export] +macro_rules! module_phy_driver { + (@replace_expr $_t:tt $sub:expr) => {$sub}; + + (@count_devices $($x:expr),*) => { + 0usize $(+ $crate::module_phy_driver!(@replace_expr $x 1usize))* + }; + + (@device_table [$($dev:expr),+]) => { + // SAFETY: C will not read off the end of this constant since the last element is zero. + #[cfg(MODULE)] + #[no_mangle] + static __mod_mdio__phydev_device_table: [$crate::bindings::mdio_device_id; + $crate::module_phy_driver!(@count_devices $($dev),+) + 1] = [ + $($dev.mdio_device_id()),+, + $crate::bindings::mdio_device_id { + phy_id: 0, + phy_id_mask: 0 + } + ]; + }; + + (drivers: [$($driver:ident),+ $(,)?], device_table: [$($dev:expr),+ $(,)?], $($f:tt)*) => { + struct Module { + _reg: $crate::net::phy::Registration, + } + + $crate::prelude::module! { + type: Module, + $($f)* + } + + const _: () = { + static mut DRIVERS: [$crate::net::phy::DriverVTable; + $crate::module_phy_driver!(@count_devices $($driver),+)] = + [$($crate::net::phy::create_phy_driver::<$driver>()),+]; + + impl $crate::Module for Module { + fn init(module: &'static ThisModule) -> Result<Self> { + // SAFETY: The anonymous constant guarantees that nobody else can access + // the `DRIVERS` static. The array is used only in the C side. + let drivers = unsafe { &mut DRIVERS }; + let mut reg = $crate::net::phy::Registration::register( + module, + ::core::pin::Pin::static_mut(drivers), + )?; + Ok(Module { _reg: reg }) + } + } + }; + + $crate::module_phy_driver!(@device_table [$($dev),+]); + } +} diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs index c28587d68ebc..b37a0b3180fb 100644 --- a/rust/kernel/prelude.rs +++ b/rust/kernel/prelude.rs @@ -14,11 +14,13 @@ #[doc(no_inline)] pub use core::pin::Pin; +pub use crate::alloc::{box_ext::BoxExt, flags::*, vec_ext::VecExt}; + #[doc(no_inline)] pub use alloc::{boxed::Box, vec::Vec}; #[doc(no_inline)] -pub use macros::{module, pin_data, pinned_drop, vtable}; +pub use macros::{module, pin_data, pinned_drop, vtable, Zeroable}; pub use super::build_assert; diff --git a/rust/kernel/print.rs b/rust/kernel/print.rs index 8009184bf6d7..a78aa3514a0a 100644 --- a/rust/kernel/print.rs +++ b/rust/kernel/print.rs @@ -2,7 +2,7 @@ //! Printing facilities. //! -//! C header: [`include/linux/printk.h`](../../../../include/linux/printk.h) +//! C header: [`include/linux/printk.h`](srctree/include/linux/printk.h) //! //! Reference: <https://www.kernel.org/doc/html/latest/core-api/printk-basics.html> @@ -13,9 +13,6 @@ use core::{ use crate::str::RawFormatter; -#[cfg(CONFIG_PRINTK)] -use crate::bindings; - // Called from `vsprintf` with format specifier `%pA`. #[no_mangle] unsafe extern "C" fn rust_fmt_argument( @@ -35,8 +32,6 @@ unsafe extern "C" fn rust_fmt_argument( /// Public but hidden since it should only be used from public macros. #[doc(hidden)] pub mod format_strings { - use crate::bindings; - /// The length we copy from the `KERN_*` kernel prefixes. const LENGTH_PREFIX: usize = 2; @@ -48,7 +43,7 @@ pub mod format_strings { /// The format string is always the same for a given level, i.e. for a /// given `prefix`, which are the kernel's `KERN_*` constants. /// - /// [`_printk`]: ../../../../include/linux/printk.h + /// [`_printk`]: srctree/include/linux/printk.h const fn generate(is_cont: bool, prefix: &[u8; 3]) -> [u8; LENGTH] { // Ensure the `KERN_*` macros are what we expect. assert!(prefix[0] == b'\x01'); @@ -97,7 +92,7 @@ pub mod format_strings { /// The format string must be one of the ones in [`format_strings`], and /// the module name must be null-terminated. /// -/// [`_printk`]: ../../../../include/linux/_printk.h +/// [`_printk`]: srctree/include/linux/_printk.h #[doc(hidden)] #[cfg_attr(not(CONFIG_PRINTK), allow(unused_variables))] pub unsafe fn call_printk( @@ -120,7 +115,7 @@ pub unsafe fn call_printk( /// /// Public but hidden since it should only be used from public macros. /// -/// [`_printk`]: ../../../../include/linux/printk.h +/// [`_printk`]: srctree/include/linux/printk.h #[doc(hidden)] #[cfg_attr(not(CONFIG_PRINTK), allow(unused_variables))] pub fn call_printk_cont(args: fmt::Arguments<'_>) { @@ -399,6 +394,7 @@ macro_rules! pr_debug ( /// Mimics the interface of [`std::print!`]. See [`core::fmt`] and /// `alloc::format!` for information about the formatting syntax. /// +/// [`pr_info!`]: crate::pr_info! /// [`pr_cont`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_cont /// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html /// diff --git a/rust/kernel/std_vendor.rs b/rust/kernel/std_vendor.rs index 388d6a5147a2..39679a960c1a 100644 --- a/rust/kernel/std_vendor.rs +++ b/rust/kernel/std_vendor.rs @@ -146,15 +146,16 @@ macro_rules! dbg { // `$val` expression could be a block (`{ .. }`), in which case the `pr_info!` // will be malformed. () => { - $crate::pr_info!("[{}:{}]\n", ::core::file!(), ::core::line!()) + $crate::pr_info!("[{}:{}:{}]\n", ::core::file!(), ::core::line!(), ::core::column!()) }; ($val:expr $(,)?) => { // Use of `match` here is intentional because it affects the lifetimes // of temporaries - https://stackoverflow.com/a/48732525/1063961 match $val { tmp => { - $crate::pr_info!("[{}:{}] {} = {:#?}\n", - ::core::file!(), ::core::line!(), ::core::stringify!($val), &tmp); + $crate::pr_info!("[{}:{}:{}] {} = {:#?}\n", + ::core::file!(), ::core::line!(), ::core::column!(), + ::core::stringify!($val), &tmp); tmp } } diff --git a/rust/kernel/str.rs b/rust/kernel/str.rs index c9dd3bf59e34..bb8d4f41475b 100644 --- a/rust/kernel/str.rs +++ b/rust/kernel/str.rs @@ -2,20 +2,110 @@ //! String representations. -use alloc::alloc::AllocError; +use crate::alloc::{flags::*, vec_ext::VecExt, AllocError}; use alloc::vec::Vec; use core::fmt::{self, Write}; -use core::ops::{self, Deref, Index}; +use core::ops::{self, Deref, DerefMut, Index}; -use crate::{ - bindings, - error::{code::*, Error}, -}; +use crate::error::{code::*, Error}; /// Byte string without UTF-8 validity guarantee. -/// -/// `BStr` is simply an alias to `[u8]`, but has a more evident semantical meaning. -pub type BStr = [u8]; +#[repr(transparent)] +pub struct BStr([u8]); + +impl BStr { + /// Returns the length of this string. + #[inline] + pub const fn len(&self) -> usize { + self.0.len() + } + + /// Returns `true` if the string is empty. + #[inline] + pub const fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Creates a [`BStr`] from a `[u8]`. + #[inline] + pub const fn from_bytes(bytes: &[u8]) -> &Self { + // SAFETY: `BStr` is transparent to `[u8]`. + unsafe { &*(bytes as *const [u8] as *const BStr) } + } +} + +impl fmt::Display for BStr { + /// Formats printable ASCII characters, escaping the rest. + /// + /// ``` + /// # use kernel::{fmt, b_str, str::{BStr, CString}}; + /// let ascii = b_str!("Hello, BStr!"); + /// let s = CString::try_from_fmt(fmt!("{}", ascii)).unwrap(); + /// assert_eq!(s.as_bytes(), "Hello, BStr!".as_bytes()); + /// + /// let non_ascii = b_str!("🦀"); + /// let s = CString::try_from_fmt(fmt!("{}", non_ascii)).unwrap(); + /// assert_eq!(s.as_bytes(), "\\xf0\\x9f\\xa6\\x80".as_bytes()); + /// ``` + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for &b in &self.0 { + match b { + // Common escape codes. + b'\t' => f.write_str("\\t")?, + b'\n' => f.write_str("\\n")?, + b'\r' => f.write_str("\\r")?, + // Printable characters. + 0x20..=0x7e => f.write_char(b as char)?, + _ => write!(f, "\\x{:02x}", b)?, + } + } + Ok(()) + } +} + +impl fmt::Debug for BStr { + /// Formats printable ASCII characters with a double quote on either end, + /// escaping the rest. + /// + /// ``` + /// # use kernel::{fmt, b_str, str::{BStr, CString}}; + /// // Embedded double quotes are escaped. + /// let ascii = b_str!("Hello, \"BStr\"!"); + /// let s = CString::try_from_fmt(fmt!("{:?}", ascii)).unwrap(); + /// assert_eq!(s.as_bytes(), "\"Hello, \\\"BStr\\\"!\"".as_bytes()); + /// + /// let non_ascii = b_str!("😺"); + /// let s = CString::try_from_fmt(fmt!("{:?}", non_ascii)).unwrap(); + /// assert_eq!(s.as_bytes(), "\"\\xf0\\x9f\\x98\\xba\"".as_bytes()); + /// ``` + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_char('"')?; + for &b in &self.0 { + match b { + // Common escape codes. + b'\t' => f.write_str("\\t")?, + b'\n' => f.write_str("\\n")?, + b'\r' => f.write_str("\\r")?, + // String escape characters. + b'\"' => f.write_str("\\\"")?, + b'\\' => f.write_str("\\\\")?, + // Printable characters. + 0x20..=0x7e => f.write_char(b as char)?, + _ => write!(f, "\\x{:02x}", b)?, + } + } + f.write_char('"') + } +} + +impl Deref for BStr { + type Target = [u8]; + + #[inline] + fn deref(&self) -> &Self::Target { + &self.0 + } +} /// Creates a new [`BStr`] from a string literal. /// @@ -33,7 +123,7 @@ pub type BStr = [u8]; macro_rules! b_str { ($str:literal) => {{ const S: &'static str = $str; - const C: &'static $crate::str::BStr = S.as_bytes(); + const C: &'static $crate::str::BStr = $crate::str::BStr::from_bytes(S.as_bytes()); C }}; } @@ -143,19 +233,32 @@ impl CStr { unsafe { core::mem::transmute(bytes) } } + /// Creates a mutable [`CStr`] from a `[u8]` without performing any + /// additional checks. + /// + /// # Safety + /// + /// `bytes` *must* end with a `NUL` byte, and should only have a single + /// `NUL` byte (or the string will be truncated). + #[inline] + pub unsafe fn from_bytes_with_nul_unchecked_mut(bytes: &mut [u8]) -> &mut CStr { + // SAFETY: Properties of `bytes` guaranteed by the safety precondition. + unsafe { &mut *(bytes as *mut [u8] as *mut CStr) } + } + /// Returns a C pointer to the string. #[inline] pub const fn as_char_ptr(&self) -> *const core::ffi::c_char { self.0.as_ptr() as _ } - /// Convert the string to a byte slice without the trailing 0 byte. + /// Convert the string to a byte slice without the trailing `NUL` byte. #[inline] pub fn as_bytes(&self) -> &[u8] { &self.0[..self.len()] } - /// Convert the string to a byte slice containing the trailing 0 byte. + /// Convert the string to a byte slice containing the trailing `NUL` byte. #[inline] pub const fn as_bytes_with_nul(&self) -> &[u8] { &self.0 @@ -191,9 +294,9 @@ impl CStr { /// ``` /// # use kernel::c_str; /// # use kernel::str::CStr; + /// let bar = c_str!("ツ"); /// // SAFETY: String literals are guaranteed to be valid UTF-8 /// // by the Rust compiler. - /// let bar = c_str!("ツ"); /// assert_eq!(unsafe { bar.as_str_unchecked() }, "ツ"); /// ``` #[inline] @@ -206,6 +309,70 @@ impl CStr { pub fn to_cstring(&self) -> Result<CString, AllocError> { CString::try_from(self) } + + /// Converts this [`CStr`] to its ASCII lower case equivalent in-place. + /// + /// ASCII letters 'A' to 'Z' are mapped to 'a' to 'z', + /// but non-ASCII letters are unchanged. + /// + /// To return a new lowercased value without modifying the existing one, use + /// [`to_ascii_lowercase()`]. + /// + /// [`to_ascii_lowercase()`]: #method.to_ascii_lowercase + pub fn make_ascii_lowercase(&mut self) { + // INVARIANT: This doesn't introduce or remove NUL bytes in the C + // string. + self.0.make_ascii_lowercase(); + } + + /// Converts this [`CStr`] to its ASCII upper case equivalent in-place. + /// + /// ASCII letters 'a' to 'z' are mapped to 'A' to 'Z', + /// but non-ASCII letters are unchanged. + /// + /// To return a new uppercased value without modifying the existing one, use + /// [`to_ascii_uppercase()`]. + /// + /// [`to_ascii_uppercase()`]: #method.to_ascii_uppercase + pub fn make_ascii_uppercase(&mut self) { + // INVARIANT: This doesn't introduce or remove NUL bytes in the C + // string. + self.0.make_ascii_uppercase(); + } + + /// Returns a copy of this [`CString`] where each character is mapped to its + /// ASCII lower case equivalent. + /// + /// ASCII letters 'A' to 'Z' are mapped to 'a' to 'z', + /// but non-ASCII letters are unchanged. + /// + /// To lowercase the value in-place, use [`make_ascii_lowercase`]. + /// + /// [`make_ascii_lowercase`]: str::make_ascii_lowercase + pub fn to_ascii_lowercase(&self) -> Result<CString, AllocError> { + let mut s = self.to_cstring()?; + + s.make_ascii_lowercase(); + + Ok(s) + } + + /// Returns a copy of this [`CString`] where each character is mapped to its + /// ASCII upper case equivalent. + /// + /// ASCII letters 'a' to 'z' are mapped to 'A' to 'Z', + /// but non-ASCII letters are unchanged. + /// + /// To uppercase the value in-place, use [`make_ascii_uppercase`]. + /// + /// [`make_ascii_uppercase`]: str::make_ascii_uppercase + pub fn to_ascii_uppercase(&self) -> Result<CString, AllocError> { + let mut s = self.to_cstring()?; + + s.make_ascii_uppercase(); + + Ok(s) + } } impl fmt::Display for CStr { @@ -213,6 +380,7 @@ impl fmt::Display for CStr { /// /// ``` /// # use kernel::c_str; + /// # use kernel::fmt; /// # use kernel::str::CStr; /// # use kernel::str::CString; /// let penguin = c_str!("🐧"); @@ -241,6 +409,7 @@ impl fmt::Debug for CStr { /// /// ``` /// # use kernel::c_str; + /// # use kernel::fmt; /// # use kernel::str::CStr; /// # use kernel::str::CString; /// let penguin = c_str!("🐧"); @@ -269,7 +438,7 @@ impl fmt::Debug for CStr { impl AsRef<BStr> for CStr { #[inline] fn as_ref(&self) -> &BStr { - self.as_bytes() + BStr::from_bytes(self.as_bytes()) } } @@ -278,7 +447,7 @@ impl Deref for CStr { #[inline] fn deref(&self) -> &Self::Target { - self.as_bytes() + self.as_ref() } } @@ -325,7 +494,7 @@ where #[inline] fn index(&self, index: Idx) -> &Self::Output { - &self.as_bytes()[index] + &self.as_ref()[index] } } @@ -355,6 +524,21 @@ macro_rules! c_str { #[cfg(test)] mod tests { use super::*; + use alloc::format; + + const ALL_ASCII_CHARS: &'static str = + "\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\x09\\x0a\\x0b\\x0c\\x0d\\x0e\\x0f\ + \\x10\\x11\\x12\\x13\\x14\\x15\\x16\\x17\\x18\\x19\\x1a\\x1b\\x1c\\x1d\\x1e\\x1f \ + !\"#$%&'()*+,-./0123456789:;<=>?@\ + ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\x7f\ + \\x80\\x81\\x82\\x83\\x84\\x85\\x86\\x87\\x88\\x89\\x8a\\x8b\\x8c\\x8d\\x8e\\x8f\ + \\x90\\x91\\x92\\x93\\x94\\x95\\x96\\x97\\x98\\x99\\x9a\\x9b\\x9c\\x9d\\x9e\\x9f\ + \\xa0\\xa1\\xa2\\xa3\\xa4\\xa5\\xa6\\xa7\\xa8\\xa9\\xaa\\xab\\xac\\xad\\xae\\xaf\ + \\xb0\\xb1\\xb2\\xb3\\xb4\\xb5\\xb6\\xb7\\xb8\\xb9\\xba\\xbb\\xbc\\xbd\\xbe\\xbf\ + \\xc0\\xc1\\xc2\\xc3\\xc4\\xc5\\xc6\\xc7\\xc8\\xc9\\xca\\xcb\\xcc\\xcd\\xce\\xcf\ + \\xd0\\xd1\\xd2\\xd3\\xd4\\xd5\\xd6\\xd7\\xd8\\xd9\\xda\\xdb\\xdc\\xdd\\xde\\xdf\ + \\xe0\\xe1\\xe2\\xe3\\xe4\\xe5\\xe6\\xe7\\xe8\\xe9\\xea\\xeb\\xec\\xed\\xee\\xef\ + \\xf0\\xf1\\xf2\\xf3\\xf4\\xf5\\xf6\\xf7\\xf8\\xf9\\xfa\\xfb\\xfc\\xfd\\xfe\\xff"; #[test] fn test_cstr_to_str() { @@ -379,6 +563,69 @@ mod tests { let unchecked_str = unsafe { checked_cstr.as_str_unchecked() }; assert_eq!(unchecked_str, "🐧"); } + + #[test] + fn test_cstr_display() { + let hello_world = CStr::from_bytes_with_nul(b"hello, world!\0").unwrap(); + assert_eq!(format!("{}", hello_world), "hello, world!"); + let non_printables = CStr::from_bytes_with_nul(b"\x01\x09\x0a\0").unwrap(); + assert_eq!(format!("{}", non_printables), "\\x01\\x09\\x0a"); + let non_ascii = CStr::from_bytes_with_nul(b"d\xe9j\xe0 vu\0").unwrap(); + assert_eq!(format!("{}", non_ascii), "d\\xe9j\\xe0 vu"); + let good_bytes = CStr::from_bytes_with_nul(b"\xf0\x9f\xa6\x80\0").unwrap(); + assert_eq!(format!("{}", good_bytes), "\\xf0\\x9f\\xa6\\x80"); + } + + #[test] + fn test_cstr_display_all_bytes() { + let mut bytes: [u8; 256] = [0; 256]; + // fill `bytes` with [1..=255] + [0] + for i in u8::MIN..=u8::MAX { + bytes[i as usize] = i.wrapping_add(1); + } + let cstr = CStr::from_bytes_with_nul(&bytes).unwrap(); + assert_eq!(format!("{}", cstr), ALL_ASCII_CHARS); + } + + #[test] + fn test_cstr_debug() { + let hello_world = CStr::from_bytes_with_nul(b"hello, world!\0").unwrap(); + assert_eq!(format!("{:?}", hello_world), "\"hello, world!\""); + let non_printables = CStr::from_bytes_with_nul(b"\x01\x09\x0a\0").unwrap(); + assert_eq!(format!("{:?}", non_printables), "\"\\x01\\x09\\x0a\""); + let non_ascii = CStr::from_bytes_with_nul(b"d\xe9j\xe0 vu\0").unwrap(); + assert_eq!(format!("{:?}", non_ascii), "\"d\\xe9j\\xe0 vu\""); + let good_bytes = CStr::from_bytes_with_nul(b"\xf0\x9f\xa6\x80\0").unwrap(); + assert_eq!(format!("{:?}", good_bytes), "\"\\xf0\\x9f\\xa6\\x80\""); + } + + #[test] + fn test_bstr_display() { + let hello_world = BStr::from_bytes(b"hello, world!"); + assert_eq!(format!("{}", hello_world), "hello, world!"); + let escapes = BStr::from_bytes(b"_\t_\n_\r_\\_\'_\"_"); + assert_eq!(format!("{}", escapes), "_\\t_\\n_\\r_\\_'_\"_"); + let others = BStr::from_bytes(b"\x01"); + assert_eq!(format!("{}", others), "\\x01"); + let non_ascii = BStr::from_bytes(b"d\xe9j\xe0 vu"); + assert_eq!(format!("{}", non_ascii), "d\\xe9j\\xe0 vu"); + let good_bytes = BStr::from_bytes(b"\xf0\x9f\xa6\x80"); + assert_eq!(format!("{}", good_bytes), "\\xf0\\x9f\\xa6\\x80"); + } + + #[test] + fn test_bstr_debug() { + let hello_world = BStr::from_bytes(b"hello, world!"); + assert_eq!(format!("{:?}", hello_world), "\"hello, world!\""); + let escapes = BStr::from_bytes(b"_\t_\n_\r_\\_\'_\"_"); + assert_eq!(format!("{:?}", escapes), "\"_\\t_\\n_\\r_\\\\_'_\\\"_\""); + let others = BStr::from_bytes(b"\x01"); + assert_eq!(format!("{:?}", others), "\"\\x01\""); + let non_ascii = BStr::from_bytes(b"d\xe9j\xe0 vu"); + assert_eq!(format!("{:?}", non_ascii), "\"d\\xe9j\\xe0 vu\""); + let good_bytes = BStr::from_bytes(b"\xf0\x9f\xa6\x80"); + assert_eq!(format!("{:?}", good_bytes), "\"\\xf0\\x9f\\xa6\\x80\""); + } } /// Allows formatting of [`fmt::Arguments`] into a raw buffer. @@ -447,7 +694,7 @@ impl RawFormatter { self.pos as _ } - /// Return the number of bytes written to the formatter. + /// Returns the number of bytes written to the formatter. pub(crate) fn bytes_written(&self) -> usize { self.pos - self.beg } @@ -529,7 +776,7 @@ impl fmt::Write for Formatter { /// # Examples /// /// ``` -/// use kernel::str::CString; +/// use kernel::{str::CString, fmt}; /// /// let s = CString::try_from_fmt(fmt!("{}{}{}", "abc", 10, 20)).unwrap(); /// assert_eq!(s.as_bytes_with_nul(), "abc1020\0".as_bytes()); @@ -556,7 +803,7 @@ impl CString { let size = f.bytes_written(); // Allocate a vector with the required number of bytes, and write to it. - let mut buf = Vec::try_with_capacity(size)?; + let mut buf = <Vec<_> as VecExt<_>>::with_capacity(size, GFP_KERNEL)?; // SAFETY: The buffer stored in `buf` is at least of size `size` and is valid for writes. let mut f = unsafe { Formatter::from_buffer(buf.as_mut_ptr(), size) }; f.write_fmt(args)?; @@ -591,13 +838,21 @@ impl Deref for CString { } } +impl DerefMut for CString { + fn deref_mut(&mut self) -> &mut Self::Target { + // SAFETY: A `CString` is always NUL-terminated and contains no other + // NUL bytes. + unsafe { CStr::from_bytes_with_nul_unchecked_mut(self.buf.as_mut_slice()) } + } +} + impl<'a> TryFrom<&'a CStr> for CString { type Error = AllocError; fn try_from(cstr: &'a CStr) -> Result<CString, AllocError> { let mut buf = Vec::new(); - buf.try_extend_from_slice(cstr.as_bytes_with_nul()) + <Vec<_> as VecExt<_>>::extend_from_slice(&mut buf, cstr.as_bytes_with_nul(), GFP_KERNEL) .map_err(|_| AllocError)?; // INVARIANT: The `CStr` and `CString` types have the same invariants for @@ -606,6 +861,12 @@ impl<'a> TryFrom<&'a CStr> for CString { } } +impl fmt::Debug for CString { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&**self, f) + } +} + /// A convenience alias for [`core::format_args`]. #[macro_export] macro_rules! fmt { diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs index d219ee518eff..0ab20975a3b5 100644 --- a/rust/kernel/sync.rs +++ b/rust/kernel/sync.rs @@ -13,8 +13,9 @@ pub mod lock; mod locked_by; pub use arc::{Arc, ArcBorrow, UniqueArc}; -pub use condvar::CondVar; -pub use lock::{mutex::Mutex, spinlock::SpinLock}; +pub use condvar::{new_condvar, CondVar, CondVarTimeoutResult}; +pub use lock::mutex::{new_mutex, Mutex}; +pub use lock::spinlock::{new_spinlock, SpinLock}; pub use locked_by::LockedBy; /// Represents a lockdep class. It's a wrapper around C's `lock_class_key`. @@ -36,6 +37,12 @@ impl LockClassKey { } } +impl Default for LockClassKey { + fn default() -> Self { + Self::new() + } +} + /// Defines a new static lock class and returns a pointer to it. #[doc(hidden)] #[macro_export] diff --git a/rust/kernel/sync/arc.rs b/rust/kernel/sync/arc.rs index 172f563976a9..3673496c2363 100644 --- a/rust/kernel/sync/arc.rs +++ b/rust/kernel/sync/arc.rs @@ -16,7 +16,7 @@ //! [`Arc`]: https://doc.rust-lang.org/std/sync/struct.Arc.html use crate::{ - bindings, + alloc::{box_ext::BoxExt, AllocError, Flags}, error::{self, Error}, init::{self, InPlaceInit, Init, PinInit}, try_init, @@ -24,7 +24,7 @@ use crate::{ }; use alloc::boxed::Box; use core::{ - alloc::AllocError, + alloc::Layout, fmt, marker::{PhantomData, Unsize}, mem::{ManuallyDrop, MaybeUninit}, @@ -56,8 +56,8 @@ mod std_vendor; /// b: u32, /// } /// -/// // Create a ref-counted instance of `Example`. -/// let obj = Arc::try_new(Example { a: 10, b: 20 })?; +/// // Create a refcounted instance of `Example`. +/// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?; /// /// // Get a new pointer to `obj` and increment the refcount. /// let cloned = obj.clone(); @@ -73,6 +73,7 @@ mod std_vendor; /// assert_eq!(cloned.b, 20); /// /// // The refcount drops to zero when `cloned` goes out of scope, and the memory is freed. +/// # Ok::<(), Error>(()) /// ``` /// /// Using `Arc<T>` as the type of `self`: @@ -95,9 +96,10 @@ mod std_vendor; /// } /// } /// -/// let obj = Arc::try_new(Example { a: 10, b: 20 })?; +/// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?; /// obj.use_reference(); /// obj.take_over(); +/// # Ok::<(), Error>(()) /// ``` /// /// Coercion from `Arc<Example>` to `Arc<dyn MyTrait>`: @@ -117,10 +119,11 @@ mod std_vendor; /// impl MyTrait for Example {} /// /// // `obj` has type `Arc<Example>`. -/// let obj: Arc<Example> = Arc::try_new(Example)?; +/// let obj: Arc<Example> = Arc::new(Example, GFP_KERNEL)?; /// /// // `coerced` has type `Arc<dyn MyTrait>`. /// let coerced: Arc<dyn MyTrait> = obj; +/// # Ok::<(), Error>(()) /// ``` pub struct Arc<T: ?Sized> { ptr: NonNull<ArcInner<T>>, @@ -134,6 +137,39 @@ struct ArcInner<T: ?Sized> { data: T, } +impl<T: ?Sized> ArcInner<T> { + /// Converts a pointer to the contents of an [`Arc`] into a pointer to the [`ArcInner`]. + /// + /// # Safety + /// + /// `ptr` must have been returned by a previous call to [`Arc::into_raw`], and the `Arc` must + /// not yet have been destroyed. + unsafe fn container_of(ptr: *const T) -> NonNull<ArcInner<T>> { + let refcount_layout = Layout::new::<bindings::refcount_t>(); + // SAFETY: The caller guarantees that the pointer is valid. + let val_layout = Layout::for_value(unsafe { &*ptr }); + // SAFETY: We're computing the layout of a real struct that existed when compiling this + // binary, so its layout is not so large that it can trigger arithmetic overflow. + let val_offset = unsafe { refcount_layout.extend(val_layout).unwrap_unchecked().1 }; + + // Pointer casts leave the metadata unchanged. This is okay because the metadata of `T` and + // `ArcInner<T>` is the same since `ArcInner` is a struct with `T` as its last field. + // + // This is documented at: + // <https://doc.rust-lang.org/std/ptr/trait.Pointee.html>. + let ptr = ptr as *const ArcInner<T>; + + // SAFETY: The pointer is in-bounds of an allocation both before and after offsetting the + // pointer, since it originates from a previous call to `Arc::into_raw` on an `Arc` that is + // still valid. + let ptr = unsafe { ptr.byte_sub(val_offset) }; + + // SAFETY: The pointer can't be null since you can't have an `ArcInner<T>` value at the null + // address. + unsafe { NonNull::new_unchecked(ptr.cast_mut()) } + } +} + // This is to allow [`Arc`] (and variants) to be used as the type of `self`. impl<T: ?Sized> core::ops::Receiver for Arc<T> {} @@ -159,7 +195,7 @@ unsafe impl<T: ?Sized + Sync + Send> Sync for Arc<T> {} impl<T> Arc<T> { /// Constructs a new reference counted instance of `T`. - pub fn try_new(contents: T) -> Result<Self, AllocError> { + pub fn new(contents: T, flags: Flags) -> Result<Self, AllocError> { // INVARIANT: The refcount is initialised to a non-zero value. let value = ArcInner { // SAFETY: There are no safety requirements for this FFI call. @@ -167,7 +203,7 @@ impl<T> Arc<T> { data: contents, }; - let inner = Box::try_new(value)?; + let inner = <Box<_> as BoxExt<_>>::new(value, flags)?; // SAFETY: We just created `inner` with a reference count of 1, which is owned by the new // `Arc` object. @@ -178,22 +214,22 @@ impl<T> Arc<T> { /// /// If `T: !Unpin` it will not be able to move afterwards. #[inline] - pub fn pin_init<E>(init: impl PinInit<T, E>) -> error::Result<Self> + pub fn pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> error::Result<Self> where Error: From<E>, { - UniqueArc::pin_init(init).map(|u| u.into()) + UniqueArc::pin_init(init, flags).map(|u| u.into()) } /// Use the given initializer to in-place initialize a `T`. /// /// This is equivalent to [`Arc<T>::pin_init`], since an [`Arc`] is always pinned. #[inline] - pub fn init<E>(init: impl Init<T, E>) -> error::Result<Self> + pub fn init<E>(init: impl Init<T, E>, flags: Flags) -> error::Result<Self> where Error: From<E>, { - UniqueArc::init(init).map(|u| u.into()) + UniqueArc::init(init, flags).map(|u| u.into()) } } @@ -212,6 +248,32 @@ impl<T: ?Sized> Arc<T> { } } + /// Convert the [`Arc`] into a raw pointer. + /// + /// The raw pointer has ownership of the refcount that this Arc object owned. + pub fn into_raw(self) -> *const T { + let ptr = self.ptr.as_ptr(); + core::mem::forget(self); + // SAFETY: The pointer is valid. + unsafe { core::ptr::addr_of!((*ptr).data) } + } + + /// Recreates an [`Arc`] instance previously deconstructed via [`Arc::into_raw`]. + /// + /// # Safety + /// + /// `ptr` must have been returned by a previous call to [`Arc::into_raw`]. Additionally, it + /// must not be called more than once for each previous call to [`Arc::into_raw`]. + pub unsafe fn from_raw(ptr: *const T) -> Self { + // SAFETY: The caller promises that this pointer originates from a call to `into_raw` on an + // `Arc` that is still valid. + let ptr = unsafe { ArcInner::container_of(ptr) }; + + // SAFETY: By the safety requirements we know that `ptr` came from `Arc::into_raw`, so the + // reference count held then will be owned by the new `Arc` object. + unsafe { Self::from_inner(ptr) } + } + /// Returns an [`ArcBorrow`] from the given [`Arc`]. /// /// This is useful when the argument of a function call is an [`ArcBorrow`] (e.g., in a method @@ -228,6 +290,68 @@ impl<T: ?Sized> Arc<T> { pub fn ptr_eq(this: &Self, other: &Self) -> bool { core::ptr::eq(this.ptr.as_ptr(), other.ptr.as_ptr()) } + + /// Converts this [`Arc`] into a [`UniqueArc`], or destroys it if it is not unique. + /// + /// When this destroys the `Arc`, it does so while properly avoiding races. This means that + /// this method will never call the destructor of the value. + /// + /// # Examples + /// + /// ``` + /// use kernel::sync::{Arc, UniqueArc}; + /// + /// let arc = Arc::new(42, GFP_KERNEL)?; + /// let unique_arc = arc.into_unique_or_drop(); + /// + /// // The above conversion should succeed since refcount of `arc` is 1. + /// assert!(unique_arc.is_some()); + /// + /// assert_eq!(*(unique_arc.unwrap()), 42); + /// + /// # Ok::<(), Error>(()) + /// ``` + /// + /// ``` + /// use kernel::sync::{Arc, UniqueArc}; + /// + /// let arc = Arc::new(42, GFP_KERNEL)?; + /// let another = arc.clone(); + /// + /// let unique_arc = arc.into_unique_or_drop(); + /// + /// // The above conversion should fail since refcount of `arc` is >1. + /// assert!(unique_arc.is_none()); + /// + /// # Ok::<(), Error>(()) + /// ``` + pub fn into_unique_or_drop(self) -> Option<Pin<UniqueArc<T>>> { + // We will manually manage the refcount in this method, so we disable the destructor. + let me = ManuallyDrop::new(self); + // SAFETY: We own a refcount, so the pointer is still valid. + let refcount = unsafe { me.ptr.as_ref() }.refcount.get(); + + // If the refcount reaches a non-zero value, then we have destroyed this `Arc` and will + // return without further touching the `Arc`. If the refcount reaches zero, then there are + // no other arcs, and we can create a `UniqueArc`. + // + // SAFETY: We own a refcount, so the pointer is not dangling. + let is_zero = unsafe { bindings::refcount_dec_and_test(refcount) }; + if is_zero { + // SAFETY: We have exclusive access to the arc, so we can perform unsynchronized + // accesses to the refcount. + unsafe { core::ptr::write(refcount, bindings::REFCOUNT_INIT(1)) }; + + // INVARIANT: We own the only refcount to this arc, so we may create a `UniqueArc`. We + // must pin the `UniqueArc` because the values was previously in an `Arc`, and they pin + // their values. + Some(Pin::from(UniqueArc { + inner: ManuallyDrop::into_inner(me), + })) + } else { + None + } + } } impl<T: 'static> ForeignOwnable for Arc<T> { @@ -299,7 +423,7 @@ impl<T: ?Sized> Drop for Arc<T> { // The count reached zero, we must free the memory. // // SAFETY: The pointer was initialised from the result of `Box::leak`. - unsafe { Box::from_raw(self.ptr.as_ptr()) }; + unsafe { drop(Box::from_raw(self.ptr.as_ptr())) }; } } } @@ -320,12 +444,12 @@ impl<T: ?Sized> From<Pin<UniqueArc<T>>> for Arc<T> { /// A borrowed reference to an [`Arc`] instance. /// /// For cases when one doesn't ever need to increment the refcount on the allocation, it is simpler -/// to use just `&T`, which we can trivially get from an `Arc<T>` instance. +/// to use just `&T`, which we can trivially get from an [`Arc<T>`] instance. /// /// However, when one may need to increment the refcount, it is preferable to use an `ArcBorrow<T>` /// over `&Arc<T>` because the latter results in a double-indirection: a pointer (shared reference) -/// to a pointer (`Arc<T>`) to the object (`T`). An [`ArcBorrow`] eliminates this double -/// indirection while still allowing one to increment the refcount and getting an `Arc<T>` when/if +/// to a pointer ([`Arc<T>`]) to the object (`T`). An [`ArcBorrow`] eliminates this double +/// indirection while still allowing one to increment the refcount and getting an [`Arc<T>`] when/if /// needed. /// /// # Invariants @@ -336,7 +460,7 @@ impl<T: ?Sized> From<Pin<UniqueArc<T>>> for Arc<T> { /// # Example /// /// ``` -/// use crate::sync::{Arc, ArcBorrow}; +/// use kernel::sync::{Arc, ArcBorrow}; /// /// struct Example; /// @@ -344,17 +468,18 @@ impl<T: ?Sized> From<Pin<UniqueArc<T>>> for Arc<T> { /// e.into() /// } /// -/// let obj = Arc::try_new(Example)?; +/// let obj = Arc::new(Example, GFP_KERNEL)?; /// let cloned = do_something(obj.as_arc_borrow()); /// /// // Assert that both `obj` and `cloned` point to the same underlying object. /// assert!(core::ptr::eq(&*obj, &*cloned)); +/// # Ok::<(), Error>(()) /// ``` /// /// Using `ArcBorrow<T>` as the type of `self`: /// /// ``` -/// use crate::sync::{Arc, ArcBorrow}; +/// use kernel::sync::{Arc, ArcBorrow}; /// /// struct Example { /// a: u32, @@ -367,8 +492,9 @@ impl<T: ?Sized> From<Pin<UniqueArc<T>>> for Arc<T> { /// } /// } /// -/// let obj = Arc::try_new(Example { a: 10, b: 20 })?; +/// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?; /// obj.as_arc_borrow().use_reference(); +/// # Ok::<(), Error>(()) /// ``` pub struct ArcBorrow<'a, T: ?Sized + 'a> { inner: NonNull<ArcInner<T>>, @@ -408,6 +534,27 @@ impl<T: ?Sized> ArcBorrow<'_, T> { _p: PhantomData, } } + + /// Creates an [`ArcBorrow`] to an [`Arc`] that has previously been deconstructed with + /// [`Arc::into_raw`]. + /// + /// # Safety + /// + /// * The provided pointer must originate from a call to [`Arc::into_raw`]. + /// * For the duration of the lifetime annotated on this `ArcBorrow`, the reference count must + /// not hit zero. + /// * For the duration of the lifetime annotated on this `ArcBorrow`, there must not be a + /// [`UniqueArc`] reference to this value. + pub unsafe fn from_raw(ptr: *const T) -> Self { + // SAFETY: The caller promises that this pointer originates from a call to `into_raw` on an + // `Arc` that is still valid. + let ptr = unsafe { ArcInner::container_of(ptr) }; + + // SAFETY: The caller promises that the value remains valid since the reference count must + // not hit zero, and no mutable reference will be created since that would involve a + // `UniqueArc`. + unsafe { Self::new(ptr) } + } } impl<T: ?Sized> From<ArcBorrow<'_, T>> for Arc<T> { @@ -454,7 +601,7 @@ impl<T: ?Sized> Deref for ArcBorrow<'_, T> { /// } /// /// fn test() -> Result<Arc<Example>> { -/// let mut x = UniqueArc::try_new(Example { a: 10, b: 20 })?; +/// let mut x = UniqueArc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?; /// x.a += 1; /// x.b += 1; /// Ok(x.into()) @@ -463,7 +610,7 @@ impl<T: ?Sized> Deref for ArcBorrow<'_, T> { /// # test().unwrap(); /// ``` /// -/// In the following example we first allocate memory for a ref-counted `Example` but we don't +/// In the following example we first allocate memory for a refcounted `Example` but we don't /// initialise it on allocation. We do initialise it later with a call to [`UniqueArc::write`], /// followed by a conversion to `Arc<Example>`. This is particularly useful when allocation happens /// in one context (e.g., sleepable) and initialisation in another (e.g., atomic): @@ -477,7 +624,7 @@ impl<T: ?Sized> Deref for ArcBorrow<'_, T> { /// } /// /// fn test() -> Result<Arc<Example>> { -/// let x = UniqueArc::try_new_uninit()?; +/// let x = UniqueArc::new_uninit(GFP_KERNEL)?; /// Ok(x.write(Example { a: 10, b: 20 }).into()) /// } /// @@ -497,7 +644,7 @@ impl<T: ?Sized> Deref for ArcBorrow<'_, T> { /// } /// /// fn test() -> Result<Arc<Example>> { -/// let mut pinned = Pin::from(UniqueArc::try_new(Example { a: 10, b: 20 })?); +/// let mut pinned = Pin::from(UniqueArc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?); /// // We can modify `pinned` because it is `Unpin`. /// pinned.as_mut().a += 1; /// Ok(pinned.into()) @@ -511,23 +658,26 @@ pub struct UniqueArc<T: ?Sized> { impl<T> UniqueArc<T> { /// Tries to allocate a new [`UniqueArc`] instance. - pub fn try_new(value: T) -> Result<Self, AllocError> { + pub fn new(value: T, flags: Flags) -> Result<Self, AllocError> { Ok(Self { - // INVARIANT: The newly-created object has a ref-count of 1. - inner: Arc::try_new(value)?, + // INVARIANT: The newly-created object has a refcount of 1. + inner: Arc::new(value, flags)?, }) } /// Tries to allocate a new [`UniqueArc`] instance whose contents are not initialised yet. - pub fn try_new_uninit() -> Result<UniqueArc<MaybeUninit<T>>, AllocError> { + pub fn new_uninit(flags: Flags) -> Result<UniqueArc<MaybeUninit<T>>, AllocError> { // INVARIANT: The refcount is initialised to a non-zero value. - let inner = Box::try_init::<AllocError>(try_init!(ArcInner { - // SAFETY: There are no safety requirements for this FFI call. - refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }), - data <- init::uninit::<T, AllocError>(), - }? AllocError))?; + let inner = Box::try_init::<AllocError>( + try_init!(ArcInner { + // SAFETY: There are no safety requirements for this FFI call. + refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }), + data <- init::uninit::<T, AllocError>(), + }? AllocError), + flags, + )?; Ok(UniqueArc { - // INVARIANT: The newly-created object has a ref-count of 1. + // INVARIANT: The newly-created object has a refcount of 1. // SAFETY: The pointer from the `Box` is valid. inner: unsafe { Arc::from_inner(Box::leak(inner).into()) }, }) diff --git a/rust/kernel/sync/condvar.rs b/rust/kernel/sync/condvar.rs index ed353399c4e5..2b306afbe56d 100644 --- a/rust/kernel/sync/condvar.rs +++ b/rust/kernel/sync/condvar.rs @@ -6,8 +6,17 @@ //! variable. use super::{lock::Backend, lock::Guard, LockClassKey}; -use crate::{bindings, init::PinInit, pin_init, str::CStr, types::Opaque}; +use crate::{ + init::PinInit, + pin_init, + str::CStr, + task::{MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE, TASK_NORMAL, TASK_UNINTERRUPTIBLE}, + time::Jiffies, + types::Opaque, +}; +use core::ffi::{c_int, c_long}; use core::marker::PhantomPinned; +use core::ptr; use macros::pin_data; /// Creates a [`CondVar`] initialiser with the given name and a newly-created lock class. @@ -17,6 +26,7 @@ macro_rules! new_condvar { $crate::sync::CondVar::new($crate::optional_name!($($name)?), $crate::static_lock_class!()) }; } +pub use new_condvar; /// A conditional variable. /// @@ -34,8 +44,7 @@ macro_rules! new_condvar { /// The following is an example of using a condvar with a mutex: /// /// ``` -/// use kernel::sync::{CondVar, Mutex}; -/// use kernel::{new_condvar, new_mutex}; +/// use kernel::sync::{new_condvar, new_mutex, CondVar, Mutex}; /// /// #[pin_data] /// pub struct Example { @@ -50,7 +59,7 @@ macro_rules! new_condvar { /// fn wait_for_value(e: &Example, v: u32) { /// let mut guard = e.value.lock(); /// while *guard != v { -/// e.value_changed.wait_uninterruptible(&mut guard); +/// e.value_changed.wait(&mut guard); /// } /// } /// @@ -65,18 +74,20 @@ macro_rules! new_condvar { /// Box::pin_init(pin_init!(Example { /// value <- new_mutex!(0), /// value_changed <- new_condvar!(), -/// })) +/// }), GFP_KERNEL) /// } /// ``` /// -/// [`struct wait_queue_head`]: ../../../include/linux/wait.h +/// [`struct wait_queue_head`]: srctree/include/linux/wait.h #[pin_data] pub struct CondVar { #[pin] - pub(crate) wait_list: Opaque<bindings::wait_queue_head>, + pub(crate) wait_queue_head: Opaque<bindings::wait_queue_head>, /// A condvar needs to be pinned because it contains a [`struct list_head`] that is /// self-referential, so it cannot be safely moved once it is initialised. + /// + /// [`struct list_head`]: srctree/include/linux/types.h #[pin] _pin: PhantomPinned, } @@ -91,77 +102,114 @@ unsafe impl Sync for CondVar {} impl CondVar { /// Constructs a new condvar initialiser. - #[allow(clippy::new_ret_no_self)] pub fn new(name: &'static CStr, key: &'static LockClassKey) -> impl PinInit<Self> { pin_init!(Self { _pin: PhantomPinned, // SAFETY: `slot` is valid while the closure is called and both `name` and `key` have // static lifetimes so they live indefinitely. - wait_list <- Opaque::ffi_init(|slot| unsafe { + wait_queue_head <- Opaque::ffi_init(|slot| unsafe { bindings::__init_waitqueue_head(slot, name.as_char_ptr(), key.as_ptr()) }), }) } - fn wait_internal<T: ?Sized, B: Backend>(&self, wait_state: u32, guard: &mut Guard<'_, T, B>) { + fn wait_internal<T: ?Sized, B: Backend>( + &self, + wait_state: c_int, + guard: &mut Guard<'_, T, B>, + timeout_in_jiffies: c_long, + ) -> c_long { let wait = Opaque::<bindings::wait_queue_entry>::uninit(); // SAFETY: `wait` points to valid memory. unsafe { bindings::init_wait(wait.get()) }; - // SAFETY: Both `wait` and `wait_list` point to valid memory. + // SAFETY: Both `wait` and `wait_queue_head` point to valid memory. unsafe { - bindings::prepare_to_wait_exclusive(self.wait_list.get(), wait.get(), wait_state as _) + bindings::prepare_to_wait_exclusive(self.wait_queue_head.get(), wait.get(), wait_state) }; - // SAFETY: No arguments, switches to another thread. - guard.do_unlocked(|| unsafe { bindings::schedule() }); + // SAFETY: Switches to another thread. The timeout can be any number. + let ret = guard.do_unlocked(|| unsafe { bindings::schedule_timeout(timeout_in_jiffies) }); + + // SAFETY: Both `wait` and `wait_queue_head` point to valid memory. + unsafe { bindings::finish_wait(self.wait_queue_head.get(), wait.get()) }; - // SAFETY: Both `wait` and `wait_list` point to valid memory. - unsafe { bindings::finish_wait(self.wait_list.get(), wait.get()) }; + ret } - /// Releases the lock and waits for a notification in interruptible mode. + /// Releases the lock and waits for a notification in uninterruptible mode. /// /// Atomically releases the given lock (whose ownership is proven by the guard) and puts the /// thread to sleep, reacquiring the lock on wake up. It wakes up when notified by - /// [`CondVar::notify_one`] or [`CondVar::notify_all`], or when the thread receives a signal. - /// It may also wake up spuriously. + /// [`CondVar::notify_one`] or [`CondVar::notify_all`]. Note that it may also wake up + /// spuriously. + pub fn wait<T: ?Sized, B: Backend>(&self, guard: &mut Guard<'_, T, B>) { + self.wait_internal(TASK_UNINTERRUPTIBLE, guard, MAX_SCHEDULE_TIMEOUT); + } + + /// Releases the lock and waits for a notification in interruptible mode. + /// + /// Similar to [`CondVar::wait`], except that the wait is interruptible. That is, the thread may + /// wake up due to signals. It may also wake up spuriously. /// /// Returns whether there is a signal pending. - #[must_use = "wait returns if a signal is pending, so the caller must check the return value"] - pub fn wait<T: ?Sized, B: Backend>(&self, guard: &mut Guard<'_, T, B>) -> bool { - self.wait_internal(bindings::TASK_INTERRUPTIBLE, guard); + #[must_use = "wait_interruptible returns if a signal is pending, so the caller must check the return value"] + pub fn wait_interruptible<T: ?Sized, B: Backend>(&self, guard: &mut Guard<'_, T, B>) -> bool { + self.wait_internal(TASK_INTERRUPTIBLE, guard, MAX_SCHEDULE_TIMEOUT); crate::current!().signal_pending() } - /// Releases the lock and waits for a notification in uninterruptible mode. + /// Releases the lock and waits for a notification in interruptible mode. /// - /// Similar to [`CondVar::wait`], except that the wait is not interruptible. That is, the - /// thread won't wake up due to signals. It may, however, wake up supirously. - pub fn wait_uninterruptible<T: ?Sized, B: Backend>(&self, guard: &mut Guard<'_, T, B>) { - self.wait_internal(bindings::TASK_UNINTERRUPTIBLE, guard) + /// Atomically releases the given lock (whose ownership is proven by the guard) and puts the + /// thread to sleep. It wakes up when notified by [`CondVar::notify_one`] or + /// [`CondVar::notify_all`], or when a timeout occurs, or when the thread receives a signal. + #[must_use = "wait_interruptible_timeout returns if a signal is pending, so the caller must check the return value"] + pub fn wait_interruptible_timeout<T: ?Sized, B: Backend>( + &self, + guard: &mut Guard<'_, T, B>, + jiffies: Jiffies, + ) -> CondVarTimeoutResult { + let jiffies = jiffies.try_into().unwrap_or(MAX_SCHEDULE_TIMEOUT); + let res = self.wait_internal(TASK_INTERRUPTIBLE, guard, jiffies); + + match (res as Jiffies, crate::current!().signal_pending()) { + (jiffies, true) => CondVarTimeoutResult::Signal { jiffies }, + (0, false) => CondVarTimeoutResult::Timeout, + (jiffies, false) => CondVarTimeoutResult::Woken { jiffies }, + } } - /// Calls the kernel function to notify the appropriate number of threads with the given flags. - fn notify(&self, count: i32, flags: u32) { - // SAFETY: `wait_list` points to valid memory. + /// Calls the kernel function to notify the appropriate number of threads. + fn notify(&self, count: c_int) { + // SAFETY: `wait_queue_head` points to valid memory. unsafe { bindings::__wake_up( - self.wait_list.get(), - bindings::TASK_NORMAL, + self.wait_queue_head.get(), + TASK_NORMAL, count, - flags as _, + ptr::null_mut(), ) }; } + /// Calls the kernel function to notify one thread synchronously. + /// + /// This method behaves like `notify_one`, except that it hints to the scheduler that the + /// current thread is about to go to sleep, so it should schedule the target thread on the same + /// CPU. + pub fn notify_sync(&self) { + // SAFETY: `wait_queue_head` points to valid memory. + unsafe { bindings::__wake_up_sync(self.wait_queue_head.get(), TASK_NORMAL) }; + } + /// Wakes a single waiter up, if any. /// /// This is not 'sticky' in the sense that if no thread is waiting, the notification is lost /// completely (as opposed to automatically waking up the next waiter). pub fn notify_one(&self) { - self.notify(1, 0); + self.notify(1); } /// Wakes all waiters up, if any. @@ -169,6 +217,22 @@ impl CondVar { /// This is not 'sticky' in the sense that if no thread is waiting, the notification is lost /// completely (as opposed to automatically waking up the next waiter). pub fn notify_all(&self) { - self.notify(0, 0); + self.notify(0); } } + +/// The return type of `wait_timeout`. +pub enum CondVarTimeoutResult { + /// The timeout was reached. + Timeout, + /// Somebody woke us up. + Woken { + /// Remaining sleep duration. + jiffies: Jiffies, + }, + /// A signal occurred. + Signal { + /// Remaining sleep duration. + jiffies: Jiffies, + }, +} diff --git a/rust/kernel/sync/lock.rs b/rust/kernel/sync/lock.rs index a2216325632d..f6c34ca4d819 100644 --- a/rust/kernel/sync/lock.rs +++ b/rust/kernel/sync/lock.rs @@ -6,7 +6,7 @@ //! spinlocks, raw spinlocks) to be provided with minimal effort. use super::LockClassKey; -use crate::{bindings, init::PinInit, pin_init, str::CStr, types::Opaque, types::ScopeGuard}; +use crate::{init::PinInit, pin_init, str::CStr, types::Opaque, types::ScopeGuard}; use core::{cell::UnsafeCell, marker::PhantomData, marker::PhantomPinned}; use macros::pin_data; @@ -21,14 +21,21 @@ pub mod spinlock; /// # Safety /// /// - Implementers must ensure that only one thread/CPU may access the protected data once the lock -/// is owned, that is, between calls to `lock` and `unlock`. -/// - Implementers must also ensure that `relock` uses the same locking method as the original -/// lock operation. +/// is owned, that is, between calls to [`lock`] and [`unlock`]. +/// - Implementers must also ensure that [`relock`] uses the same locking method as the original +/// lock operation. +/// +/// [`lock`]: Backend::lock +/// [`unlock`]: Backend::unlock +/// [`relock`]: Backend::relock pub unsafe trait Backend { /// The state required by the lock. type State; - /// The state required to be kept between lock and unlock. + /// The state required to be kept between [`lock`] and [`unlock`]. + /// + /// [`lock`]: Backend::lock + /// [`unlock`]: Backend::unlock type GuardState; /// Initialises the lock. @@ -72,8 +79,8 @@ pub unsafe trait Backend { /// A mutual exclusion primitive. /// -/// Exposes one of the kernel locking primitives. Which one is exposed depends on the lock backend -/// specified as the generic parameter `B`. +/// Exposes one of the kernel locking primitives. Which one is exposed depends on the lock +/// [`Backend`] specified as the generic parameter `B`. #[pin_data] pub struct Lock<T: ?Sized, B: Backend> { /// The kernel lock object. @@ -99,7 +106,6 @@ unsafe impl<T: ?Sized + Send, B: Backend> Sync for Lock<T, B> {} impl<T, B: Backend> Lock<T, B> { /// Constructs a new lock initialiser. - #[allow(clippy::new_ret_no_self)] pub fn new(t: T, name: &'static CStr, key: &'static LockClassKey) -> impl PinInit<Self> { pin_init!(Self { data: UnsafeCell::new(t), @@ -126,7 +132,7 @@ impl<T: ?Sized, B: Backend> Lock<T, B> { /// A lock guard. /// -/// Allows mutual exclusion primitives that implement the `Backend` trait to automatically unlock +/// Allows mutual exclusion primitives that implement the [`Backend`] trait to automatically unlock /// when a guard goes out of scope. It also provides a safe and convenient way to access the data /// protected by the lock. #[must_use = "the lock unlocks immediately when the guard is unused"] @@ -140,7 +146,7 @@ pub struct Guard<'a, T: ?Sized, B: Backend> { unsafe impl<T: Sync + ?Sized, B: Backend> Sync for Guard<'_, T, B> {} impl<T: ?Sized, B: Backend> Guard<'_, T, B> { - pub(crate) fn do_unlocked(&mut self, cb: impl FnOnce()) { + pub(crate) fn do_unlocked<U>(&mut self, cb: impl FnOnce() -> U) -> U { // SAFETY: The caller owns the lock, so it is safe to unlock it. unsafe { B::unlock(self.lock.state.get(), &self.state) }; @@ -148,7 +154,7 @@ impl<T: ?Sized, B: Backend> Guard<'_, T, B> { let _relock = ScopeGuard::new(|| unsafe { B::relock(self.lock.state.get(), &mut self.state) }); - cb(); + cb() } } diff --git a/rust/kernel/sync/lock/mutex.rs b/rust/kernel/sync/lock/mutex.rs index 923472f04af4..30632070ee67 100644 --- a/rust/kernel/sync/lock/mutex.rs +++ b/rust/kernel/sync/lock/mutex.rs @@ -4,8 +4,6 @@ //! //! This module allows Rust code to use the kernel's `struct mutex`. -use crate::bindings; - /// Creates a [`Mutex`] initialiser with the given name and a newly-created lock class. /// /// It uses the name if one is given, otherwise it generates one based on the file name and line @@ -17,6 +15,7 @@ macro_rules! new_mutex { $inner, $crate::optional_name!($($name)?), $crate::static_lock_class!()) }; } +pub use new_mutex; /// A mutual exclusion primitive. /// @@ -35,7 +34,7 @@ macro_rules! new_mutex { /// contains an inner struct (`Inner`) that is protected by a mutex. /// /// ``` -/// use kernel::{init::InPlaceInit, init::PinInit, new_mutex, pin_init, sync::Mutex}; +/// use kernel::sync::{new_mutex, Mutex}; /// /// struct Inner { /// a: u32, @@ -59,10 +58,11 @@ macro_rules! new_mutex { /// } /// /// // Allocate a boxed `Example`. -/// let e = Box::pin_init(Example::new())?; +/// let e = Box::pin_init(Example::new(), GFP_KERNEL)?; /// assert_eq!(e.c, 10); /// assert_eq!(e.d.lock().a, 20); /// assert_eq!(e.d.lock().b, 30); +/// # Ok::<(), Error>(()) /// ``` /// /// The following example shows how to use interior mutability to modify the contents of a struct @@ -83,7 +83,7 @@ macro_rules! new_mutex { /// } /// ``` /// -/// [`struct mutex`]: ../../../../include/linux/mutex.h +/// [`struct mutex`]: srctree/include/linux/mutex.h pub type Mutex<T> = super::Lock<T, MutexBackend>; /// A kernel `struct mutex` lock backend. diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spinlock.rs index 979b56464a4e..ea5c5bc1ce12 100644 --- a/rust/kernel/sync/lock/spinlock.rs +++ b/rust/kernel/sync/lock/spinlock.rs @@ -4,8 +4,6 @@ //! //! This module allows Rust code to use the kernel's `spinlock_t`. -use crate::bindings; - /// Creates a [`SpinLock`] initialiser with the given name and a newly-created lock class. /// /// It uses the name if one is given, otherwise it generates one based on the file name and line @@ -17,6 +15,7 @@ macro_rules! new_spinlock { $inner, $crate::optional_name!($($name)?), $crate::static_lock_class!()) }; } +pub use new_spinlock; /// A spinlock. /// @@ -33,7 +32,7 @@ macro_rules! new_spinlock { /// contains an inner struct (`Inner`) that is protected by a spinlock. /// /// ``` -/// use kernel::{init::InPlaceInit, init::PinInit, new_spinlock, pin_init, sync::SpinLock}; +/// use kernel::sync::{new_spinlock, SpinLock}; /// /// struct Inner { /// a: u32, @@ -57,10 +56,11 @@ macro_rules! new_spinlock { /// } /// /// // Allocate a boxed `Example`. -/// let e = Box::pin_init(Example::new())?; +/// let e = Box::pin_init(Example::new(), GFP_KERNEL)?; /// assert_eq!(e.c, 10); /// assert_eq!(e.d.lock().a, 20); /// assert_eq!(e.d.lock().b, 30); +/// # Ok::<(), Error>(()) /// ``` /// /// The following example shows how to use interior mutability to modify the contents of a struct @@ -81,7 +81,7 @@ macro_rules! new_spinlock { /// } /// ``` /// -/// [`spinlock_t`]: ../../../../include/linux/spinlock.h +/// [`spinlock_t`]: srctree/include/linux/spinlock.h pub type SpinLock<T> = super::Lock<T, SpinLockBackend>; /// A kernel `spinlock_t` lock backend. @@ -111,7 +111,7 @@ unsafe impl super::Backend for SpinLockBackend { unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardState) { // SAFETY: The safety requirements of this function ensure that `ptr` is valid and that the - // caller is the owner of the mutex. + // caller is the owner of the spinlock. unsafe { bindings::spin_unlock(ptr) } } } diff --git a/rust/kernel/sync/locked_by.rs b/rust/kernel/sync/locked_by.rs index b17ee5cd98f3..babc731bd5f6 100644 --- a/rust/kernel/sync/locked_by.rs +++ b/rust/kernel/sync/locked_by.rs @@ -9,14 +9,17 @@ use core::{cell::UnsafeCell, mem::size_of, ptr}; /// Allows access to some data to be serialised by a lock that does not wrap it. /// /// In most cases, data protected by a lock is wrapped by the appropriate lock type, e.g., -/// [`super::Mutex`] or [`super::SpinLock`]. [`LockedBy`] is meant for cases when this is not -/// possible. For example, if a container has a lock and some data in the contained elements needs +/// [`Mutex`] or [`SpinLock`]. [`LockedBy`] is meant for cases when this is not possible. +/// For example, if a container has a lock and some data in the contained elements needs /// to be protected by the same lock. /// /// [`LockedBy`] wraps the data in lieu of another locking primitive, and only allows access to it /// when the caller shows evidence that the 'external' lock is locked. It panics if the evidence /// refers to the wrong instance of the lock. /// +/// [`Mutex`]: super::Mutex +/// [`SpinLock`]: super::SpinLock +/// /// # Examples /// /// The following is an example for illustrative purposes: `InnerDirectory::bytes_used` is an diff --git a/rust/kernel/task.rs b/rust/kernel/task.rs index 7eda15e5f1b3..55dff7e088bf 100644 --- a/rust/kernel/task.rs +++ b/rust/kernel/task.rs @@ -2,10 +2,26 @@ //! Tasks (threads and processes). //! -//! C header: [`include/linux/sched.h`](../../../../include/linux/sched.h). - -use crate::{bindings, types::Opaque}; -use core::{marker::PhantomData, ops::Deref, ptr}; +//! C header: [`include/linux/sched.h`](srctree/include/linux/sched.h). + +use crate::types::Opaque; +use core::{ + ffi::{c_int, c_long, c_uint}, + marker::PhantomData, + ops::Deref, + ptr, +}; + +/// A sentinel value used for infinite timeouts. +pub const MAX_SCHEDULE_TIMEOUT: c_long = c_long::MAX; + +/// Bitmask for tasks that are sleeping in an interruptible state. +pub const TASK_INTERRUPTIBLE: c_int = bindings::TASK_INTERRUPTIBLE as c_int; +/// Bitmask for tasks that are sleeping in an uninterruptible state. +pub const TASK_UNINTERRUPTIBLE: c_int = bindings::TASK_UNINTERRUPTIBLE as c_int; +/// Convenience constant for waking up tasks regardless of whether they are in interruptible or +/// uninterruptible sleep. +pub const TASK_NORMAL: c_uint = bindings::TASK_NORMAL as c_uint; /// Returns the currently running task. #[macro_export] @@ -23,7 +39,7 @@ macro_rules! current { /// /// All instances are valid tasks created by the C portion of the kernel. /// -/// Instances of this type are always ref-counted, that is, a call to `get_task_struct` ensures +/// Instances of this type are always refcounted, that is, a call to `get_task_struct` ensures /// that the allocation remains valid at least until the matching call to `put_task_struct`. /// /// # Examples @@ -82,7 +98,7 @@ impl Task { /// Returns a task reference for the currently executing task/thread. /// /// The recommended way to get the current task/thread is to use the - /// [`current`](crate::current) macro because it is safe. + /// [`current`] macro because it is safe. /// /// # Safety /// @@ -116,7 +132,7 @@ impl Task { /// Returns the group leader of the given task. pub fn group_leader(&self) -> &Task { // SAFETY: By the type invariant, we know that `self.0` is a valid task. Valid tasks always - // have a valid group_leader. + // have a valid `group_leader`. let ptr = unsafe { *ptr::addr_of!((*self.0.get()).group_leader) }; // SAFETY: The lifetime of the returned task reference is tied to the lifetime of `self`, @@ -147,7 +163,7 @@ impl Task { } } -// SAFETY: The type invariants guarantee that `Task` is always ref-counted. +// SAFETY: The type invariants guarantee that `Task` is always refcounted. unsafe impl crate::types::AlwaysRefCounted for Task { fn inc_ref(&self) { // SAFETY: The existence of a shared reference means that the refcount is nonzero. diff --git a/rust/kernel/time.rs b/rust/kernel/time.rs new file mode 100644 index 000000000000..e3bb5e89f88d --- /dev/null +++ b/rust/kernel/time.rs @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Time related primitives. +//! +//! This module contains the kernel APIs related to time and timers that +//! have been ported or wrapped for usage by Rust code in the kernel. +//! +//! C header: [`include/linux/jiffies.h`](srctree/include/linux/jiffies.h). +//! C header: [`include/linux/ktime.h`](srctree/include/linux/ktime.h). + +/// The number of nanoseconds per millisecond. +pub const NSEC_PER_MSEC: i64 = bindings::NSEC_PER_MSEC as i64; + +/// The time unit of Linux kernel. One jiffy equals (1/HZ) second. +pub type Jiffies = core::ffi::c_ulong; + +/// The millisecond time unit. +pub type Msecs = core::ffi::c_uint; + +/// Converts milliseconds to jiffies. +#[inline] +pub fn msecs_to_jiffies(msecs: Msecs) -> Jiffies { + // SAFETY: The `__msecs_to_jiffies` function is always safe to call no + // matter what the argument is. + unsafe { bindings::__msecs_to_jiffies(msecs) } +} + +/// A Rust wrapper around a `ktime_t`. +#[repr(transparent)] +#[derive(Copy, Clone)] +pub struct Ktime { + inner: bindings::ktime_t, +} + +impl Ktime { + /// Create a `Ktime` from a raw `ktime_t`. + #[inline] + pub fn from_raw(inner: bindings::ktime_t) -> Self { + Self { inner } + } + + /// Get the current time using `CLOCK_MONOTONIC`. + #[inline] + pub fn ktime_get() -> Self { + // SAFETY: It is always safe to call `ktime_get` outside of NMI context. + Self::from_raw(unsafe { bindings::ktime_get() }) + } + + /// Divide the number of nanoseconds by a compile-time constant. + #[inline] + fn divns_constant<const DIV: i64>(self) -> i64 { + self.to_ns() / DIV + } + + /// Returns the number of nanoseconds. + #[inline] + pub fn to_ns(self) -> i64 { + self.inner + } + + /// Returns the number of milliseconds. + #[inline] + pub fn to_ms(self) -> i64 { + self.divns_constant::<NSEC_PER_MSEC>() + } +} + +/// Returns the number of milliseconds between two ktimes. +#[inline] +pub fn ktime_ms_delta(later: Ktime, earlier: Ktime) -> i64 { + (later - earlier).to_ms() +} + +impl core::ops::Sub for Ktime { + type Output = Ktime; + + #[inline] + fn sub(self, other: Ktime) -> Ktime { + Self { + inner: self.inner - other.inner, + } + } +} diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index d479f8da8f38..2e7c9008621f 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -6,7 +6,7 @@ use crate::init::{self, PinInit}; use alloc::boxed::Box; use core::{ cell::UnsafeCell, - marker::PhantomData, + marker::{PhantomData, PhantomPinned}, mem::MaybeUninit, ops::{Deref, DerefMut}, ptr::NonNull, @@ -46,6 +46,25 @@ pub trait ForeignOwnable: Sized { /// Additionally, all instances (if any) of values returned by [`ForeignOwnable::borrow`] for /// this object must have been dropped. unsafe fn from_foreign(ptr: *const core::ffi::c_void) -> Self; + + /// Tries to convert a foreign-owned object back to a Rust-owned one. + /// + /// A convenience wrapper over [`ForeignOwnable::from_foreign`] that returns [`None`] if `ptr` + /// is null. + /// + /// # Safety + /// + /// `ptr` must either be null or satisfy the safety requirements for + /// [`ForeignOwnable::from_foreign`]. + unsafe fn try_from_foreign(ptr: *const core::ffi::c_void) -> Option<Self> { + if ptr.is_null() { + None + } else { + // SAFETY: Since `ptr` is not null here, then `ptr` satisfies the safety requirements + // of `from_foreign` given the safety requirements of this function. + unsafe { Some(Self::from_foreign(ptr)) } + } + } } impl<T: 'static> ForeignOwnable for Box<T> { @@ -90,8 +109,9 @@ impl ForeignOwnable for () { /// /// In the example below, we have multiple exit paths and we want to log regardless of which one is /// taken: +/// /// ``` -/// # use kernel::ScopeGuard; +/// # use kernel::types::ScopeGuard; /// fn example1(arg: bool) { /// let _log = ScopeGuard::new(|| pr_info!("example1 completed\n")); /// @@ -108,8 +128,9 @@ impl ForeignOwnable for () { /// /// In the example below, we want to log the same message on all early exits but a different one on /// the main exit path: +/// /// ``` -/// # use kernel::ScopeGuard; +/// # use kernel::types::ScopeGuard; /// fn example2(arg: bool) { /// let log = ScopeGuard::new(|| pr_info!("example2 returned early\n")); /// @@ -129,17 +150,18 @@ impl ForeignOwnable for () { /// /// In the example below, we need a mutable object (the vector) to be accessible within the log /// function, so we wrap it in the [`ScopeGuard`]: +/// /// ``` -/// # use kernel::ScopeGuard; +/// # use kernel::types::ScopeGuard; /// fn example3(arg: bool) -> Result { /// let mut vec = /// ScopeGuard::new_with_data(Vec::new(), |v| pr_info!("vec had {} elements\n", v.len())); /// -/// vec.try_push(10u8)?; +/// vec.push(10u8, GFP_KERNEL)?; /// if arg { /// return Ok(()); /// } -/// vec.try_push(20u8)?; +/// vec.push(20u8, GFP_KERNEL)?; /// Ok(()) /// } /// @@ -206,17 +228,26 @@ impl<T, F: FnOnce(T)> Drop for ScopeGuard<T, F> { /// /// This is meant to be used with FFI objects that are never interpreted by Rust code. #[repr(transparent)] -pub struct Opaque<T>(MaybeUninit<UnsafeCell<T>>); +pub struct Opaque<T> { + value: UnsafeCell<MaybeUninit<T>>, + _pin: PhantomPinned, +} impl<T> Opaque<T> { /// Creates a new opaque value. pub const fn new(value: T) -> Self { - Self(MaybeUninit::new(UnsafeCell::new(value))) + Self { + value: UnsafeCell::new(MaybeUninit::new(value)), + _pin: PhantomPinned, + } } /// Creates an uninitialised value. pub const fn uninit() -> Self { - Self(MaybeUninit::uninit()) + Self { + value: UnsafeCell::new(MaybeUninit::uninit()), + _pin: PhantomPinned, + } } /// Creates a pin-initializer from the given initializer closure. @@ -239,8 +270,8 @@ impl<T> Opaque<T> { } /// Returns a raw pointer to the opaque data. - pub fn get(&self) -> *mut T { - UnsafeCell::raw_get(self.0.as_ptr()) + pub const fn get(&self) -> *mut T { + UnsafeCell::get(&self.value).cast::<T>() } /// Gets the value behind `this`. @@ -248,7 +279,7 @@ impl<T> Opaque<T> { /// This function is useful to get access to the value without creating intermediate /// references. pub const fn raw_get(this: *const Self) -> *mut T { - UnsafeCell::raw_get(this.cast::<UnsafeCell<T>>()) + UnsafeCell::raw_get(this.cast::<UnsafeCell<MaybeUninit<T>>>()).cast::<T>() } } diff --git a/rust/kernel/workqueue.rs b/rust/kernel/workqueue.rs new file mode 100644 index 000000000000..1cec63a2aea8 --- /dev/null +++ b/rust/kernel/workqueue.rs @@ -0,0 +1,683 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Work queues. +//! +//! This file has two components: The raw work item API, and the safe work item API. +//! +//! One pattern that is used in both APIs is the `ID` const generic, which exists to allow a single +//! type to define multiple `work_struct` fields. This is done by choosing an id for each field, +//! and using that id to specify which field you wish to use. (The actual value doesn't matter, as +//! long as you use different values for different fields of the same struct.) Since these IDs are +//! generic, they are used only at compile-time, so they shouldn't exist in the final binary. +//! +//! # The raw API +//! +//! The raw API consists of the [`RawWorkItem`] trait, where the work item needs to provide an +//! arbitrary function that knows how to enqueue the work item. It should usually not be used +//! directly, but if you want to, you can use it without using the pieces from the safe API. +//! +//! # The safe API +//! +//! The safe API is used via the [`Work`] struct and [`WorkItem`] traits. Furthermore, it also +//! includes a trait called [`WorkItemPointer`], which is usually not used directly by the user. +//! +//! * The [`Work`] struct is the Rust wrapper for the C `work_struct` type. +//! * The [`WorkItem`] trait is implemented for structs that can be enqueued to a workqueue. +//! * The [`WorkItemPointer`] trait is implemented for the pointer type that points at a something +//! that implements [`WorkItem`]. +//! +//! ## Example +//! +//! This example defines a struct that holds an integer and can be scheduled on the workqueue. When +//! the struct is executed, it will print the integer. Since there is only one `work_struct` field, +//! we do not need to specify ids for the fields. +//! +//! ``` +//! use kernel::sync::Arc; +//! use kernel::workqueue::{self, impl_has_work, new_work, Work, WorkItem}; +//! +//! #[pin_data] +//! struct MyStruct { +//! value: i32, +//! #[pin] +//! work: Work<MyStruct>, +//! } +//! +//! impl_has_work! { +//! impl HasWork<Self> for MyStruct { self.work } +//! } +//! +//! impl MyStruct { +//! fn new(value: i32) -> Result<Arc<Self>> { +//! Arc::pin_init(pin_init!(MyStruct { +//! value, +//! work <- new_work!("MyStruct::work"), +//! }), GFP_KERNEL) +//! } +//! } +//! +//! impl WorkItem for MyStruct { +//! type Pointer = Arc<MyStruct>; +//! +//! fn run(this: Arc<MyStruct>) { +//! pr_info!("The value is: {}", this.value); +//! } +//! } +//! +//! /// This method will enqueue the struct for execution on the system workqueue, where its value +//! /// will be printed. +//! fn print_later(val: Arc<MyStruct>) { +//! let _ = workqueue::system().enqueue(val); +//! } +//! ``` +//! +//! The following example shows how multiple `work_struct` fields can be used: +//! +//! ``` +//! use kernel::sync::Arc; +//! use kernel::workqueue::{self, impl_has_work, new_work, Work, WorkItem}; +//! +//! #[pin_data] +//! struct MyStruct { +//! value_1: i32, +//! value_2: i32, +//! #[pin] +//! work_1: Work<MyStruct, 1>, +//! #[pin] +//! work_2: Work<MyStruct, 2>, +//! } +//! +//! impl_has_work! { +//! impl HasWork<Self, 1> for MyStruct { self.work_1 } +//! impl HasWork<Self, 2> for MyStruct { self.work_2 } +//! } +//! +//! impl MyStruct { +//! fn new(value_1: i32, value_2: i32) -> Result<Arc<Self>> { +//! Arc::pin_init(pin_init!(MyStruct { +//! value_1, +//! value_2, +//! work_1 <- new_work!("MyStruct::work_1"), +//! work_2 <- new_work!("MyStruct::work_2"), +//! }), GFP_KERNEL) +//! } +//! } +//! +//! impl WorkItem<1> for MyStruct { +//! type Pointer = Arc<MyStruct>; +//! +//! fn run(this: Arc<MyStruct>) { +//! pr_info!("The value is: {}", this.value_1); +//! } +//! } +//! +//! impl WorkItem<2> for MyStruct { +//! type Pointer = Arc<MyStruct>; +//! +//! fn run(this: Arc<MyStruct>) { +//! pr_info!("The second value is: {}", this.value_2); +//! } +//! } +//! +//! fn print_1_later(val: Arc<MyStruct>) { +//! let _ = workqueue::system().enqueue::<Arc<MyStruct>, 1>(val); +//! } +//! +//! fn print_2_later(val: Arc<MyStruct>) { +//! let _ = workqueue::system().enqueue::<Arc<MyStruct>, 2>(val); +//! } +//! ``` +//! +//! C header: [`include/linux/workqueue.h`](srctree/include/linux/workqueue.h) + +use crate::alloc::{AllocError, Flags}; +use crate::{prelude::*, sync::Arc, sync::LockClassKey, types::Opaque}; +use core::marker::PhantomData; + +/// Creates a [`Work`] initialiser with the given name and a newly-created lock class. +#[macro_export] +macro_rules! new_work { + ($($name:literal)?) => { + $crate::workqueue::Work::new($crate::optional_name!($($name)?), $crate::static_lock_class!()) + }; +} +pub use new_work; + +/// A kernel work queue. +/// +/// Wraps the kernel's C `struct workqueue_struct`. +/// +/// It allows work items to be queued to run on thread pools managed by the kernel. Several are +/// always available, for example, `system`, `system_highpri`, `system_long`, etc. +#[repr(transparent)] +pub struct Queue(Opaque<bindings::workqueue_struct>); + +// SAFETY: Accesses to workqueues used by [`Queue`] are thread-safe. +unsafe impl Send for Queue {} +// SAFETY: Accesses to workqueues used by [`Queue`] are thread-safe. +unsafe impl Sync for Queue {} + +impl Queue { + /// Use the provided `struct workqueue_struct` with Rust. + /// + /// # Safety + /// + /// The caller must ensure that the provided raw pointer is not dangling, that it points at a + /// valid workqueue, and that it remains valid until the end of `'a`. + pub unsafe fn from_raw<'a>(ptr: *const bindings::workqueue_struct) -> &'a Queue { + // SAFETY: The `Queue` type is `#[repr(transparent)]`, so the pointer cast is valid. The + // caller promises that the pointer is not dangling. + unsafe { &*(ptr as *const Queue) } + } + + /// Enqueues a work item. + /// + /// This may fail if the work item is already enqueued in a workqueue. + /// + /// The work item will be submitted using `WORK_CPU_UNBOUND`. + pub fn enqueue<W, const ID: u64>(&self, w: W) -> W::EnqueueOutput + where + W: RawWorkItem<ID> + Send + 'static, + { + let queue_ptr = self.0.get(); + + // SAFETY: We only return `false` if the `work_struct` is already in a workqueue. The other + // `__enqueue` requirements are not relevant since `W` is `Send` and static. + // + // The call to `bindings::queue_work_on` will dereference the provided raw pointer, which + // is ok because `__enqueue` guarantees that the pointer is valid for the duration of this + // closure. + // + // Furthermore, if the C workqueue code accesses the pointer after this call to + // `__enqueue`, then the work item was successfully enqueued, and `bindings::queue_work_on` + // will have returned true. In this case, `__enqueue` promises that the raw pointer will + // stay valid until we call the function pointer in the `work_struct`, so the access is ok. + unsafe { + w.__enqueue(move |work_ptr| { + bindings::queue_work_on( + bindings::wq_misc_consts_WORK_CPU_UNBOUND as _, + queue_ptr, + work_ptr, + ) + }) + } + } + + /// Tries to spawn the given function or closure as a work item. + /// + /// This method can fail because it allocates memory to store the work item. + pub fn try_spawn<T: 'static + Send + FnOnce()>( + &self, + flags: Flags, + func: T, + ) -> Result<(), AllocError> { + let init = pin_init!(ClosureWork { + work <- new_work!("Queue::try_spawn"), + func: Some(func), + }); + + self.enqueue(Box::pin_init(init, flags).map_err(|_| AllocError)?); + Ok(()) + } +} + +/// A helper type used in [`try_spawn`]. +/// +/// [`try_spawn`]: Queue::try_spawn +#[pin_data] +struct ClosureWork<T> { + #[pin] + work: Work<ClosureWork<T>>, + func: Option<T>, +} + +impl<T> ClosureWork<T> { + fn project(self: Pin<&mut Self>) -> &mut Option<T> { + // SAFETY: The `func` field is not structurally pinned. + unsafe { &mut self.get_unchecked_mut().func } + } +} + +impl<T: FnOnce()> WorkItem for ClosureWork<T> { + type Pointer = Pin<Box<Self>>; + + fn run(mut this: Pin<Box<Self>>) { + if let Some(func) = this.as_mut().project().take() { + (func)() + } + } +} + +/// A raw work item. +/// +/// This is the low-level trait that is designed for being as general as possible. +/// +/// The `ID` parameter to this trait exists so that a single type can provide multiple +/// implementations of this trait. For example, if a struct has multiple `work_struct` fields, then +/// you will implement this trait once for each field, using a different id for each field. The +/// actual value of the id is not important as long as you use different ids for different fields +/// of the same struct. (Fields of different structs need not use different ids.) +/// +/// Note that the id is used only to select the right method to call during compilation. It won't be +/// part of the final executable. +/// +/// # Safety +/// +/// Implementers must ensure that any pointers passed to a `queue_work_on` closure by [`__enqueue`] +/// remain valid for the duration specified in the guarantees section of the documentation for +/// [`__enqueue`]. +/// +/// [`__enqueue`]: RawWorkItem::__enqueue +pub unsafe trait RawWorkItem<const ID: u64> { + /// The return type of [`Queue::enqueue`]. + type EnqueueOutput; + + /// Enqueues this work item on a queue using the provided `queue_work_on` method. + /// + /// # Guarantees + /// + /// If this method calls the provided closure, then the raw pointer is guaranteed to point at a + /// valid `work_struct` for the duration of the call to the closure. If the closure returns + /// true, then it is further guaranteed that the pointer remains valid until someone calls the + /// function pointer stored in the `work_struct`. + /// + /// # Safety + /// + /// The provided closure may only return `false` if the `work_struct` is already in a workqueue. + /// + /// If the work item type is annotated with any lifetimes, then you must not call the function + /// pointer after any such lifetime expires. (Never calling the function pointer is okay.) + /// + /// If the work item type is not [`Send`], then the function pointer must be called on the same + /// thread as the call to `__enqueue`. + unsafe fn __enqueue<F>(self, queue_work_on: F) -> Self::EnqueueOutput + where + F: FnOnce(*mut bindings::work_struct) -> bool; +} + +/// Defines the method that should be called directly when a work item is executed. +/// +/// This trait is implemented by `Pin<Box<T>>` and [`Arc<T>`], and is mainly intended to be +/// implemented for smart pointer types. For your own structs, you would implement [`WorkItem`] +/// instead. The [`run`] method on this trait will usually just perform the appropriate +/// `container_of` translation and then call into the [`run`][WorkItem::run] method from the +/// [`WorkItem`] trait. +/// +/// This trait is used when the `work_struct` field is defined using the [`Work`] helper. +/// +/// # Safety +/// +/// Implementers must ensure that [`__enqueue`] uses a `work_struct` initialized with the [`run`] +/// method of this trait as the function pointer. +/// +/// [`__enqueue`]: RawWorkItem::__enqueue +/// [`run`]: WorkItemPointer::run +pub unsafe trait WorkItemPointer<const ID: u64>: RawWorkItem<ID> { + /// Run this work item. + /// + /// # Safety + /// + /// The provided `work_struct` pointer must originate from a previous call to [`__enqueue`] + /// where the `queue_work_on` closure returned true, and the pointer must still be valid. + /// + /// [`__enqueue`]: RawWorkItem::__enqueue + unsafe extern "C" fn run(ptr: *mut bindings::work_struct); +} + +/// Defines the method that should be called when this work item is executed. +/// +/// This trait is used when the `work_struct` field is defined using the [`Work`] helper. +pub trait WorkItem<const ID: u64 = 0> { + /// The pointer type that this struct is wrapped in. This will typically be `Arc<Self>` or + /// `Pin<Box<Self>>`. + type Pointer: WorkItemPointer<ID>; + + /// The method that should be called when this work item is executed. + fn run(this: Self::Pointer); +} + +/// Links for a work item. +/// +/// This struct contains a function pointer to the [`run`] function from the [`WorkItemPointer`] +/// trait, and defines the linked list pointers necessary to enqueue a work item in a workqueue. +/// +/// Wraps the kernel's C `struct work_struct`. +/// +/// This is a helper type used to associate a `work_struct` with the [`WorkItem`] that uses it. +/// +/// [`run`]: WorkItemPointer::run +#[pin_data] +#[repr(transparent)] +pub struct Work<T: ?Sized, const ID: u64 = 0> { + #[pin] + work: Opaque<bindings::work_struct>, + _inner: PhantomData<T>, +} + +// SAFETY: Kernel work items are usable from any thread. +// +// We do not need to constrain `T` since the work item does not actually contain a `T`. +unsafe impl<T: ?Sized, const ID: u64> Send for Work<T, ID> {} +// SAFETY: Kernel work items are usable from any thread. +// +// We do not need to constrain `T` since the work item does not actually contain a `T`. +unsafe impl<T: ?Sized, const ID: u64> Sync for Work<T, ID> {} + +impl<T: ?Sized, const ID: u64> Work<T, ID> { + /// Creates a new instance of [`Work`]. + #[inline] + #[allow(clippy::new_ret_no_self)] + pub fn new(name: &'static CStr, key: &'static LockClassKey) -> impl PinInit<Self> + where + T: WorkItem<ID>, + { + pin_init!(Self { + work <- Opaque::ffi_init(|slot| { + // SAFETY: The `WorkItemPointer` implementation promises that `run` can be used as + // the work item function. + unsafe { + bindings::init_work_with_key( + slot, + Some(T::Pointer::run), + false, + name.as_char_ptr(), + key.as_ptr(), + ) + } + }), + _inner: PhantomData, + }) + } + + /// Get a pointer to the inner `work_struct`. + /// + /// # Safety + /// + /// The provided pointer must not be dangling and must be properly aligned. (But the memory + /// need not be initialized.) + #[inline] + pub unsafe fn raw_get(ptr: *const Self) -> *mut bindings::work_struct { + // SAFETY: The caller promises that the pointer is aligned and not dangling. + // + // A pointer cast would also be ok due to `#[repr(transparent)]`. We use `addr_of!` so that + // the compiler does not complain that the `work` field is unused. + unsafe { Opaque::raw_get(core::ptr::addr_of!((*ptr).work)) } + } +} + +/// Declares that a type has a [`Work<T, ID>`] field. +/// +/// The intended way of using this trait is via the [`impl_has_work!`] macro. You can use the macro +/// like this: +/// +/// ```no_run +/// use kernel::workqueue::{impl_has_work, Work}; +/// +/// struct MyWorkItem { +/// work_field: Work<MyWorkItem, 1>, +/// } +/// +/// impl_has_work! { +/// impl HasWork<MyWorkItem, 1> for MyWorkItem { self.work_field } +/// } +/// ``` +/// +/// Note that since the [`Work`] type is annotated with an id, you can have several `work_struct` +/// fields by using a different id for each one. +/// +/// # Safety +/// +/// The [`OFFSET`] constant must be the offset of a field in `Self` of type [`Work<T, ID>`]. The +/// methods on this trait must have exactly the behavior that the definitions given below have. +/// +/// [`impl_has_work!`]: crate::impl_has_work +/// [`OFFSET`]: HasWork::OFFSET +pub unsafe trait HasWork<T, const ID: u64 = 0> { + /// The offset of the [`Work<T, ID>`] field. + const OFFSET: usize; + + /// Returns the offset of the [`Work<T, ID>`] field. + /// + /// This method exists because the [`OFFSET`] constant cannot be accessed if the type is not + /// [`Sized`]. + /// + /// [`OFFSET`]: HasWork::OFFSET + #[inline] + fn get_work_offset(&self) -> usize { + Self::OFFSET + } + + /// Returns a pointer to the [`Work<T, ID>`] field. + /// + /// # Safety + /// + /// The provided pointer must point at a valid struct of type `Self`. + #[inline] + unsafe fn raw_get_work(ptr: *mut Self) -> *mut Work<T, ID> { + // SAFETY: The caller promises that the pointer is valid. + unsafe { (ptr as *mut u8).add(Self::OFFSET) as *mut Work<T, ID> } + } + + /// Returns a pointer to the struct containing the [`Work<T, ID>`] field. + /// + /// # Safety + /// + /// The pointer must point at a [`Work<T, ID>`] field in a struct of type `Self`. + #[inline] + unsafe fn work_container_of(ptr: *mut Work<T, ID>) -> *mut Self + where + Self: Sized, + { + // SAFETY: The caller promises that the pointer points at a field of the right type in the + // right kind of struct. + unsafe { (ptr as *mut u8).sub(Self::OFFSET) as *mut Self } + } +} + +/// Used to safely implement the [`HasWork<T, ID>`] trait. +/// +/// # Examples +/// +/// ``` +/// use kernel::sync::Arc; +/// use kernel::workqueue::{self, impl_has_work, Work}; +/// +/// struct MyStruct { +/// work_field: Work<MyStruct, 17>, +/// } +/// +/// impl_has_work! { +/// impl HasWork<MyStruct, 17> for MyStruct { self.work_field } +/// } +/// ``` +#[macro_export] +macro_rules! impl_has_work { + ($(impl$(<$($implarg:ident),*>)? + HasWork<$work_type:ty $(, $id:tt)?> + for $self:ident $(<$($selfarg:ident),*>)? + { self.$field:ident } + )*) => {$( + // SAFETY: The implementation of `raw_get_work` only compiles if the field has the right + // type. + unsafe impl$(<$($implarg),*>)? $crate::workqueue::HasWork<$work_type $(, $id)?> for $self $(<$($selfarg),*>)? { + const OFFSET: usize = ::core::mem::offset_of!(Self, $field) as usize; + + #[inline] + unsafe fn raw_get_work(ptr: *mut Self) -> *mut $crate::workqueue::Work<$work_type $(, $id)?> { + // SAFETY: The caller promises that the pointer is not dangling. + unsafe { + ::core::ptr::addr_of_mut!((*ptr).$field) + } + } + } + )*}; +} +pub use impl_has_work; + +impl_has_work! { + impl<T> HasWork<Self> for ClosureWork<T> { self.work } +} + +unsafe impl<T, const ID: u64> WorkItemPointer<ID> for Arc<T> +where + T: WorkItem<ID, Pointer = Self>, + T: HasWork<T, ID>, +{ + unsafe extern "C" fn run(ptr: *mut bindings::work_struct) { + // SAFETY: The `__enqueue` method always uses a `work_struct` stored in a `Work<T, ID>`. + let ptr = ptr as *mut Work<T, ID>; + // SAFETY: This computes the pointer that `__enqueue` got from `Arc::into_raw`. + let ptr = unsafe { T::work_container_of(ptr) }; + // SAFETY: This pointer comes from `Arc::into_raw` and we've been given back ownership. + let arc = unsafe { Arc::from_raw(ptr) }; + + T::run(arc) + } +} + +unsafe impl<T, const ID: u64> RawWorkItem<ID> for Arc<T> +where + T: WorkItem<ID, Pointer = Self>, + T: HasWork<T, ID>, +{ + type EnqueueOutput = Result<(), Self>; + + unsafe fn __enqueue<F>(self, queue_work_on: F) -> Self::EnqueueOutput + where + F: FnOnce(*mut bindings::work_struct) -> bool, + { + // Casting between const and mut is not a problem as long as the pointer is a raw pointer. + let ptr = Arc::into_raw(self).cast_mut(); + + // SAFETY: Pointers into an `Arc` point at a valid value. + let work_ptr = unsafe { T::raw_get_work(ptr) }; + // SAFETY: `raw_get_work` returns a pointer to a valid value. + let work_ptr = unsafe { Work::raw_get(work_ptr) }; + + if queue_work_on(work_ptr) { + Ok(()) + } else { + // SAFETY: The work queue has not taken ownership of the pointer. + Err(unsafe { Arc::from_raw(ptr) }) + } + } +} + +unsafe impl<T, const ID: u64> WorkItemPointer<ID> for Pin<Box<T>> +where + T: WorkItem<ID, Pointer = Self>, + T: HasWork<T, ID>, +{ + unsafe extern "C" fn run(ptr: *mut bindings::work_struct) { + // SAFETY: The `__enqueue` method always uses a `work_struct` stored in a `Work<T, ID>`. + let ptr = ptr as *mut Work<T, ID>; + // SAFETY: This computes the pointer that `__enqueue` got from `Arc::into_raw`. + let ptr = unsafe { T::work_container_of(ptr) }; + // SAFETY: This pointer comes from `Arc::into_raw` and we've been given back ownership. + let boxed = unsafe { Box::from_raw(ptr) }; + // SAFETY: The box was already pinned when it was enqueued. + let pinned = unsafe { Pin::new_unchecked(boxed) }; + + T::run(pinned) + } +} + +unsafe impl<T, const ID: u64> RawWorkItem<ID> for Pin<Box<T>> +where + T: WorkItem<ID, Pointer = Self>, + T: HasWork<T, ID>, +{ + type EnqueueOutput = (); + + unsafe fn __enqueue<F>(self, queue_work_on: F) -> Self::EnqueueOutput + where + F: FnOnce(*mut bindings::work_struct) -> bool, + { + // SAFETY: We're not going to move `self` or any of its fields, so its okay to temporarily + // remove the `Pin` wrapper. + let boxed = unsafe { Pin::into_inner_unchecked(self) }; + let ptr = Box::into_raw(boxed); + + // SAFETY: Pointers into a `Box` point at a valid value. + let work_ptr = unsafe { T::raw_get_work(ptr) }; + // SAFETY: `raw_get_work` returns a pointer to a valid value. + let work_ptr = unsafe { Work::raw_get(work_ptr) }; + + if !queue_work_on(work_ptr) { + // SAFETY: This method requires exclusive ownership of the box, so it cannot be in a + // workqueue. + unsafe { ::core::hint::unreachable_unchecked() } + } + } +} + +/// Returns the system work queue (`system_wq`). +/// +/// It is the one used by `schedule[_delayed]_work[_on]()`. Multi-CPU multi-threaded. There are +/// users which expect relatively short queue flush time. +/// +/// Callers shouldn't queue work items which can run for too long. +pub fn system() -> &'static Queue { + // SAFETY: `system_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_wq) } +} + +/// Returns the system high-priority work queue (`system_highpri_wq`). +/// +/// It is similar to the one returned by [`system`] but for work items which require higher +/// scheduling priority. +pub fn system_highpri() -> &'static Queue { + // SAFETY: `system_highpri_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_highpri_wq) } +} + +/// Returns the system work queue for potentially long-running work items (`system_long_wq`). +/// +/// It is similar to the one returned by [`system`] but may host long running work items. Queue +/// flushing might take relatively long. +pub fn system_long() -> &'static Queue { + // SAFETY: `system_long_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_long_wq) } +} + +/// Returns the system unbound work queue (`system_unbound_wq`). +/// +/// Workers are not bound to any specific CPU, not concurrency managed, and all queued work items +/// are executed immediately as long as `max_active` limit is not reached and resources are +/// available. +pub fn system_unbound() -> &'static Queue { + // SAFETY: `system_unbound_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_unbound_wq) } +} + +/// Returns the system freezable work queue (`system_freezable_wq`). +/// +/// It is equivalent to the one returned by [`system`] except that it's freezable. +/// +/// A freezable workqueue participates in the freeze phase of the system suspend operations. Work +/// items on the workqueue are drained and no new work item starts execution until thawed. +pub fn system_freezable() -> &'static Queue { + // SAFETY: `system_freezable_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_freezable_wq) } +} + +/// Returns the system power-efficient work queue (`system_power_efficient_wq`). +/// +/// It is inclined towards saving power and is converted to "unbound" variants if the +/// `workqueue.power_efficient` kernel parameter is specified; otherwise, it is similar to the one +/// returned by [`system`]. +pub fn system_power_efficient() -> &'static Queue { + // SAFETY: `system_power_efficient_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_power_efficient_wq) } +} + +/// Returns the system freezable power-efficient work queue (`system_freezable_power_efficient_wq`). +/// +/// It is similar to the one returned by [`system_power_efficient`] except that is freezable. +/// +/// A freezable workqueue participates in the freeze phase of the system suspend operations. Work +/// items on the workqueue are drained and no new work item starts execution until thawed. +pub fn system_freezable_power_efficient() -> &'static Queue { + // SAFETY: `system_freezable_power_efficient_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_freezable_power_efficient_wq) } +} diff --git a/rust/macros/helpers.rs b/rust/macros/helpers.rs index afb0f2e3a36a..563dcd2b7ace 100644 --- a/rust/macros/helpers.rs +++ b/rust/macros/helpers.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 -use proc_macro::{token_stream, Group, Punct, Spacing, TokenStream, TokenTree}; +use proc_macro::{token_stream, Group, TokenStream, TokenTree}; pub(crate) fn try_ident(it: &mut token_stream::IntoIter) -> Option<String> { if let Some(TokenTree::Ident(ident)) = it.next() { @@ -70,8 +70,40 @@ pub(crate) fn expect_end(it: &mut token_stream::IntoIter) { } } +/// Parsed generics. +/// +/// See the field documentation for an explanation what each of the fields represents. +/// +/// # Examples +/// +/// ```rust,ignore +/// # let input = todo!(); +/// let (Generics { decl_generics, impl_generics, ty_generics }, rest) = parse_generics(input); +/// quote! { +/// struct Foo<$($decl_generics)*> { +/// // ... +/// } +/// +/// impl<$impl_generics> Foo<$ty_generics> { +/// fn foo() { +/// // ... +/// } +/// } +/// } +/// ``` pub(crate) struct Generics { + /// The generics with bounds and default values (e.g. `T: Clone, const N: usize = 0`). + /// + /// Use this on type definitions e.g. `struct Foo<$decl_generics> ...` (or `union`/`enum`). + pub(crate) decl_generics: Vec<TokenTree>, + /// The generics with bounds (e.g. `T: Clone, const N: usize`). + /// + /// Use this on `impl` blocks e.g. `impl<$impl_generics> Trait for ...`. pub(crate) impl_generics: Vec<TokenTree>, + /// The generics without bounds and without default values (e.g. `T, N`). + /// + /// Use this when you use the type that is declared with these generics e.g. + /// `Foo<$ty_generics>`. pub(crate) ty_generics: Vec<TokenTree>, } @@ -79,6 +111,8 @@ pub(crate) struct Generics { /// /// The generics are not present in the rest, but a where clause might remain. pub(crate) fn parse_generics(input: TokenStream) -> (Generics, Vec<TokenTree>) { + // The generics with bounds and default values. + let mut decl_generics = vec![]; // `impl_generics`, the declared generics with their bounds. let mut impl_generics = vec![]; // Only the names of the generics, without any bounds. @@ -90,10 +124,17 @@ pub(crate) fn parse_generics(input: TokenStream) -> (Generics, Vec<TokenTree>) { let mut toks = input.into_iter(); // If we are at the beginning of a generic parameter. let mut at_start = true; - for tt in &mut toks { + let mut skip_until_comma = false; + while let Some(tt) = toks.next() { + if nesting == 1 && matches!(&tt, TokenTree::Punct(p) if p.as_char() == '>') { + // Found the end of the generics. + break; + } else if nesting >= 1 { + decl_generics.push(tt.clone()); + } match tt.clone() { TokenTree::Punct(p) if p.as_char() == '<' => { - if nesting >= 1 { + if nesting >= 1 && !skip_until_comma { // This is inside of the generics and part of some bound. impl_generics.push(tt); } @@ -105,49 +146,70 @@ pub(crate) fn parse_generics(input: TokenStream) -> (Generics, Vec<TokenTree>) { break; } else { nesting -= 1; - if nesting >= 1 { + if nesting >= 1 && !skip_until_comma { // We are still inside of the generics and part of some bound. impl_generics.push(tt); } - if nesting == 0 { - break; - } } } - tt => { + TokenTree::Punct(p) if skip_until_comma && p.as_char() == ',' => { if nesting == 1 { - // Here depending on the token, it might be a generic variable name. - match &tt { - // Ignore const. - TokenTree::Ident(i) if i.to_string() == "const" => {} - TokenTree::Ident(_) if at_start => { - ty_generics.push(tt.clone()); - // We also already push the `,` token, this makes it easier to append - // generics. - ty_generics.push(TokenTree::Punct(Punct::new(',', Spacing::Alone))); - at_start = false; - } - TokenTree::Punct(p) if p.as_char() == ',' => at_start = true, - // Lifetimes begin with `'`. - TokenTree::Punct(p) if p.as_char() == '\'' && at_start => { - ty_generics.push(tt.clone()); - } - _ => {} - } - } - if nesting >= 1 { + impl_generics.push(tt.clone()); impl_generics.push(tt); - } else if nesting == 0 { + skip_until_comma = false; + } + } + _ if !skip_until_comma => { + match nesting { // If we haven't entered the generics yet, we still want to keep these tokens. - rest.push(tt); + 0 => rest.push(tt), + 1 => { + // Here depending on the token, it might be a generic variable name. + match tt.clone() { + TokenTree::Ident(i) if at_start && i.to_string() == "const" => { + let Some(name) = toks.next() else { + // Parsing error. + break; + }; + impl_generics.push(tt); + impl_generics.push(name.clone()); + ty_generics.push(name.clone()); + decl_generics.push(name); + at_start = false; + } + TokenTree::Ident(_) if at_start => { + impl_generics.push(tt.clone()); + ty_generics.push(tt); + at_start = false; + } + TokenTree::Punct(p) if p.as_char() == ',' => { + impl_generics.push(tt.clone()); + ty_generics.push(tt); + at_start = true; + } + // Lifetimes begin with `'`. + TokenTree::Punct(p) if p.as_char() == '\'' && at_start => { + impl_generics.push(tt.clone()); + ty_generics.push(tt); + } + // Generics can have default values, we skip these. + TokenTree::Punct(p) if p.as_char() == '=' => { + skip_until_comma = true; + } + _ => impl_generics.push(tt), + } + } + _ => impl_generics.push(tt), } } + _ => {} } } rest.extend(toks); ( Generics { impl_generics, + decl_generics, ty_generics, }, rest, diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs index 3fc74cb4ea19..520eae5fd792 100644 --- a/rust/macros/lib.rs +++ b/rust/macros/lib.rs @@ -7,9 +7,11 @@ mod quote; mod concat_idents; mod helpers; mod module; +mod paste; mod pin_data; mod pinned_drop; mod vtable; +mod zeroable; use proc_macro::TokenStream; @@ -18,7 +20,7 @@ use proc_macro::TokenStream; /// The `type` argument should be a type which implements the [`Module`] /// trait. Also accepts various forms of kernel metadata. /// -/// C header: [`include/linux/moduleparam.h`](../../../include/linux/moduleparam.h) +/// C header: [`include/linux/moduleparam.h`](srctree/include/linux/moduleparam.h) /// /// [`Module`]: ../kernel/trait.Module.html /// @@ -33,18 +35,6 @@ use proc_macro::TokenStream; /// author: "Rust for Linux Contributors", /// description: "My very own kernel module!", /// license: "GPL", -/// params: { -/// my_i32: i32 { -/// default: 42, -/// permissions: 0o000, -/// description: "Example of i32", -/// }, -/// writeable_i32: i32 { -/// default: 42, -/// permissions: 0o644, -/// description: "Example of i32", -/// }, -/// }, /// } /// /// struct MyModule; @@ -85,27 +75,49 @@ pub fn module(ts: TokenStream) -> TokenStream { /// implementation could just return `Error::EINVAL`); Linux typically use C /// `NULL` pointers to represent these functions. /// -/// This attribute is intended to close the gap. Traits can be declared and -/// implemented with the `#[vtable]` attribute, and a `HAS_*` associated constant -/// will be generated for each method in the trait, indicating if the implementor -/// has overridden a method. +/// This attribute closes that gap. A trait can be annotated with the +/// `#[vtable]` attribute. Implementers of the trait will then also have to +/// annotate the trait with `#[vtable]`. This attribute generates a `HAS_*` +/// associated constant bool for each method in the trait that is set to true if +/// the implementer has overridden the associated method. +/// +/// For a trait method to be optional, it must have a default implementation. +/// This is also the case for traits annotated with `#[vtable]`, but in this +/// case the default implementation will never be executed. The reason for this +/// is that the functions will be called through function pointers installed in +/// C side vtables. When an optional method is not implemented on a `#[vtable]` +/// trait, a NULL entry is installed in the vtable. Thus the default +/// implementation is never called. Since these traits are not designed to be +/// used on the Rust side, it should not be possible to call the default +/// implementation. This is done to ensure that we call the vtable methods +/// through the C vtable, and not through the Rust vtable. Therefore, the +/// default implementation should call `kernel::build_error`, which prevents +/// calls to this function at compile time: +/// +/// ```compile_fail +/// # use kernel::error::VTABLE_DEFAULT_ERROR; +/// kernel::build_error(VTABLE_DEFAULT_ERROR) +/// ``` +/// +/// Note that you might need to import [`kernel::error::VTABLE_DEFAULT_ERROR`]. /// -/// This attribute is not needed if all methods are required. +/// This macro should not be used when all functions are required. /// /// # Examples /// /// ```ignore +/// use kernel::error::VTABLE_DEFAULT_ERROR; /// use kernel::prelude::*; /// /// // Declares a `#[vtable]` trait /// #[vtable] /// pub trait Operations: Send + Sync + Sized { /// fn foo(&self) -> Result<()> { -/// Err(EINVAL) +/// kernel::build_error(VTABLE_DEFAULT_ERROR) /// } /// /// fn bar(&self) -> Result<()> { -/// Err(EINVAL) +/// kernel::build_error(VTABLE_DEFAULT_ERROR) /// } /// } /// @@ -123,6 +135,8 @@ pub fn module(ts: TokenStream) -> TokenStream { /// assert_eq!(<Foo as Operations>::HAS_FOO, true); /// assert_eq!(<Foo as Operations>::HAS_BAR, false); /// ``` +/// +/// [`kernel::error::VTABLE_DEFAULT_ERROR`]: ../kernel/error/constant.VTABLE_DEFAULT_ERROR.html #[proc_macro_attribute] pub fn vtable(attr: TokenStream, ts: TokenStream) -> TokenStream { vtable::vtable(attr, ts) @@ -246,3 +260,136 @@ pub fn pin_data(inner: TokenStream, item: TokenStream) -> TokenStream { pub fn pinned_drop(args: TokenStream, input: TokenStream) -> TokenStream { pinned_drop::pinned_drop(args, input) } + +/// Paste identifiers together. +/// +/// Within the `paste!` macro, identifiers inside `[<` and `>]` are concatenated together to form a +/// single identifier. +/// +/// This is similar to the [`paste`] crate, but with pasting feature limited to identifiers and +/// literals (lifetimes and documentation strings are not supported). There is a difference in +/// supported modifiers as well. +/// +/// # Example +/// +/// ```ignore +/// use kernel::macro::paste; +/// +/// macro_rules! pub_no_prefix { +/// ($prefix:ident, $($newname:ident),+) => { +/// paste! { +/// $(pub(crate) const $newname: u32 = [<$prefix $newname>];)+ +/// } +/// }; +/// } +/// +/// pub_no_prefix!( +/// binder_driver_return_protocol_, +/// BR_OK, +/// BR_ERROR, +/// BR_TRANSACTION, +/// BR_REPLY, +/// BR_DEAD_REPLY, +/// BR_TRANSACTION_COMPLETE, +/// BR_INCREFS, +/// BR_ACQUIRE, +/// BR_RELEASE, +/// BR_DECREFS, +/// BR_NOOP, +/// BR_SPAWN_LOOPER, +/// BR_DEAD_BINDER, +/// BR_CLEAR_DEATH_NOTIFICATION_DONE, +/// BR_FAILED_REPLY +/// ); +/// +/// assert_eq!(BR_OK, binder_driver_return_protocol_BR_OK); +/// ``` +/// +/// # Modifiers +/// +/// For each identifier, it is possible to attach one or multiple modifiers to +/// it. +/// +/// Currently supported modifiers are: +/// * `span`: change the span of concatenated identifier to the span of the specified token. By +/// default the span of the `[< >]` group is used. +/// * `lower`: change the identifier to lower case. +/// * `upper`: change the identifier to upper case. +/// +/// ```ignore +/// use kernel::macro::paste; +/// +/// macro_rules! pub_no_prefix { +/// ($prefix:ident, $($newname:ident),+) => { +/// kernel::macros::paste! { +/// $(pub(crate) const fn [<$newname:lower:span>]: u32 = [<$prefix $newname:span>];)+ +/// } +/// }; +/// } +/// +/// pub_no_prefix!( +/// binder_driver_return_protocol_, +/// BR_OK, +/// BR_ERROR, +/// BR_TRANSACTION, +/// BR_REPLY, +/// BR_DEAD_REPLY, +/// BR_TRANSACTION_COMPLETE, +/// BR_INCREFS, +/// BR_ACQUIRE, +/// BR_RELEASE, +/// BR_DECREFS, +/// BR_NOOP, +/// BR_SPAWN_LOOPER, +/// BR_DEAD_BINDER, +/// BR_CLEAR_DEATH_NOTIFICATION_DONE, +/// BR_FAILED_REPLY +/// ); +/// +/// assert_eq!(br_ok(), binder_driver_return_protocol_BR_OK); +/// ``` +/// +/// # Literals +/// +/// Literals can also be concatenated with other identifiers: +/// +/// ```ignore +/// macro_rules! create_numbered_fn { +/// ($name:literal, $val:literal) => { +/// kernel::macros::paste! { +/// fn [<some_ $name _fn $val>]() -> u32 { $val } +/// } +/// }; +/// } +/// +/// create_numbered_fn!("foo", 100); +/// +/// assert_eq!(some_foo_fn100(), 100) +/// ``` +/// +/// [`paste`]: https://docs.rs/paste/ +#[proc_macro] +pub fn paste(input: TokenStream) -> TokenStream { + let mut tokens = input.into_iter().collect(); + paste::expand(&mut tokens); + tokens.into_iter().collect() +} + +/// Derives the [`Zeroable`] trait for the given struct. +/// +/// This can only be used for structs where every field implements the [`Zeroable`] trait. +/// +/// # Examples +/// +/// ```rust,ignore +/// #[derive(Zeroable)] +/// pub struct DriverData { +/// id: i64, +/// buf_ptr: *mut u8, +/// len: usize, +/// } +/// ``` +#[proc_macro_derive(Zeroable)] +pub fn derive_zeroable(input: TokenStream) -> TokenStream { + zeroable::derive(input) +} diff --git a/rust/macros/module.rs b/rust/macros/module.rs index fb1244f8c2e6..acd0393b5095 100644 --- a/rust/macros/module.rs +++ b/rust/macros/module.rs @@ -199,17 +199,6 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { /// Used by the printing macros, e.g. [`info!`]. const __LOG_PREFIX: &[u8] = b\"{name}\\0\"; - /// The \"Rust loadable module\" mark, for `scripts/is_rust_module.sh`. - // - // This may be best done another way later on, e.g. as a new modinfo - // key or a new section. For the moment, keep it simple. - #[cfg(MODULE)] - #[doc(hidden)] - #[used] - static __IS_RUST_MODULE: () = (); - - static mut __MOD: Option<{type_}> = None; - // SAFETY: `__this_module` is constructed by the kernel at load time and will not be // freed until the module is unloaded. #[cfg(MODULE)] @@ -221,76 +210,132 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { kernel::ThisModule::from_ptr(core::ptr::null_mut()) }}; - // Loadable modules need to export the `{{init,cleanup}}_module` identifiers. - #[cfg(MODULE)] - #[doc(hidden)] - #[no_mangle] - pub extern \"C\" fn init_module() -> core::ffi::c_int {{ - __init() - }} - - #[cfg(MODULE)] - #[doc(hidden)] - #[no_mangle] - pub extern \"C\" fn cleanup_module() {{ - __exit() - }} + // Double nested modules, since then nobody can access the public items inside. + mod __module_init {{ + mod __module_init {{ + use super::super::{type_}; + + /// The \"Rust loadable module\" mark. + // + // This may be best done another way later on, e.g. as a new modinfo + // key or a new section. For the moment, keep it simple. + #[cfg(MODULE)] + #[doc(hidden)] + #[used] + static __IS_RUST_MODULE: () = (); + + static mut __MOD: Option<{type_}> = None; + + // Loadable modules need to export the `{{init,cleanup}}_module` identifiers. + /// # Safety + /// + /// This function must not be called after module initialization, because it may be + /// freed after that completes. + #[cfg(MODULE)] + #[doc(hidden)] + #[no_mangle] + #[link_section = \".init.text\"] + pub unsafe extern \"C\" fn init_module() -> core::ffi::c_int {{ + // SAFETY: This function is inaccessible to the outside due to the double + // module wrapping it. It is called exactly once by the C side via its + // unique name. + unsafe {{ __init() }} + }} - // Built-in modules are initialized through an initcall pointer - // and the identifiers need to be unique. - #[cfg(not(MODULE))] - #[cfg(not(CONFIG_HAVE_ARCH_PREL32_RELOCATIONS))] - #[doc(hidden)] - #[link_section = \"{initcall_section}\"] - #[used] - pub static __{name}_initcall: extern \"C\" fn() -> core::ffi::c_int = __{name}_init; + #[cfg(MODULE)] + #[doc(hidden)] + #[no_mangle] + pub extern \"C\" fn cleanup_module() {{ + // SAFETY: + // - This function is inaccessible to the outside due to the double + // module wrapping it. It is called exactly once by the C side via its + // unique name, + // - furthermore it is only called after `init_module` has returned `0` + // (which delegates to `__init`). + unsafe {{ __exit() }} + }} - #[cfg(not(MODULE))] - #[cfg(CONFIG_HAVE_ARCH_PREL32_RELOCATIONS)] - core::arch::global_asm!( - r#\".section \"{initcall_section}\", \"a\" - __{name}_initcall: - .long __{name}_init - . - .previous - \"# - ); + // Built-in modules are initialized through an initcall pointer + // and the identifiers need to be unique. + #[cfg(not(MODULE))] + #[cfg(not(CONFIG_HAVE_ARCH_PREL32_RELOCATIONS))] + #[doc(hidden)] + #[link_section = \"{initcall_section}\"] + #[used] + pub static __{name}_initcall: extern \"C\" fn() -> core::ffi::c_int = __{name}_init; + + #[cfg(not(MODULE))] + #[cfg(CONFIG_HAVE_ARCH_PREL32_RELOCATIONS)] + core::arch::global_asm!( + r#\".section \"{initcall_section}\", \"a\" + __{name}_initcall: + .long __{name}_init - . + .previous + \"# + ); + + #[cfg(not(MODULE))] + #[doc(hidden)] + #[no_mangle] + pub extern \"C\" fn __{name}_init() -> core::ffi::c_int {{ + // SAFETY: This function is inaccessible to the outside due to the double + // module wrapping it. It is called exactly once by the C side via its + // placement above in the initcall section. + unsafe {{ __init() }} + }} - #[cfg(not(MODULE))] - #[doc(hidden)] - #[no_mangle] - pub extern \"C\" fn __{name}_init() -> core::ffi::c_int {{ - __init() - }} + #[cfg(not(MODULE))] + #[doc(hidden)] + #[no_mangle] + pub extern \"C\" fn __{name}_exit() {{ + // SAFETY: + // - This function is inaccessible to the outside due to the double + // module wrapping it. It is called exactly once by the C side via its + // unique name, + // - furthermore it is only called after `__{name}_init` has returned `0` + // (which delegates to `__init`). + unsafe {{ __exit() }} + }} - #[cfg(not(MODULE))] - #[doc(hidden)] - #[no_mangle] - pub extern \"C\" fn __{name}_exit() {{ - __exit() - }} + /// # Safety + /// + /// This function must only be called once. + unsafe fn __init() -> core::ffi::c_int {{ + match <{type_} as kernel::Module>::init(&super::super::THIS_MODULE) {{ + Ok(m) => {{ + // SAFETY: No data race, since `__MOD` can only be accessed by this + // module and there only `__init` and `__exit` access it. These + // functions are only called once and `__exit` cannot be called + // before or during `__init`. + unsafe {{ + __MOD = Some(m); + }} + return 0; + }} + Err(e) => {{ + return e.to_errno(); + }} + }} + }} - fn __init() -> core::ffi::c_int {{ - match <{type_} as kernel::Module>::init(&THIS_MODULE) {{ - Ok(m) => {{ + /// # Safety + /// + /// This function must + /// - only be called once, + /// - be called after `__init` has been called and returned `0`. + unsafe fn __exit() {{ + // SAFETY: No data race, since `__MOD` can only be accessed by this module + // and there only `__init` and `__exit` access it. These functions are only + // called once and `__init` was already called. unsafe {{ - __MOD = Some(m); + // Invokes `drop()` on `__MOD`, which should be used for cleanup. + __MOD = None; }} - return 0; - }} - Err(e) => {{ - return e.to_errno(); }} - }} - }} - fn __exit() {{ - unsafe {{ - // Invokes `drop()` on `__MOD`, which should be used for cleanup. - __MOD = None; + {modinfo} }} }} - - {modinfo} ", type_ = info.type_, name = info.name, diff --git a/rust/macros/paste.rs b/rust/macros/paste.rs new file mode 100644 index 000000000000..f40d42b35b58 --- /dev/null +++ b/rust/macros/paste.rs @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0 + +use proc_macro::{Delimiter, Group, Ident, Spacing, Span, TokenTree}; + +fn concat(tokens: &[TokenTree], group_span: Span) -> TokenTree { + let mut tokens = tokens.iter(); + let mut segments = Vec::new(); + let mut span = None; + loop { + match tokens.next() { + None => break, + Some(TokenTree::Literal(lit)) => { + // Allow us to concat string literals by stripping quotes + let mut value = lit.to_string(); + if value.starts_with('"') && value.ends_with('"') { + value.remove(0); + value.pop(); + } + segments.push((value, lit.span())); + } + Some(TokenTree::Ident(ident)) => { + let mut value = ident.to_string(); + if value.starts_with("r#") { + value.replace_range(0..2, ""); + } + segments.push((value, ident.span())); + } + Some(TokenTree::Punct(p)) if p.as_char() == ':' => { + let Some(TokenTree::Ident(ident)) = tokens.next() else { + panic!("expected identifier as modifier"); + }; + + let (mut value, sp) = segments.pop().expect("expected identifier before modifier"); + match ident.to_string().as_str() { + // Set the overall span of concatenated token as current span + "span" => { + assert!( + span.is_none(), + "span modifier should only appear at most once" + ); + span = Some(sp); + } + "lower" => value = value.to_lowercase(), + "upper" => value = value.to_uppercase(), + v => panic!("unknown modifier `{v}`"), + }; + segments.push((value, sp)); + } + _ => panic!("unexpected token in paste segments"), + }; + } + + let pasted: String = segments.into_iter().map(|x| x.0).collect(); + TokenTree::Ident(Ident::new(&pasted, span.unwrap_or(group_span))) +} + +pub(crate) fn expand(tokens: &mut Vec<TokenTree>) { + for token in tokens.iter_mut() { + if let TokenTree::Group(group) = token { + let delimiter = group.delimiter(); + let span = group.span(); + let mut stream: Vec<_> = group.stream().into_iter().collect(); + // Find groups that looks like `[< A B C D >]` + if delimiter == Delimiter::Bracket + && stream.len() >= 3 + && matches!(&stream[0], TokenTree::Punct(p) if p.as_char() == '<') + && matches!(&stream[stream.len() - 1], TokenTree::Punct(p) if p.as_char() == '>') + { + // Replace the group with concatenated token + *token = concat(&stream[1..stream.len() - 1], span); + } else { + // Recursively expand tokens inside the group + expand(&mut stream); + let mut group = Group::new(delimiter, stream.into_iter().collect()); + group.set_span(span); + *token = TokenTree::Group(group); + } + } + } + + // Path segments cannot contain invisible delimiter group, so remove them if any. + for i in (0..tokens.len().saturating_sub(3)).rev() { + // Looking for a double colon + if matches!( + (&tokens[i + 1], &tokens[i + 2]), + (TokenTree::Punct(a), TokenTree::Punct(b)) + if a.as_char() == ':' && a.spacing() == Spacing::Joint && b.as_char() == ':' + ) { + match &tokens[i + 3] { + TokenTree::Group(group) if group.delimiter() == Delimiter::None => { + tokens.splice(i + 3..i + 4, group.stream()); + } + _ => (), + } + + match &tokens[i] { + TokenTree::Group(group) if group.delimiter() == Delimiter::None => { + tokens.splice(i..i + 1, group.stream()); + } + _ => (), + } + } + } +} diff --git a/rust/macros/pin_data.rs b/rust/macros/pin_data.rs index 6d58cfda9872..1d4a3547c684 100644 --- a/rust/macros/pin_data.rs +++ b/rust/macros/pin_data.rs @@ -10,6 +10,7 @@ pub(crate) fn pin_data(args: TokenStream, input: TokenStream) -> TokenStream { let ( Generics { impl_generics, + decl_generics, ty_generics, }, rest, @@ -76,6 +77,7 @@ pub(crate) fn pin_data(args: TokenStream, input: TokenStream) -> TokenStream { @sig(#(#rest)*), @impl_generics(#(#impl_generics)*), @ty_generics(#(#ty_generics)*), + @decl_generics(#(#decl_generics)*), @body(#last), }); quoted.extend(errs); diff --git a/rust/macros/quote.rs b/rust/macros/quote.rs index dddbb4e6f4cb..33a199e4f176 100644 --- a/rust/macros/quote.rs +++ b/rust/macros/quote.rs @@ -124,6 +124,18 @@ macro_rules! quote_spanned { )); quote_spanned!(@proc $v $span $($tt)*); }; + (@proc $v:ident $span:ident ; $($tt:tt)*) => { + $v.push(::proc_macro::TokenTree::Punct( + ::proc_macro::Punct::new(';', ::proc_macro::Spacing::Alone) + )); + quote_spanned!(@proc $v $span $($tt)*); + }; + (@proc $v:ident $span:ident + $($tt:tt)*) => { + $v.push(::proc_macro::TokenTree::Punct( + ::proc_macro::Punct::new('+', ::proc_macro::Spacing::Alone) + )); + quote_spanned!(@proc $v $span $($tt)*); + }; (@proc $v:ident $span:ident $id:ident $($tt:tt)*) => { $v.push(::proc_macro::TokenTree::Ident(::proc_macro::Ident::new(stringify!($id), $span))); quote_spanned!(@proc $v $span $($tt)*); diff --git a/rust/macros/zeroable.rs b/rust/macros/zeroable.rs new file mode 100644 index 000000000000..cfee2cec18d5 --- /dev/null +++ b/rust/macros/zeroable.rs @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0 + +use crate::helpers::{parse_generics, Generics}; +use proc_macro::{TokenStream, TokenTree}; + +pub(crate) fn derive(input: TokenStream) -> TokenStream { + let ( + Generics { + impl_generics, + decl_generics: _, + ty_generics, + }, + mut rest, + ) = parse_generics(input); + // This should be the body of the struct `{...}`. + let last = rest.pop(); + // Now we insert `Zeroable` as a bound for every generic parameter in `impl_generics`. + let mut new_impl_generics = Vec::with_capacity(impl_generics.len()); + // Are we inside of a generic where we want to add `Zeroable`? + let mut in_generic = !impl_generics.is_empty(); + // Have we already inserted `Zeroable`? + let mut inserted = false; + // Level of `<>` nestings. + let mut nested = 0; + for tt in impl_generics { + match &tt { + // If we find a `,`, then we have finished a generic/constant/lifetime parameter. + TokenTree::Punct(p) if nested == 0 && p.as_char() == ',' => { + if in_generic && !inserted { + new_impl_generics.extend(quote! { : ::kernel::init::Zeroable }); + } + in_generic = true; + inserted = false; + new_impl_generics.push(tt); + } + // If we find `'`, then we are entering a lifetime. + TokenTree::Punct(p) if nested == 0 && p.as_char() == '\'' => { + in_generic = false; + new_impl_generics.push(tt); + } + TokenTree::Punct(p) if nested == 0 && p.as_char() == ':' => { + new_impl_generics.push(tt); + if in_generic { + new_impl_generics.extend(quote! { ::kernel::init::Zeroable + }); + inserted = true; + } + } + TokenTree::Punct(p) if p.as_char() == '<' => { + nested += 1; + new_impl_generics.push(tt); + } + TokenTree::Punct(p) if p.as_char() == '>' => { + assert!(nested > 0); + nested -= 1; + new_impl_generics.push(tt); + } + _ => new_impl_generics.push(tt), + } + } + assert_eq!(nested, 0); + if in_generic && !inserted { + new_impl_generics.extend(quote! { : ::kernel::init::Zeroable }); + } + quote! { + ::kernel::__derive_zeroable!( + parse_input: + @sig(#(#rest)*), + @impl_generics(#(#new_impl_generics)*), + @ty_generics(#(#ty_generics)*), + @body(#last), + ); + } +} diff --git a/rust/uapi/uapi_helper.h b/rust/uapi/uapi_helper.h index 301f5207f023..08f5e9334c9e 100644 --- a/rust/uapi/uapi_helper.h +++ b/rust/uapi/uapi_helper.h @@ -7,3 +7,5 @@ */ #include <uapi/asm-generic/ioctl.h> +#include <uapi/linux/mii.h> +#include <uapi/linux/ethtool.h> |