Merge pull request #218 from soby-mathew/sm/add_cpu_ops_warning

Add level specific cache operations and changes to errata workaround mechanism

10 files changed, 240 insertions, 47 deletions

diff --git a/Makefile b/Makefile
index af2ddd4e..9b15c213 100644
--- a/Makefile
+++ b/Makefile
@@ -138,9 +138,10 @@ msg_start:
 
 include plat/${PLAT}/platform.mk
 
-# By default all CPU errata workarounds are disabled. This can be
+# Include the CPU specific operations makefile. By default all CPU errata
+# workarounds and CPU specifc optimisations are disabled. This can be
 # overridden by the platform.
-include lib/cpus/cpu-errata.mk
+include lib/cpus/cpu-ops.mk
 
 ifdef BL1_SOURCES
 NEED_BL1 := yes
diff --git a/docs/cpu-errata-workarounds.md b/docs/cpu-specific-build-macros.md
index 8285d89d..246381a8 100644
--- a/docs/cpu-errata-workarounds.md
+++ b/docs/cpu-specific-build-macros.md
@@ -1,11 +1,28 @@
-ARM CPU Errata Workarounds
-==========================
+ARM CPU Specific Build Macros
+=============================
+
+Contents
+--------
+
+1.  Introduction
+2.  CPU Errata Workarounds
+3.  CPU Specific optimizations
+
+1.  Introduction
+----------------
+
+This document describes the various build options present in the CPU specific
+operations framework to enable errata workarounds and to enable optimizations
+for a specific CPU on a platform.
+
+2.  CPU Errata Workarounds
+--------------------------
 
 ARM Trusted Firmware exports a series of build flags which control the
 errata workarounds that are applied to each CPU by the reset handler. The
 errata details can be found in the CPU specifc errata documents published
 by ARM. The errata workarounds are implemented for a particular revision
-or a set of processor revisions. This check is done in the debug build.
+or a set of processor revisions. This is checked by reset handler at runtime.
 Each errata workaround is identified by its `ID` as specified in the processor's
 errata notice document. The format of the define used to enable/disable the
 errata is `ERRATA_<Processor name>_<ID>` where the `Processor name`
@@ -13,7 +30,10 @@ is either `A57` for the `Cortex_A57` CPU or `A53` for `Cortex_A53` CPU.
 
 All workarounds are disabled by default. The platform is reponsible for
 enabling these workarounds according to its requirement by defining the
-errata workaround build flags in the platform specific makefile.
+errata workaround build flags in the platform specific makefile. In case
+these workarounds are enabled for the wrong CPU revision then the errata
+workaround is not applied. In the DEBUG build, this is indicated by
+printing a warning to the crash console.
 
 In the current implementation, a platform which has more than 1 variant
 with different revisions of a processor has no runtime mechanism available
@@ -22,14 +42,28 @@ for it to specify which errata workarounds should be enabled or not.
 The value of the build flags are 0 by default, that is, disabled. Any other
 value will enable it.
 
-For Cortex A57, following errata build flags are defined :
+For Cortex-A57, following errata build flags are defined :
 
-*   `ERRATA_A57_806969`: This applies errata 806969 workaround to cortex a57
+*   `ERRATA_A57_806969`: This applies errata 806969 workaround to Cortex-A57
      CPU. This needs to be enabled only for revision r0p0 of the CPU.
 
-*   `ERRATA_A57_813420`: This applies errata 813420 workaround to cortex a57
+*   `ERRATA_A57_813420`: This applies errata 813420 workaround to Cortex-A57
      CPU. This needs to be enabled only for revision r0p0 of the CPU.
 
+3.  CPU Specific optimizations
+------------------------------
+
+This section describes some of the optimizations allowed by the CPU micro
+architecture that can be enabled by the platform as desired.
+
+*    `SKIP_A57_L1_FLUSH_PWR_DWN`: This flag enables an optimization in the
+     Cortex-A57 cluster power down sequence by not flushing the Level 1 data
+     cache. The L1 data cache and the L2 unified cache are inclusive. A flush
+     of the L2 by set/way flushes any dirty lines from the L1 as well. This
+     is a known safe deviation from the Cortex-A57 TRM defined power down
+     sequence. Each Cortex-A57 based platform must make its own decision on
+     whether to use the optimization.
+
 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 
 _Copyright (c) 2014, ARM Limited and Contributors. All rights reserved._
diff --git a/docs/firmware-design.md b/docs/firmware-design.md
index fde61da7..bc075da3 100644
--- a/docs/firmware-design.md
+++ b/docs/firmware-design.md
@@ -982,9 +982,10 @@ Please note that only 2. is mandated by the TRM.
 
 The CPU specific operations framework scales to accommodate a large number of
 different CPUs during power down and reset handling. The platform can specify
+any CPU optimization it wants to enable for each CPU. It can also specify
 the CPU errata workarounds to be applied for each CPU type during reset
 handling by defining CPU errata compile time macros. Details on these macros
-can be found in the [cpu-errata-workarounds.md][ERRW] file.
+can be found in the [cpu-specific-build-macros.md][CPUBM] file.
 
 The CPU specific operations framework depends on the `cpu_ops` structure which
 needs to be exported for each type of CPU in the platform. It is defined in
@@ -1485,4 +1486,4 @@ _Copyright (c) 2013-2014, ARM Limited and Contributors. All rights reserved._
 [User Guide]:       ./user-guide.md
 [Porting Guide]:    ./porting-guide.md
 [INTRG]:            ./interrupt-framework-design.md
-[ERRW]:             ./cpu-errata-workarounds.md
+[CPUBM]:            ./cpu-specific-build-macros.md.md
diff --git a/include/lib/aarch64/arch.h b/include/lib/aarch64/arch.h
index e5b2bf8c..476c9c5c 100644
--- a/include/lib/aarch64/arch.h
+++ b/include/lib/aarch64/arch.h
@@ -38,7 +38,9 @@
 #define MIDR_IMPL_MASK		0xff
 #define MIDR_IMPL_SHIFT		0x18
 #define MIDR_VAR_SHIFT		20
+#define MIDR_VAR_BITS		4
 #define MIDR_REV_SHIFT		0
+#define MIDR_REV_BITS		4
 #define MIDR_PN_MASK		0xfff
 #define MIDR_PN_SHIFT		0x4
 
diff --git a/lib/aarch64/cache_helpers.S b/lib/aarch64/cache_helpers.S
index 1c805504..dc601021 100644
--- a/lib/aarch64/cache_helpers.S
+++ b/lib/aarch64/cache_helpers.S
@@ -35,6 +35,9 @@
 	.globl	inv_dcache_range
 	.globl	dcsw_op_louis
 	.globl	dcsw_op_all
+	.globl	dcsw_op_level1
+	.globl	dcsw_op_level2
+	.globl	dcsw_op_level3
 
 	/* ------------------------------------------
 	 * Clean+Invalidate from base address till
@@ -81,6 +84,7 @@ inv_loop:
 	 * x0: The operation type (0-2), as defined in arch.h
 	 * x3: The last cache level to operate on
 	 * x9: clidr_el1
+	 * x10: The cache level to begin operation from
 	 * and will carry out the operation on each data cache from level 0
 	 * to the level in x3 in sequence
 	 *
@@ -93,12 +97,12 @@ inv_loop:
 	mrs	x9, clidr_el1
 	ubfx	x3, x9, \shift, \fw
 	lsl	x3, x3, \ls
+	mov	x10, xzr
 	b	do_dcsw_op
 	.endm
 
 func do_dcsw_op
 	cbz	x3, exit
-	mov	x10, xzr
 	adr	x14, dcsw_loop_table	// compute inner loop address
 	add	x14, x14, x0, lsl #5	// inner loop is 8x32-bit instructions
 	mov	x0, x9
@@ -163,3 +167,45 @@ func dcsw_op_louis
 
 func dcsw_op_all
 	dcsw_op #LOC_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT
+
+	/* ---------------------------------------------------------------
+	 *  Helper macro for data cache operations by set/way for the
+	 *  level specified
+	 * ---------------------------------------------------------------
+	 */
+	.macro dcsw_op_level level
+	mrs	x9, clidr_el1
+	mov	x3, \level
+	sub	x10, x3, #2
+	b	do_dcsw_op
+	.endm
+
+	/* ---------------------------------------------------------------
+	 * Data cache operations by set/way for level 1 cache
+	 *
+	 * The main function, do_dcsw_op requires:
+	 * x0: The operation type (0-2), as defined in arch.h
+	 * ---------------------------------------------------------------
+	 */
+func dcsw_op_level1
+	dcsw_op_level #(1 << LEVEL_SHIFT)
+
+	/* ---------------------------------------------------------------
+	 * Data cache operations by set/way for level 2 cache
+	 *
+	 * The main function, do_dcsw_op requires:
+	 * x0: The operation type (0-2), as defined in arch.h
+	 * ---------------------------------------------------------------
+	 */
+func dcsw_op_level2
+	dcsw_op_level #(2 << LEVEL_SHIFT)
+
+	/* ---------------------------------------------------------------
+	 * Data cache operations by set/way for level 3 cache
+	 *
+	 * The main function, do_dcsw_op requires:
+	 * x0: The operation type (0-2), as defined in arch.h
+	 * ---------------------------------------------------------------
+	 */
+func dcsw_op_level3
+	dcsw_op_level #(3 << LEVEL_SHIFT)
diff --git a/lib/cpus/aarch64/cortex_a53.S b/lib/cpus/aarch64/cortex_a53.S
index 722ce7af..ec184641 100644
--- a/lib/cpus/aarch64/cortex_a53.S
+++ b/lib/cpus/aarch64/cortex_a53.S
@@ -77,11 +77,11 @@ func cortex_a53_core_pwr_dwn
 	bl	cortex_a53_disable_dcache
 
 	/* ---------------------------------------------
-	 * Flush L1 cache to PoU.
+	 * Flush L1 caches.
 	 * ---------------------------------------------
 	 */
 	mov	x0, #DCCISW
-	bl	dcsw_op_louis
+	bl	dcsw_op_level1
 
 	/* ---------------------------------------------
 	 * Come out of intra cluster coherency
@@ -100,17 +100,24 @@ func cortex_a53_cluster_pwr_dwn
 	bl	cortex_a53_disable_dcache
 
 	/* ---------------------------------------------
+	 * Flush L1 caches.
+	 * ---------------------------------------------
+	 */
+	mov	x0, #DCCISW
+	bl	dcsw_op_level1
+
+	/* ---------------------------------------------
 	 * Disable the optional ACP.
 	 * ---------------------------------------------
 	 */
 	bl	plat_disable_acp
 
 	/* ---------------------------------------------
-	 * Flush L1 and L2 caches to PoC.
+	 * Flush L2 caches.
 	 * ---------------------------------------------
 	 */
 	mov	x0, #DCCISW
-	bl	dcsw_op_all
+	bl	dcsw_op_level2
 
 	/* ---------------------------------------------
 	 * Come out of intra cluster coherency
diff --git a/lib/cpus/aarch64/cortex_a57.S b/lib/cpus/aarch64/cortex_a57.S
index eed1bbb9..dab16d7e 100644
--- a/lib/cpus/aarch64/cortex_a57.S
+++ b/lib/cpus/aarch64/cortex_a57.S
@@ -57,7 +57,7 @@ func cortex_a57_disable_l2_prefetch
 	bic	x0, x0, x1
 	msr	CPUECTLR_EL1, x0
 	isb
-	dsb	sy
+	dsb	ish
 	ret
 
 	/* ---------------------------------------------
@@ -81,33 +81,79 @@ func cortex_a57_disable_ext_debug
 	dsb	sy
 	ret
 
-func cortex_a57_reset_func
-#if ERRATA_A57_806969 || ERRATA_A57_813420
-	/* ---------------------------------------------
-	 * Ensure that the following errata is only
-	 * applied on r0p0 parts.
-	 * ---------------------------------------------
+	/* --------------------------------------------------
+	 * Errata Workaround for Cortex A57 Errata #806969.
+	 * This applies only to revision r0p0 of Cortex A57.
+	 * Inputs:
+	 * x0: variant[4:7] and revision[0:3] of current cpu.
+	 * --------------------------------------------------
 	 */
-#if ASM_ASSERTION
-	mrs	x0, midr_el1
-	ubfx	x1, x0, #MIDR_VAR_SHIFT, #4
-	ubfx	x2, x0, #MIDR_REV_SHIFT, #4
-	orr	x0, x1, x2
-	cmp	x0, #0
-	ASM_ASSERT(eq)
+func errata_a57_806969_wa
+	/*
+	 * Compare x0 against revision r0p0
+	 */
+	cbz	x0, apply_806969
+#if DEBUG
+	b	print_revision_warning
+#else
+	ret
 #endif
-	mov	x1, xzr
-#if ERRATA_A57_806969
+apply_806969:
+	mrs	x1, CPUACTLR_EL1
 	orr	x1, x1, #CPUACTLR_NO_ALLOC_WBWA
+	msr	CPUACTLR_EL1, x1
+	ret
+
+
+	/* ---------------------------------------------------
+	 * Errata Workaround for Cortex A57 Errata #813420.
+	 * This applies only to revision r0p0 of Cortex A57.
+	 * Inputs:
+	 * x0: variant[4:7] and revision[0:3] of current cpu.
+	 * ---------------------------------------------------
+	 */
+func errata_a57_813420_wa
+	/*
+	 * Compare x0 against revision r0p0
+	 */
+	cbz	x0, apply_813420
+#if DEBUG
+	b	print_revision_warning
+#else
+	ret
 #endif
-#if ERRATA_A57_813420
+apply_813420:
+	mrs	x1, CPUACTLR_EL1
 	orr	x1, x1, #CPUACTLR_DCC_AS_DCCI
-#endif
-	mrs	x0, CPUACTLR_EL1
-	orr	x0, x0, x1
-	msr	CPUACTLR_EL1, x0
+	msr	CPUACTLR_EL1, x1
+	ret
+
+	/* -------------------------------------------------
+	 * The CPU Ops reset function for Cortex-A57.
+	 * -------------------------------------------------
+	 */
+func cortex_a57_reset_func
+	mov	x19, x30
+	mrs	x0, midr_el1
+
+	/*
+	 * Extract the variant[20:23] and revision[0:3] from x0
+	 * and pack it in x20[0:7] as variant[4:7] and revision[0:3].
+	 * First extract x0[16:23] to x20[0:7] and zero fill the rest.
+	 * Then extract x0[0:3] into x20[0:3] retaining other bits.
+	 */
+	ubfx	x20, x0, #(MIDR_VAR_SHIFT - MIDR_REV_BITS), #(MIDR_REV_BITS + MIDR_VAR_BITS)
+	bfxil	x20, x0, #MIDR_REV_SHIFT, #MIDR_REV_BITS
+
+#if ERRATA_A57_806969
+	mov	x0, x20
+	bl	errata_a57_806969_wa
 #endif
 
+#if ERRATA_A57_813420
+	mov	x0, x20
+	bl	errata_a57_813420_wa
+#endif
 	/* ---------------------------------------------
 	 * As a bare minimum enable the SMP bit.
 	 * ---------------------------------------------
@@ -116,8 +162,12 @@ func cortex_a57_reset_func
 	orr	x0, x0, #CPUECTLR_SMP_BIT
 	msr	CPUECTLR_EL1, x0
 	isb
-	ret
+	ret	x19
 
+	/* ----------------------------------------------------
+	 * The CPU Ops core power down function for Cortex-A57.
+	 * ----------------------------------------------------
+	 */
 func cortex_a57_core_pwr_dwn
 	mov	x18, x30
 
@@ -134,11 +184,11 @@ func cortex_a57_core_pwr_dwn
 	bl	cortex_a57_disable_l2_prefetch
 
 	/* ---------------------------------------------
-	 * Flush L1 cache to PoU.
+	 * Flush L1 caches.
 	 * ---------------------------------------------
 	 */
 	mov	x0, #DCCISW
-	bl	dcsw_op_louis
+	bl	dcsw_op_level1
 
 	/* ---------------------------------------------
 	 * Come out of intra cluster coherency
@@ -153,6 +203,10 @@ func cortex_a57_core_pwr_dwn
 	mov	x30, x18
 	b	cortex_a57_disable_ext_debug
 
+	/* -------------------------------------------------------
+	 * The CPU Ops cluster power down function for Cortex-A57.
+	 * -------------------------------------------------------
+	 */
 func cortex_a57_cluster_pwr_dwn
 	mov	x18, x30
 
@@ -168,18 +222,26 @@ func cortex_a57_cluster_pwr_dwn
 	 */
 	bl	cortex_a57_disable_l2_prefetch
 
+#if !SKIP_A57_L1_FLUSH_PWR_DWN
+	/* -------------------------------------------------
+	 * Flush the L1 caches.
+	 * -------------------------------------------------
+	 */
+	mov	x0, #DCCISW
+	bl	dcsw_op_level1
+#endif
 	/* ---------------------------------------------
 	 * Disable the optional ACP.
 	 * ---------------------------------------------
 	 */
 	bl	plat_disable_acp
 
-	/* ---------------------------------------------
-	 * Flush L1 and L2 caches to PoC.
-	 * ---------------------------------------------
+	/* -------------------------------------------------
+	 * Flush the L2 caches.
+	 * -------------------------------------------------
 	 */
 	mov	x0, #DCCISW
-	bl	dcsw_op_all
+	bl	dcsw_op_level2
 
 	/* ---------------------------------------------
 	 * Come out of intra cluster coherency
diff --git a/lib/cpus/aarch64/cpu_helpers.S b/lib/cpus/aarch64/cpu_helpers.S
index 46584b3f..f053d44f 100644
--- a/lib/cpus/aarch64/cpu_helpers.S
+++ b/lib/cpus/aarch64/cpu_helpers.S
@@ -45,7 +45,7 @@
 	 */
 	.globl	reset_handler
 func reset_handler
-	mov	x10, x30
+	mov	x19, x30
 
 	bl	plat_reset_handler
 
@@ -58,10 +58,11 @@ func reset_handler
 
 	/* Get the cpu_ops reset handler */
 	ldr	x2, [x0, #CPU_RESET_FUNC]
+	mov	x30, x19
 	cbz	x2, 1f
-	blr	x2
+	br	x2
 1:
-	ret	x10
+	ret
 
 #endif /* IMAGE_BL1 || (IMAGE_BL31 && RESET_TO_BL31) */
 
@@ -191,3 +192,29 @@ func get_cpu_ops_ptr
 	sub	x0, x4, #(CPU_OPS_SIZE + CPU_MIDR)
 error_exit:
 	ret
+
+#if DEBUG
+	/*
+	 * This function prints a warning message to the crash console
+	 * if the CPU revision/part number does not match the errata
+	 * workaround enabled in the build.
+	 * Clobber: x30, x0 - x5
+	 */
+.section .rodata.rev_warn_str, "aS"
+rev_warn_str:
+	.asciz "Warning: Skipping Errata workaround for non matching CPU revision number.\n"
+
+	.globl	print_revision_warning
+func print_revision_warning
+	mov	x5, x30
+	/* Ensure the console is initialized */
+	bl	plat_crash_console_init
+	/* Check if the console is initialized */
+	cbz	x0, 1f
+	/* The console is initialized */
+	adr	x4, rev_warn_str
+	bl	asm_print_str
+1:
+	ret	x5
+#endif
+
diff --git a/lib/cpus/cpu-errata.mk b/lib/cpus/cpu-ops.mk
index 79f0156d..1c5512e9 100644
--- a/lib/cpus/cpu-errata.mk
+++ b/lib/cpus/cpu-ops.mk
@@ -28,6 +28,15 @@
 # POSSIBILITY OF SUCH DAMAGE.
 #
 
+# Cortex A57 specific optimisation to skip L1 cache flush when
+# cluster is powered down.
+SKIP_A57_L1_FLUSH_PWR_DWN	?=0
+
+# Process SKIP_A57_L1_FLUSH_PWR_DWN flag
+$(eval $(call assert_boolean,SKIP_A57_L1_FLUSH_PWR_DWN))
+$(eval $(call add_define,SKIP_A57_L1_FLUSH_PWR_DWN))
+
+
 # CPU Errata Build flags. These should be enabled by the
 # platform if the errata needs to be applied.
 
diff --git a/plat/juno/platform.mk b/plat/juno/platform.mk
index 0637ef3d..28004381 100644
--- a/plat/juno/platform.mk
+++ b/plat/juno/platform.mk
@@ -99,3 +99,7 @@ NEED_BL30		:=	yes
 # Enable workarounds for selected Cortex-A57 erratas.
 ERRATA_A57_806969	:=	1
 ERRATA_A57_813420	:=	1
+
+# Enable option to skip L1 data cache flush during the Cortex-A57 cluster
+# power down sequence
+SKIP_A57_L1_FLUSH_PWR_DWN	:=	 1