1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
|
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* SHA-3 optimized using the CP Assist for Cryptographic Functions (CPACF)
*
* Copyright 2025 Google LLC
*/
#include <asm/cpacf.h>
#include <linux/cpufeature.h>
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha3);
static void sha3_absorb_blocks(struct sha3_state *state, const u8 *data,
size_t nblocks, size_t block_size)
{
if (static_branch_likely(&have_sha3)) {
/*
* Note that KIMD assumes little-endian order of the state
* words. sha3_state already uses that order, though, so
* there's no need for a byteswap.
*/
switch (block_size) {
case SHA3_224_BLOCK_SIZE:
cpacf_kimd(CPACF_KIMD_SHA3_224, state,
data, nblocks * block_size);
return;
case SHA3_256_BLOCK_SIZE:
/*
* This case handles both SHA3-256 and SHAKE256, since
* they have the same block size.
*/
cpacf_kimd(CPACF_KIMD_SHA3_256, state,
data, nblocks * block_size);
return;
case SHA3_384_BLOCK_SIZE:
cpacf_kimd(CPACF_KIMD_SHA3_384, state,
data, nblocks * block_size);
return;
case SHA3_512_BLOCK_SIZE:
cpacf_kimd(CPACF_KIMD_SHA3_512, state,
data, nblocks * block_size);
return;
}
}
sha3_absorb_blocks_generic(state, data, nblocks, block_size);
}
static void sha3_keccakf(struct sha3_state *state)
{
if (static_branch_likely(&have_sha3)) {
/*
* Passing zeroes into any of CPACF_KIMD_SHA3_* gives the plain
* Keccak-f permutation, which is what we want here. Use
* SHA3-512 since it has the smallest block size.
*/
static const u8 zeroes[SHA3_512_BLOCK_SIZE];
cpacf_kimd(CPACF_KIMD_SHA3_512, state, zeroes, sizeof(zeroes));
} else {
sha3_keccakf_generic(state);
}
}
#define sha3_mod_init_arch sha3_mod_init_arch
static void sha3_mod_init_arch(void)
{
int num_present = 0;
int num_possible = 0;
if (!cpu_have_feature(S390_CPU_FEATURE_MSA))
return;
/*
* Since all the SHA-3 functions are in Message-Security-Assist
* Extension 6, just treat them as all or nothing. This way we need
* only one static_key.
*/
#define QUERY(opcode, func) \
({ num_present += !!cpacf_query_func(opcode, func); num_possible++; })
QUERY(CPACF_KIMD, CPACF_KIMD_SHA3_224);
QUERY(CPACF_KIMD, CPACF_KIMD_SHA3_256);
QUERY(CPACF_KIMD, CPACF_KIMD_SHA3_384);
QUERY(CPACF_KIMD, CPACF_KIMD_SHA3_512);
#undef QUERY
if (num_present == num_possible)
static_branch_enable(&have_sha3);
else if (num_present != 0)
pr_warn("Unsupported combination of SHA-3 facilities\n");
}
|
