1 files changed, 64 insertions, 162 deletions

diff --git a/crypto/hctr2.c b/crypto/hctr2.c
index c8932777bba8..f4cd6c29b4d3 100644
--- a/crypto/hctr2.c
+++ b/crypto/hctr2.c
@@ -17,7 +17,6 @@
  */
 
 #include <crypto/internal/cipher.h>
-#include <crypto/internal/hash.h>
 #include <crypto/internal/skcipher.h>
 #include <crypto/polyval.h>
 #include <crypto/scatterwalk.h>
@@ -37,23 +36,14 @@
 struct hctr2_instance_ctx {
 	struct crypto_cipher_spawn blockcipher_spawn;
 	struct crypto_skcipher_spawn xctr_spawn;
-	struct crypto_shash_spawn polyval_spawn;
 };
 
 struct hctr2_tfm_ctx {
 	struct crypto_cipher *blockcipher;
 	struct crypto_skcipher *xctr;
-	struct crypto_shash *polyval;
+	struct polyval_key poly_key;
+	struct polyval_elem hashed_tweaklens[2];
 	u8 L[BLOCKCIPHER_BLOCK_SIZE];
-	int hashed_tweak_offset;
-	/*
-	 * This struct is allocated with extra space for two exported hash
-	 * states.  Since the hash state size is not known at compile-time, we
-	 * can't add these to the struct directly.
-	 *
-	 * hashed_tweaklen_divisible;
-	 * hashed_tweaklen_remainder;
-	 */
 };
 
 struct hctr2_request_ctx {
@@ -63,39 +53,17 @@ struct hctr2_request_ctx {
 	struct scatterlist *bulk_part_src;
 	struct scatterlist sg_src[2];
 	struct scatterlist sg_dst[2];
+	struct polyval_elem hashed_tweak;
 	/*
-	 * Sub-request sizes are unknown at compile-time, so they need to go
-	 * after the members with known sizes.
+	 * skcipher sub-request size is unknown at compile-time, so it needs to
+	 * go after the members with known sizes.
 	 */
 	union {
-		struct shash_desc hash_desc;
+		struct polyval_ctx poly_ctx;
 		struct skcipher_request xctr_req;
 	} u;
-	/*
-	 * This struct is allocated with extra space for one exported hash
-	 * state.  Since the hash state size is not known at compile-time, we
-	 * can't add it to the struct directly.
-	 *
-	 * hashed_tweak;
-	 */
 };
 
-static inline u8 *hctr2_hashed_tweaklen(const struct hctr2_tfm_ctx *tctx,
-					bool has_remainder)
-{
-	u8 *p = (u8 *)tctx + sizeof(*tctx);
-
-	if (has_remainder) /* For messages not a multiple of block length */
-		p += crypto_shash_statesize(tctx->polyval);
-	return p;
-}
-
-static inline u8 *hctr2_hashed_tweak(const struct hctr2_tfm_ctx *tctx,
-				     struct hctr2_request_ctx *rctx)
-{
-	return (u8 *)rctx + tctx->hashed_tweak_offset;
-}
-
 /*
  * The input data for each HCTR2 hash step begins with a 16-byte block that
  * contains the tweak length and a flag that indicates whether the input is evenly
@@ -106,24 +74,23 @@ static inline u8 *hctr2_hashed_tweak(const struct hctr2_tfm_ctx *tctx,
  *
  * These precomputed hashes are stored in hctr2_tfm_ctx.
  */
-static int hctr2_hash_tweaklen(struct hctr2_tfm_ctx *tctx, bool has_remainder)
+static void hctr2_hash_tweaklens(struct hctr2_tfm_ctx *tctx)
 {
-	SHASH_DESC_ON_STACK(shash, tfm->polyval);
-	__le64 tweak_length_block[2];
-	int err;
-
-	shash->tfm = tctx->polyval;
-	memset(tweak_length_block, 0, sizeof(tweak_length_block));
-
-	tweak_length_block[0] = cpu_to_le64(TWEAK_SIZE * 8 * 2 + 2 + has_remainder);
-	err = crypto_shash_init(shash);
-	if (err)
-		return err;
-	err = crypto_shash_update(shash, (u8 *)tweak_length_block,
-				  POLYVAL_BLOCK_SIZE);
-	if (err)
-		return err;
-	return crypto_shash_export(shash, hctr2_hashed_tweaklen(tctx, has_remainder));
+	struct polyval_ctx ctx;
+
+	for (int has_remainder = 0; has_remainder < 2; has_remainder++) {
+		const __le64 tweak_length_block[2] = {
+			cpu_to_le64(TWEAK_SIZE * 8 * 2 + 2 + has_remainder),
+		};
+
+		polyval_init(&ctx, &tctx->poly_key);
+		polyval_update(&ctx, (const u8 *)&tweak_length_block,
+			       sizeof(tweak_length_block));
+		static_assert(sizeof(tweak_length_block) == POLYVAL_BLOCK_SIZE);
+		polyval_export_blkaligned(
+			&ctx, &tctx->hashed_tweaklens[has_remainder]);
+	}
+	memzero_explicit(&ctx, sizeof(ctx));
 }
 
 static int hctr2_setkey(struct crypto_skcipher *tfm, const u8 *key,
@@ -156,51 +123,42 @@ static int hctr2_setkey(struct crypto_skcipher *tfm, const u8 *key,
 	tctx->L[0] = 0x01;
 	crypto_cipher_encrypt_one(tctx->blockcipher, tctx->L, tctx->L);
 
-	crypto_shash_clear_flags(tctx->polyval, CRYPTO_TFM_REQ_MASK);
-	crypto_shash_set_flags(tctx->polyval, crypto_skcipher_get_flags(tfm) &
-			       CRYPTO_TFM_REQ_MASK);
-	err = crypto_shash_setkey(tctx->polyval, hbar, BLOCKCIPHER_BLOCK_SIZE);
-	if (err)
-		return err;
+	static_assert(sizeof(hbar) == POLYVAL_BLOCK_SIZE);
+	polyval_preparekey(&tctx->poly_key, hbar);
 	memzero_explicit(hbar, sizeof(hbar));
 
-	return hctr2_hash_tweaklen(tctx, true) ?: hctr2_hash_tweaklen(tctx, false);
+	hctr2_hash_tweaklens(tctx);
+	return 0;
 }
 
-static int hctr2_hash_tweak(struct skcipher_request *req)
+static void hctr2_hash_tweak(struct skcipher_request *req)
 {
 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 	const struct hctr2_tfm_ctx *tctx = crypto_skcipher_ctx(tfm);
 	struct hctr2_request_ctx *rctx = skcipher_request_ctx(req);
-	struct shash_desc *hash_desc = &rctx->u.hash_desc;
-	int err;
+	struct polyval_ctx *poly_ctx = &rctx->u.poly_ctx;
 	bool has_remainder = req->cryptlen % POLYVAL_BLOCK_SIZE;
 
-	hash_desc->tfm = tctx->polyval;
-	err = crypto_shash_import(hash_desc, hctr2_hashed_tweaklen(tctx, has_remainder));
-	if (err)
-		return err;
-	err = crypto_shash_update(hash_desc, req->iv, TWEAK_SIZE);
-	if (err)
-		return err;
+	polyval_import_blkaligned(poly_ctx, &tctx->poly_key,
+				  &tctx->hashed_tweaklens[has_remainder]);
+	polyval_update(poly_ctx, req->iv, TWEAK_SIZE);
 
 	// Store the hashed tweak, since we need it when computing both
 	// H(T || N) and H(T || V).
-	return crypto_shash_export(hash_desc, hctr2_hashed_tweak(tctx, rctx));
+	static_assert(TWEAK_SIZE % POLYVAL_BLOCK_SIZE == 0);
+	polyval_export_blkaligned(poly_ctx, &rctx->hashed_tweak);
 }
 
-static int hctr2_hash_message(struct skcipher_request *req,
-			      struct scatterlist *sgl,
-			      u8 digest[POLYVAL_DIGEST_SIZE])
+static void hctr2_hash_message(struct skcipher_request *req,
+			       struct scatterlist *sgl,
+			       u8 digest[POLYVAL_DIGEST_SIZE])
 {
-	static const u8 padding[BLOCKCIPHER_BLOCK_SIZE] = { 0x1 };
+	static const u8 padding = 0x1;
 	struct hctr2_request_ctx *rctx = skcipher_request_ctx(req);
-	struct shash_desc *hash_desc = &rctx->u.hash_desc;
+	struct polyval_ctx *poly_ctx = &rctx->u.poly_ctx;
 	const unsigned int bulk_len = req->cryptlen - BLOCKCIPHER_BLOCK_SIZE;
 	struct sg_mapping_iter miter;
-	unsigned int remainder = bulk_len % BLOCKCIPHER_BLOCK_SIZE;
 	int i;
-	int err = 0;
 	int n = 0;
 
 	sg_miter_start(&miter, sgl, sg_nents(sgl),
@@ -208,22 +166,13 @@ static int hctr2_hash_message(struct skcipher_request *req,
 	for (i = 0; i < bulk_len; i += n) {
 		sg_miter_next(&miter);
 		n = min_t(unsigned int, miter.length, bulk_len - i);
-		err = crypto_shash_update(hash_desc, miter.addr, n);
-		if (err)
-			break;
+		polyval_update(poly_ctx, miter.addr, n);
 	}
 	sg_miter_stop(&miter);
 
-	if (err)
-		return err;
-
-	if (remainder) {
-		err = crypto_shash_update(hash_desc, padding,
-					  BLOCKCIPHER_BLOCK_SIZE - remainder);
-		if (err)
-			return err;
-	}
-	return crypto_shash_final(hash_desc, digest);
+	if (req->cryptlen % BLOCKCIPHER_BLOCK_SIZE)
+		polyval_update(poly_ctx, &padding, 1);
+	polyval_final(poly_ctx, digest);
 }
 
 static int hctr2_finish(struct skcipher_request *req)
@@ -231,19 +180,14 @@ static int hctr2_finish(struct skcipher_request *req)
 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 	const struct hctr2_tfm_ctx *tctx = crypto_skcipher_ctx(tfm);
 	struct hctr2_request_ctx *rctx = skcipher_request_ctx(req);
+	struct polyval_ctx *poly_ctx = &rctx->u.poly_ctx;
 	u8 digest[POLYVAL_DIGEST_SIZE];
-	struct shash_desc *hash_desc = &rctx->u.hash_desc;
-	int err;
 
 	// U = UU ^ H(T || V)
 	// or M = MM ^ H(T || N)
-	hash_desc->tfm = tctx->polyval;
-	err = crypto_shash_import(hash_desc, hctr2_hashed_tweak(tctx, rctx));
-	if (err)
-		return err;
-	err = hctr2_hash_message(req, rctx->bulk_part_dst, digest);
-	if (err)
-		return err;
+	polyval_import_blkaligned(poly_ctx, &tctx->poly_key,
+				  &rctx->hashed_tweak);
+	hctr2_hash_message(req, rctx->bulk_part_dst, digest);
 	crypto_xor(rctx->first_block, digest, BLOCKCIPHER_BLOCK_SIZE);
 
 	// Copy U (or M) into dst scatterlist
@@ -269,7 +213,6 @@ static int hctr2_crypt(struct skcipher_request *req, bool enc)
 	struct hctr2_request_ctx *rctx = skcipher_request_ctx(req);
 	u8 digest[POLYVAL_DIGEST_SIZE];
 	int bulk_len = req->cryptlen - BLOCKCIPHER_BLOCK_SIZE;
-	int err;
 
 	// Requests must be at least one block
 	if (req->cryptlen < BLOCKCIPHER_BLOCK_SIZE)
@@ -287,12 +230,8 @@ static int hctr2_crypt(struct skcipher_request *req, bool enc)
 
 	// MM = M ^ H(T || N)
 	// or UU = U ^ H(T || V)
-	err = hctr2_hash_tweak(req);
-	if (err)
-		return err;
-	err = hctr2_hash_message(req, rctx->bulk_part_src, digest);
-	if (err)
-		return err;
+	hctr2_hash_tweak(req);
+	hctr2_hash_message(req, rctx->bulk_part_src, digest);
 	crypto_xor(digest, rctx->first_block, BLOCKCIPHER_BLOCK_SIZE);
 
 	// UU = E(MM)
@@ -338,8 +277,6 @@ static int hctr2_init_tfm(struct crypto_skcipher *tfm)
 	struct hctr2_tfm_ctx *tctx = crypto_skcipher_ctx(tfm);
 	struct crypto_skcipher *xctr;
 	struct crypto_cipher *blockcipher;
-	struct crypto_shash *polyval;
-	unsigned int subreq_size;
 	int err;
 
 	xctr = crypto_spawn_skcipher(&ictx->xctr_spawn);
@@ -352,31 +289,17 @@ static int hctr2_init_tfm(struct crypto_skcipher *tfm)
 		goto err_free_xctr;
 	}
 
-	polyval = crypto_spawn_shash(&ictx->polyval_spawn);
-	if (IS_ERR(polyval)) {
-		err = PTR_ERR(polyval);
-		goto err_free_blockcipher;
-	}
-
 	tctx->xctr = xctr;
 	tctx->blockcipher = blockcipher;
-	tctx->polyval = polyval;
 
 	BUILD_BUG_ON(offsetofend(struct hctr2_request_ctx, u) !=
 				 sizeof(struct hctr2_request_ctx));
-	subreq_size = max(sizeof_field(struct hctr2_request_ctx, u.hash_desc) +
-			  crypto_shash_descsize(polyval),
-			  sizeof_field(struct hctr2_request_ctx, u.xctr_req) +
-			  crypto_skcipher_reqsize(xctr));
-
-	tctx->hashed_tweak_offset = offsetof(struct hctr2_request_ctx, u) +
-				    subreq_size;
-	crypto_skcipher_set_reqsize(tfm, tctx->hashed_tweak_offset +
-				    crypto_shash_statesize(polyval));
+	crypto_skcipher_set_reqsize(
+		tfm, max(sizeof(struct hctr2_request_ctx),
+			 offsetofend(struct hctr2_request_ctx, u.xctr_req) +
+				 crypto_skcipher_reqsize(xctr)));
 	return 0;
 
-err_free_blockcipher:
-	crypto_free_cipher(blockcipher);
 err_free_xctr:
 	crypto_free_skcipher(xctr);
 	return err;
@@ -388,7 +311,6 @@ static void hctr2_exit_tfm(struct crypto_skcipher *tfm)
 
 	crypto_free_cipher(tctx->blockcipher);
 	crypto_free_skcipher(tctx->xctr);
-	crypto_free_shash(tctx->polyval);
 }
 
 static void hctr2_free_instance(struct skcipher_instance *inst)
@@ -397,21 +319,17 @@ static void hctr2_free_instance(struct skcipher_instance *inst)
 
 	crypto_drop_cipher(&ictx->blockcipher_spawn);
 	crypto_drop_skcipher(&ictx->xctr_spawn);
-	crypto_drop_shash(&ictx->polyval_spawn);
 	kfree(inst);
 }
 
-static int hctr2_create_common(struct crypto_template *tmpl,
-			       struct rtattr **tb,
-			       const char *xctr_name,
-			       const char *polyval_name)
+static int hctr2_create_common(struct crypto_template *tmpl, struct rtattr **tb,
+			       const char *xctr_name)
 {
 	struct skcipher_alg_common *xctr_alg;
 	u32 mask;
 	struct skcipher_instance *inst;
 	struct hctr2_instance_ctx *ictx;
 	struct crypto_alg *blockcipher_alg;
-	struct shash_alg *polyval_alg;
 	char blockcipher_name[CRYPTO_MAX_ALG_NAME];
 	int len;
 	int err;
@@ -457,19 +375,6 @@ static int hctr2_create_common(struct crypto_template *tmpl,
 	if (blockcipher_alg->cra_blocksize != BLOCKCIPHER_BLOCK_SIZE)
 		goto err_free_inst;
 
-	/* Polyval ε-∆U hash function */
-	err = crypto_grab_shash(&ictx->polyval_spawn,
-				skcipher_crypto_instance(inst),
-				polyval_name, 0, mask);
-	if (err)
-		goto err_free_inst;
-	polyval_alg = crypto_spawn_shash_alg(&ictx->polyval_spawn);
-
-	/* Ensure Polyval is being used */
-	err = -EINVAL;
-	if (strcmp(polyval_alg->base.cra_name, "polyval") != 0)
-		goto err_free_inst;
-
 	/* Instance fields */
 
 	err = -ENAMETOOLONG;
@@ -477,22 +382,16 @@ static int hctr2_create_common(struct crypto_template *tmpl,
 		     blockcipher_alg->cra_name) >= CRYPTO_MAX_ALG_NAME)
 		goto err_free_inst;
 	if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
-		     "hctr2_base(%s,%s)",
-		     xctr_alg->base.cra_driver_name,
-		     polyval_alg->base.cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
+		     "hctr2_base(%s,polyval-lib)",
+		     xctr_alg->base.cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
 		goto err_free_inst;
 
 	inst->alg.base.cra_blocksize = BLOCKCIPHER_BLOCK_SIZE;
-	inst->alg.base.cra_ctxsize = sizeof(struct hctr2_tfm_ctx) +
-				     polyval_alg->statesize * 2;
+	inst->alg.base.cra_ctxsize = sizeof(struct hctr2_tfm_ctx);
 	inst->alg.base.cra_alignmask = xctr_alg->base.cra_alignmask;
-	/*
-	 * The hash function is called twice, so it is weighted higher than the
-	 * xctr and blockcipher.
-	 */
 	inst->alg.base.cra_priority = (2 * xctr_alg->base.cra_priority +
-				       4 * polyval_alg->base.cra_priority +
-				       blockcipher_alg->cra_priority) / 7;
+				       blockcipher_alg->cra_priority) /
+				      3;
 
 	inst->alg.setkey = hctr2_setkey;
 	inst->alg.encrypt = hctr2_encrypt;
@@ -525,8 +424,11 @@ static int hctr2_create_base(struct crypto_template *tmpl, struct rtattr **tb)
 	polyval_name = crypto_attr_alg_name(tb[2]);
 	if (IS_ERR(polyval_name))
 		return PTR_ERR(polyval_name);
+	if (strcmp(polyval_name, "polyval") != 0 &&
+	    strcmp(polyval_name, "polyval-lib") != 0)
+		return -ENOENT;
 
-	return hctr2_create_common(tmpl, tb, xctr_name, polyval_name);
+	return hctr2_create_common(tmpl, tb, xctr_name);
 }
 
 static int hctr2_create(struct crypto_template *tmpl, struct rtattr **tb)
@@ -542,7 +444,7 @@ static int hctr2_create(struct crypto_template *tmpl, struct rtattr **tb)
 		    blockcipher_name) >= CRYPTO_MAX_ALG_NAME)
 		return -ENAMETOOLONG;
 
-	return hctr2_create_common(tmpl, tb, xctr_name, "polyval");
+	return hctr2_create_common(tmpl, tb, xctr_name);
 }
 
 static struct crypto_template hctr2_tmpls[] = {