From 3e0f58f2499c665b633aa6e79de6529ab0845771 Mon Sep 17 00:00:00 2001 From: Jan Kasiak Date: Sun, 5 Sep 2021 22:33:34 -0400 Subject: [PATCH] Initialize sha2 module with existing implementations --- lib/libspl/include/sys/sha2.h | 2 + module/icp/algs/sha2/sha2.c | 123 +++++++++++++++++- module/icp/include/sha2/sha2_impl.h | 5 + module/icp/io/sha2_mod.c | 3 + .../tests/functional/checksum/Makefile.am | 1 + 5 files changed, 127 insertions(+), 7 deletions(-) diff --git a/lib/libspl/include/sys/sha2.h b/lib/libspl/include/sys/sha2.h index e2f66d225e25..be17687e8ee3 100644 --- a/lib/libspl/include/sys/sha2.h +++ b/lib/libspl/include/sys/sha2.h @@ -114,6 +114,8 @@ extern void SHA2Update(SHA2_CTX *, const void *, size_t); extern void SHA2Final(void *, SHA2_CTX *); +extern void sha2_impl_init(void); + #ifdef _SHA2_IMPL /* * The following types/functions are all private to the implementation diff --git a/module/icp/algs/sha2/sha2.c b/module/icp/algs/sha2/sha2.c index 75f6a3c1af4b..097993f1214b 100644 --- a/module/icp/algs/sha2/sha2.c +++ b/module/icp/algs/sha2/sha2.c @@ -39,7 +39,9 @@ #include #define _SHA2_IMPL #include +#include #include +#include #define _RESTRICT_KYWD @@ -52,10 +54,11 @@ static void Encode(uint8_t *, uint32_t *, size_t); static void Encode64(uint8_t *, uint64_t *, size_t); + /* userspace only supports the generic version */ #if defined(__amd64) && defined(_KERNEL) -#define SHA512Transform(ctx, in) SHA512TransformBlocks((ctx), (in), 1) -#define SHA256Transform(ctx, in) SHA256TransformBlocks((ctx), (in), 1) + +typedef void (*sha2_block_f)(SHA2_CTX *ctx, const void *in, size_t num); void SHA512TransformBlocks(SHA2_CTX *ctx, const void *in, size_t num); void SHA256TransformBlocks(SHA2_CTX *ctx, const void *in, size_t num); @@ -63,8 +66,63 @@ void SHA256TransformBlocks(SHA2_CTX *ctx, const void *in, size_t num); #else static void SHA256Transform(SHA2_CTX *, const uint8_t *); static void SHA512Transform(SHA2_CTX *, const uint8_t *); + +static void SHA512TransformBlocks(SHA2_CTX *ctx, const void *in, size_t num); +static void SHA256TransformBlocks(SHA2_CTX *ctx, const void *in, size_t num); + #endif /* __amd64 && _KERNEL */ +/* + * For compatibility with existing code, generic and x86_64 may point to the + * same implementation. + */ +static alg_impl_ops_t sha256_impl_generic = { + SHA256TransformBlocks, alg_impl_will_always_work, 0, "generic"}; + +#if defined(__x86_64) +static alg_impl_ops_t sha256_x86_64_generic = { + SHA256TransformBlocks, alg_impl_will_always_work, 1, "x86_64"}; +#endif + +/* All compiled in implementations */ +static const alg_impl_ops_t *sha256_all_impl[] = { + &sha256_impl_generic, +#if defined(__x86_64) + &sha256_x86_64_generic, +#endif +}; + +static alg_impl_ops_t *sha256_supp_impl[ARRAY_SIZE(sha256_all_impl)]; + +static alg_impl_conf_t sha256_conf_impl = ALG_IMPL_CONF_DECL( + sha256_all_impl, sha256_supp_impl, sha256_impl_generic); + +static alg_impl_ops_t sha512_impl_generic = { + SHA512TransformBlocks, alg_impl_will_always_work, 0, "generic"}; + +#if defined(__x86_64) +static alg_impl_ops_t sha512_x86_64_generic = { + SHA512TransformBlocks, alg_impl_will_always_work, 1, "x86_64"}; +#endif + +/* All compiled in implementations */ +static const alg_impl_ops_t *sha512_all_impl[] = { + &sha512_impl_generic, +#if defined(__x86_64) + &sha512_x86_64_generic, +#endif +}; + +static alg_impl_ops_t *sha512_supp_impl[ARRAY_SIZE(sha512_all_impl)]; + +static alg_impl_conf_t sha512_conf_impl = ALG_IMPL_CONF_DECL( + sha512_all_impl, sha512_supp_impl, sha512_impl_generic); + +void sha2_impl_init(void) { + alg_impl_init(&sha256_conf_impl); + alg_impl_init(&sha512_conf_impl); +} + static uint8_t PADDING[128] = { 0x80, /* all zeros */ }; /* @@ -146,6 +204,15 @@ static uint8_t PADDING[128] = { 0x80, /* all zeros */ }; #if !defined(__amd64) || !defined(_KERNEL) /* SHA256 Transform */ +static void SHA256TransformBlocks(SHA2_CTX *ctx, const void *in, size_t num) +{ + const uint8_t *ptr = in; + for (size_t i = 0; i < num; i++) { + SHA256Transform(ctx, ptr); + ptr += 64; + } +} + static void SHA256Transform(SHA2_CTX *ctx, const uint8_t *blk) { @@ -355,6 +422,15 @@ SHA256Transform(SHA2_CTX *ctx, const uint8_t *blk) /* SHA384 and SHA512 Transform */ +static void SHA512TransformBlocks(SHA2_CTX *ctx, const void *in, size_t num) +{ + const uint8_t *ptr = in; + for (size_t i = 0; i < num; i++) { + SHA512Transform(ctx, ptr); + ptr += 128; + } +} + static void SHA512Transform(SHA2_CTX *ctx, const uint8_t *blk) { @@ -785,6 +861,10 @@ SHA2Update(SHA2_CTX *ctx, const void *inptr, size_t input_len) const uint8_t *input = inptr; uint32_t algotype = ctx->algotype; +#if defined(__amd64) && defined(_KERNEL) + sha2_block_f sha2_impl = NULL; +#endif + /* check for noop */ if (input_len == 0) return; @@ -801,6 +881,10 @@ SHA2Update(SHA2_CTX *ctx, const void *inptr, size_t input_len) ctx->count.c32[0] += (input_len >> 29); +#if defined(__amd64) && defined(_KERNEL) + const alg_impl_ops_t *ops = alg_impl_get_ops(&sha256_conf_impl); + sha2_impl = (sha2_block_f)(ops->ctx); +#endif } else { buf_limit = 128; @@ -812,6 +896,11 @@ SHA2Update(SHA2_CTX *ctx, const void *inptr, size_t input_len) ctx->count.c64[0]++; ctx->count.c64[0] += (input_len >> 29); + +#if defined(__amd64) && defined(_KERNEL) + const alg_impl_ops_t *ops = alg_impl_get_ops(&sha512_conf_impl); + sha2_impl = (sha2_block_f)(ops->ctx); +#endif } buf_len = buf_limit - buf_index; @@ -831,11 +920,14 @@ SHA2Update(SHA2_CTX *ctx, const void *inptr, size_t input_len) */ if (buf_index) { bcopy(input, &ctx->buf_un.buf8[buf_index], buf_len); +#if !defined(__amd64) || !defined(_KERNEL) if (algotype <= SHA256_HMAC_GEN_MECH_INFO_TYPE) SHA256Transform(ctx, ctx->buf_un.buf8); else SHA512Transform(ctx, ctx->buf_un.buf8); - +#else + sha2_impl(ctx, ctx->buf_un.buf8, 1); +#endif i = buf_len; } @@ -855,15 +947,13 @@ SHA2Update(SHA2_CTX *ctx, const void *inptr, size_t input_len) if (algotype <= SHA256_HMAC_GEN_MECH_INFO_TYPE) { block_count = (input_len - i) >> 6; if (block_count > 0) { - SHA256TransformBlocks(ctx, &input[i], - block_count); + sha2_impl(ctx, &input[i], block_count); i += block_count << 6; } } else { block_count = (input_len - i) >> 7; if (block_count > 0) { - SHA512TransformBlocks(ctx, &input[i], - block_count); + sha2_impl(ctx, &input[i], block_count); i += block_count << 7; } } @@ -954,3 +1044,22 @@ EXPORT_SYMBOL(SHA2Init); EXPORT_SYMBOL(SHA2Update); EXPORT_SYMBOL(SHA2Final); #endif + +#if defined(_KERNEL) && defined(__linux__) + +static int +icp_sha256_impl_set(const char *val, zfs_kernel_param_t *kp) +{ + return (alg_impl_set(&sha256_conf_impl, val)); +} + +static int +icp_sha256_impl_get(char *buffer, zfs_kernel_param_t *kp) +{ + return (alg_impl_get(&sha256_conf_impl, buffer)); +} + +module_param_call(icp_sha256_impl, icp_sha256_impl_set, icp_sha256_impl_get, + NULL, 0644); +MODULE_PARM_DESC(icp_sha256_impl, "Select sha256 implementation."); +#endif diff --git a/module/icp/include/sha2/sha2_impl.h b/module/icp/include/sha2/sha2_impl.h index b9768d344e95..7abd2b11c305 100644 --- a/module/icp/include/sha2/sha2_impl.h +++ b/module/icp/include/sha2/sha2_impl.h @@ -57,6 +57,11 @@ typedef struct sha2_hmac_ctx { SHA2_CTX hc_ocontext; /* outer SHA2 context */ } sha2_hmac_ctx_t; +/* + * Initialize sha2 implementations. + */ +void sha2_impl_init(void); + #ifdef __cplusplus } #endif diff --git a/module/icp/io/sha2_mod.c b/module/icp/io/sha2_mod.c index d690cd0bcb05..0b3d554ecde5 100644 --- a/module/icp/io/sha2_mod.c +++ b/module/icp/io/sha2_mod.c @@ -216,6 +216,9 @@ sha2_mod_init(void) { int ret; + /* Determine the fastest available implementation. */ + sha2_impl_init(); + if ((ret = mod_install(&modlinkage)) != 0) return (ret); diff --git a/tests/zfs-tests/tests/functional/checksum/Makefile.am b/tests/zfs-tests/tests/functional/checksum/Makefile.am index ddabc0302010..0be51f94b0bf 100644 --- a/tests/zfs-tests/tests/functional/checksum/Makefile.am +++ b/tests/zfs-tests/tests/functional/checksum/Makefile.am @@ -2,6 +2,7 @@ include $(top_srcdir)/config/Rules.am LDADD = \ $(abs_top_builddir)/lib/libicp/libicp.la \ + $(abs_top_builddir)/lib/libspl/libspl.la \ $(abs_top_builddir)/lib/libspl/libspl_assert.la pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/checksum