From c9afb1acf8049eb2de7f869ed13dd077e742a20b Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Fri, 20 Dec 2024 22:41:39 +0800 Subject: [PATCH] erofs-utils: use external xxhash library if possible It's expected to be faster than the internal one. Signed-off-by: Gao Xiang Link: https://lore.kernel.org/r/20241220144139.648210-1-hsiangkao@linux.alibaba.com --- configure.ac | 34 +++++++++++++++++++++++++++++ dump/Makefile.am | 2 +- fsck/Makefile.am | 4 ++-- fuse/Makefile.am | 2 +- lib/Makefile.am | 9 ++++++-- lib/dedupe.c | 2 +- lib/{xxhash.h => liberofs_xxhash.h} | 15 +++++++++++++ lib/xattr.c | 2 +- lib/xxhash.c | 2 +- mkfs/Makefile.am | 2 +- 10 files changed, 64 insertions(+), 10 deletions(-) rename lib/{xxhash.h => liberofs_xxhash.h} (74%) diff --git a/configure.ac b/configure.ac index 45a7d335..0a069c51 100644 --- a/configure.ac +++ b/configure.ac @@ -148,6 +148,10 @@ AC_ARG_WITH(qpl, [Enable and build with Intel QPL support @<:@default=disabled@:>@])], [], [with_qpl="no"]) +AC_ARG_WITH(xxhash, + [AS_HELP_STRING([--with-xxhash], + [Enable and build with libxxhash support @<:@default=auto@:>@])]) + AC_ARG_ENABLE(fuse, [AS_HELP_STRING([--enable-fuse], [enable erofsfuse @<:@default=no@:>@])], [enable_fuse="$enableval"], [enable_fuse="no"]) @@ -531,6 +535,31 @@ AS_IF([test "x$with_qpl" != "xno"], [ ]) ]) +# Configure libxxhash +have_xxhash="no" +AS_IF([test "x$with_xxhash" != "xno"], [ + PKG_CHECK_MODULES([libxxhash], [libxxhash], [ + # Paranoia: don't trust the result reported by pkgconfig before trying out + saved_LIBS="$LIBS" + saved_CPPFLAGS=${CPPFLAGS} + CPPFLAGS="${libxxhash_CFLAGS} ${CPPFLAGS}" + LIBS="${libxxhash_LIBS} $LIBS" + AC_CHECK_HEADERS([xxhash.h],[ + AC_CHECK_LIB(xxhash, XXH32, [], [ + AC_MSG_ERROR([libxxhash doesn't work properly])]) + AC_CHECK_DECL(XXH32, [have_xxhash="yes"], + [AC_MSG_ERROR([libxxhash doesn't work properly])], [[ +#include + ]]) + ]) + LIBS="${saved_LIBS}" + CPPFLAGS="${saved_CPPFLAGS}"], [ + AS_IF([test "x$with_xxhash" = "xyes"], [ + AC_MSG_ERROR([Cannot find proper libxxhash]) + ]) + ]) +]) + # Enable 64-bit off_t CFLAGS+=" -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64" @@ -553,6 +582,7 @@ AM_CONDITIONAL([ENABLE_LIBLZMA], [test "x${have_liblzma}" = "xyes"]) AM_CONDITIONAL([ENABLE_LIBDEFLATE], [test "x${have_libdeflate}" = "xyes"]) AM_CONDITIONAL([ENABLE_LIBZSTD], [test "x${have_libzstd}" = "xyes"]) AM_CONDITIONAL([ENABLE_QPL], [test "x${have_qpl}" = "xyes"]) +AM_CONDITIONAL([ENABLE_XXHASH], [test "x${have_xxhash}" = "xyes"]) AM_CONDITIONAL([ENABLE_STATIC_FUSE], [test "x${enable_static_fuse}" = "xyes"]) if test "x$have_uuid" = "xyes"; then @@ -600,6 +630,10 @@ if test "x$have_qpl" = "xyes"; then AC_SUBST([libqpl_CFLAGS]) fi +if test "x$have_xxhash" = "xyes"; then + AC_DEFINE([HAVE_XXHASH], 1, [Define to 1 if xxhash is found]) +fi + # Dump maximum block size AS_IF([test "x$erofs_cv_max_block_size" = "x"], [$erofs_cv_max_block_size = 4096], []) diff --git a/dump/Makefile.am b/dump/Makefile.am index 2a4f67ae..2cf7fe87 100644 --- a/dump/Makefile.am +++ b/dump/Makefile.am @@ -8,4 +8,4 @@ dump_erofs_SOURCES = main.c dump_erofs_CFLAGS = -Wall -I$(top_srcdir)/include dump_erofs_LDADD = $(top_builddir)/lib/liberofs.la ${libselinux_LIBS} \ ${liblz4_LIBS} ${liblzma_LIBS} ${zlib_LIBS} ${libdeflate_LIBS} \ - ${libzstd_LIBS} ${libqpl_LIBS} + ${libzstd_LIBS} ${libqpl_LIBS} ${libxxhash_LIBS} diff --git a/fsck/Makefile.am b/fsck/Makefile.am index 5bdee4d9..3b7b5914 100644 --- a/fsck/Makefile.am +++ b/fsck/Makefile.am @@ -8,7 +8,7 @@ fsck_erofs_SOURCES = main.c fsck_erofs_CFLAGS = -Wall -I$(top_srcdir)/include fsck_erofs_LDADD = $(top_builddir)/lib/liberofs.la ${libselinux_LIBS} \ ${liblz4_LIBS} ${liblzma_LIBS} ${zlib_LIBS} ${libdeflate_LIBS} \ - ${libzstd_LIBS} ${libqpl_LIBS} + ${libzstd_LIBS} ${libqpl_LIBS} ${libxxhash_LIBS} if ENABLE_FUZZING noinst_PROGRAMS = fuzz_erofsfsck @@ -17,5 +17,5 @@ fuzz_erofsfsck_CFLAGS = -Wall -I$(top_srcdir)/include -DFUZZING fuzz_erofsfsck_LDFLAGS = -fsanitize=address,fuzzer fuzz_erofsfsck_LDADD = $(top_builddir)/lib/liberofs.la ${libselinux_LIBS} \ ${liblz4_LIBS} ${liblzma_LIBS} ${zlib_LIBS} ${libdeflate_LIBS} \ - ${libzstd_LIBS} ${libqpl_LIBS} + ${libzstd_LIBS} ${libqpl_LIBS} ${libxxhash_LIBS} endif diff --git a/fuse/Makefile.am b/fuse/Makefile.am index 50186dad..55fb61fc 100644 --- a/fuse/Makefile.am +++ b/fuse/Makefile.am @@ -8,7 +8,7 @@ erofsfuse_CFLAGS = -Wall -I$(top_srcdir)/include erofsfuse_CFLAGS += ${libfuse2_CFLAGS} ${libfuse3_CFLAGS} ${libselinux_CFLAGS} erofsfuse_LDADD = $(top_builddir)/lib/liberofs.la ${libfuse2_LIBS} ${libfuse3_LIBS} ${liblz4_LIBS} \ ${libselinux_LIBS} ${liblzma_LIBS} ${zlib_LIBS} ${libdeflate_LIBS} ${libzstd_LIBS} \ - ${libqpl_LIBS} + ${libqpl_LIBS} ${libxxhash_LIBS} if ENABLE_STATIC_FUSE lib_LTLIBRARIES = liberofsfuse.la diff --git a/lib/Makefile.am b/lib/Makefile.am index 9c0604dc..ef98377b 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -27,14 +27,14 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \ $(top_srcdir)/include/erofs/fragments.h \ $(top_srcdir)/include/erofs/rebuild.h \ $(top_srcdir)/lib/liberofs_private.h \ - $(top_srcdir)/lib/xxhash.h + $(top_srcdir)/lib/liberofs_xxhash.h noinst_HEADERS += compressor.h liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \ namei.c data.c compress.c compressor.c zmap.c decompress.c \ compress_hints.c hashmap.c sha256.c blobchunk.c dir.c \ fragments.c dedupe.c uuid_unparse.c uuid.c tar.c \ - block_list.c xxhash.c rebuild.c diskbuf.c + block_list.c rebuild.c diskbuf.c liberofs_la_CFLAGS = -Wall ${libuuid_CFLAGS} -I$(top_srcdir)/include if ENABLE_LZ4 @@ -58,6 +58,11 @@ if ENABLE_LIBZSTD liberofs_la_CFLAGS += ${libzstd_CFLAGS} liberofs_la_SOURCES += compressor_libzstd.c endif +if ENABLE_XXHASH +liberofs_la_CFLAGS += ${libxxhash_CFLAGS} +else +liberofs_la_SOURCES += xxhash.c +endif if ENABLE_EROFS_MT liberofs_la_LDFLAGS = -lpthread liberofs_la_SOURCES += workqueue.c diff --git a/lib/dedupe.c b/lib/dedupe.c index 665915a1..85ff3c98 100644 --- a/lib/dedupe.c +++ b/lib/dedupe.c @@ -6,7 +6,7 @@ #include "erofs/dedupe.h" #include "erofs/print.h" #include "rolling_hash.h" -#include "xxhash.h" +#include "liberofs_xxhash.h" #include "sha256.h" unsigned long erofs_memcmp2(const u8 *s1, const u8 *s2, diff --git a/lib/xxhash.h b/lib/liberofs_xxhash.h similarity index 74% rename from lib/xxhash.h rename to lib/liberofs_xxhash.h index 723c3a55..a0f83679 100644 --- a/lib/xxhash.h +++ b/lib/liberofs_xxhash.h @@ -8,7 +8,21 @@ extern "C" #endif #include +#ifdef HAVE_XXHASH_H +#include +#endif + +#ifdef HAVE_XXHASH +static inline uint32_t xxh32(const void *input, size_t length, uint32_t seed) +{ + return XXH32(input, length, seed); +} +static inline uint64_t xxh64(const void *input, const size_t len, const uint64_t seed) +{ + return XXH64(input, len, seed); +} +#else /* * xxh32() - calculate the 32-bit hash of the input with a given seed. * @@ -32,6 +46,7 @@ uint32_t xxh32(const void *input, size_t length, uint32_t seed); * Return: The 64-bit hash of the data. */ uint64_t xxh64(const void *input, const size_t len, const uint64_t seed); +#endif #ifdef __cplusplus } diff --git a/lib/xattr.c b/lib/xattr.c index e4207758..dc919abb 100644 --- a/lib/xattr.c +++ b/lib/xattr.c @@ -17,7 +17,7 @@ #include "erofs/xattr.h" #include "erofs/cache.h" #include "erofs/fragments.h" -#include "xxhash.h" +#include "liberofs_xxhash.h" #include "liberofs_private.h" #ifndef XATTR_SYSTEM_PREFIX diff --git a/lib/xxhash.c b/lib/xxhash.c index 2768375d..ee78ebf1 100644 --- a/lib/xxhash.c +++ b/lib/xxhash.c @@ -44,7 +44,7 @@ * - xxHash source repository: https://github.com/Cyan4973/xxHash */ #include "erofs/defs.h" -#include "xxhash.h" +#include "liberofs_xxhash.h" /*-************************************* * Macros diff --git a/mkfs/Makefile.am b/mkfs/Makefile.am index 6354712b..24992421 100644 --- a/mkfs/Makefile.am +++ b/mkfs/Makefile.am @@ -7,4 +7,4 @@ mkfs_erofs_SOURCES = main.c mkfs_erofs_CFLAGS = -Wall -I$(top_srcdir)/include mkfs_erofs_LDADD = $(top_builddir)/lib/liberofs.la ${libselinux_LIBS} \ ${libuuid_LIBS} ${liblz4_LIBS} ${liblzma_LIBS} ${zlib_LIBS} \ - ${libdeflate_LIBS} ${libzstd_LIBS} ${libqpl_LIBS} + ${libdeflate_LIBS} ${libzstd_LIBS} ${libqpl_LIBS} ${libxxhash_LIBS}