Skip to content

Commit

Permalink
Eliminate memory allocations from Compress/Decompress.
Browse files Browse the repository at this point in the history
Go allocates a copy of slice headers when they are passed to C functions.
See golang/go#24450 for details.

Avoid this by passing unsafe.Pointer as C.uintptr_t into C functions.

Benchmark results:

$ PAYLOAD=ZSTD_LICENSE go test -run=111 -bench='k[CD]' -benchmem -count=10

name             old time/op    new time/op    delta
Compression-4      21.2µs ± 5%    20.8µs ± 6%      ~     (p=0.190 n=10+10)
Decompression-4    7.37µs ± 5%    6.93µs ± 6%    -5.95%  (p=0.001 n=10+10)

name             old speed      new speed      delta
Compression-4    62.5MB/s ± 5%  63.7MB/s ± 6%      ~     (p=0.190 n=10+10)
Decompression-4   180MB/s ± 5%   191MB/s ± 6%    +6.35%  (p=0.001 n=10+10)

name             old alloc/op   new alloc/op   delta
Compression-4       96.0B ± 0%      0.0B       -100.00%  (p=0.000 n=10+10)
Decompression-4     96.0B ± 0%      0.0B       -100.00%  (p=0.000 n=10+10)

name             old allocs/op  new allocs/op  delta
Compression-4        4.00 ± 0%      0.00       -100.00%  (p=0.000 n=10+10)
Decompression-4      4.00 ± 0%      0.00       -100.00%  (p=0.000 n=10+10)
  • Loading branch information
valyala committed Mar 21, 2018
1 parent 19f0f4b commit e068f47
Showing 1 changed file with 20 additions and 6 deletions.
26 changes: 20 additions & 6 deletions zstd.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,20 @@ package zstd

/*
#include "zstd.h"
#include "stdint.h" // for uintptr_t
// The following *_wrapper function are used for removing superflouos
// memory allocations when calling the wrapped functions from Go code.
// See https://github.com/golang/go/issues/24450 for details.
static size_t ZSTD_compress_wrapper(uintptr_t dst, size_t maxDstSize, const uintptr_t src, size_t srcSize, int compressionLevel) {
return ZSTD_compress((void*)dst, maxDstSize, (const void*)src, srcSize, compressionLevel);
}
static size_t ZSTD_decompress_wrapper(uintptr_t dst, size_t maxDstSize, uintptr_t src, size_t srcSize) {
return ZSTD_decompress((void*)dst, maxDstSize, (const void *)src, srcSize);
}
*/
import "C"
import (
Expand Down Expand Up @@ -117,10 +131,10 @@ func CompressLevel(dst, src []byte, level int) ([]byte, error) {
dst = make([]byte, bound)
}

cWritten := C.ZSTD_compress(
unsafe.Pointer(&dst[0]),
cWritten := C.ZSTD_compress_wrapper(
C.uintptr_t(uintptr(unsafe.Pointer(&dst[0]))),
C.size_t(len(dst)),
unsafe.Pointer(&src[0]),
C.uintptr_t(uintptr(unsafe.Pointer(&src[0]))),
C.size_t(len(src)),
C.int(level))

Expand All @@ -141,10 +155,10 @@ func CompressLevel(dst, src []byte, level int) ([]byte, error) {
func Decompress(dst, src []byte) ([]byte, error) {
decompress := func(dst, src []byte) ([]byte, error) {

cWritten := C.ZSTD_decompress(
unsafe.Pointer(&dst[0]),
cWritten := C.ZSTD_decompress_wrapper(
C.uintptr_t(uintptr(unsafe.Pointer(&dst[0]))),
C.size_t(len(dst)),
unsafe.Pointer(&src[0]),
C.uintptr_t(uintptr(unsafe.Pointer(&src[0]))),
C.size_t(len(src)))

written := int(cWritten)
Expand Down

0 comments on commit e068f47

Please sign in to comment.