Skip to content

Commit

Permalink
Remove fastpath and unsafe variants (#14)
Browse files Browse the repository at this point in the history
* Always use safe mode (remove not unsafe)
* Run gofmt
* Fix format check
* Clean up go.mod and go.sum
  • Loading branch information
swenson authored Dec 5, 2022
1 parent cce962e commit 948591c
Show file tree
Hide file tree
Showing 55 changed files with 25,241 additions and 38,494 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ jobs:
- uses: actions/setup-go@v3
with:
go-version: ${{ needs.get-go-version.outputs.go-version }}
- run: '[ "$(gofmt -l . | wc -l)" -gt 0 ] && false'
- run: 'exit $(( $(gofmt -s -l . | wc -l) != 0 ))'

test:
name: codec test
Expand Down
24 changes: 1 addition & 23 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ This repository contains the `go-codec` library.
To install:

```
go get github.com/ugorji/go/codec
go get github.com/hashicorp/go-msgpack/codec
```

# Package Documentation
Expand Down Expand Up @@ -242,28 +242,6 @@ You can run the tag 'safe' to run tests or build in safe mode. e.g.
Please see http://github.com/ugorji/go-codec-bench .


## Managing Binary Size

This package could add up to 10MB to the size of your binaries.

This is because we include some a auto-generated file:
`fast-path.generated.go` to help with performance when encoding/decoding
slices and maps of built in numeric, boolean, string and interface{} types.

You can override this by building (or running tests and benchmarks) with the
tag: `notfastpath`.

```
go install -tags notfastpath
go build -tags notfastpath
go test -tags notfastpath
```

Be aware that, at least in our representative microbenchmarks for cbor (for
example), we see up to 33% increase in decoding and 50% increase in encoding
speeds. YMMV.


## Caveats

Struct fields matching the following are ignored during encoding and
Expand Down
2 changes: 1 addition & 1 deletion codec/bench/bench.sh
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ _gen() {
#
_suite() {
local t="alltests x"
local a=( "" "safe" "notfastpath" "notfastpath safe" "codecgen" "codecgen safe")
local a=( "" "safe" "codecgen" "codecgen safe")
local b=( "generated" "generated safe")
for i in "${a[@]}"
do
Expand Down
3 changes: 1 addition & 2 deletions codec/bench/doc.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,5 +10,4 @@ Here, we have the benchmark files comparing against other encoding libraries.
See README.md for more details.
*/
package codec

package codec
2 changes: 1 addition & 1 deletion codec/bench/shared_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ import (
"testing"
)

import . "github.com/ugorji/go/codec"
import . "github.com/hashicorp/go-msgpack/v2/codec"

type testHED struct {
H Handle
Expand Down
4 changes: 2 additions & 2 deletions codec/bench/x_bench_gen_test.go
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// +build x
// +build generated
//go:build x && generated
// +build x,generated

// Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved.
// Use of this source code is governed by a MIT license found in the LICENSE file.
Expand Down
1 change: 1 addition & 0 deletions codec/bench/x_bench_test.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
//go:build x
// +build x

// Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved.
Expand Down
1 change: 1 addition & 0 deletions codec/bench/z_all_bench_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved.
// Use of this source code is governed by a MIT license found in the LICENSE file.

//go:build (alltests || codecgen) && go1.7
// +build alltests codecgen
// +build go1.7

Expand Down
6 changes: 2 additions & 4 deletions codec/bench/z_all_x_bench_gen_test.go
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
// Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved.
// Use of this source code is governed by a MIT license found in the LICENSE file.

// +build alltests
// +build x
// +build go1.7
// +build generated
//go:build alltests && x && go1.7 && generated
// +build alltests,x,go1.7,generated

package codec

Expand Down
5 changes: 2 additions & 3 deletions codec/bench/z_all_x_bench_test.go
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
// Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved.
// Use of this source code is governed by a MIT license found in the LICENSE file.

// +build alltests
// +build x
// +build go1.7
//go:build alltests && x && go1.7
// +build alltests,x,go1.7

package codec

Expand Down
91 changes: 45 additions & 46 deletions codec/binc.go
Original file line number Diff line number Diff line change
Expand Up @@ -970,18 +970,18 @@ func (d *bincDecDriver) DecodeNaked() {

//------------------------------------

//BincHandle is a Handle for the Binc Schema-Free Encoding Format
//defined at https://github.com/ugorji/binc .
// BincHandle is a Handle for the Binc Schema-Free Encoding Format
// defined at https://github.com/ugorji/binc .
//
//BincHandle currently supports all Binc features with the following EXCEPTIONS:
// - only integers up to 64 bits of precision are supported.
// big integers are unsupported.
// - Only IEEE 754 binary32 and binary64 floats are supported (ie Go float32 and float64 types).
// extended precision and decimal IEEE 754 floats are unsupported.
// - Only UTF-8 strings supported.
// Unicode_Other Binc types (UTF16, UTF32) are currently unsupported.
// BincHandle currently supports all Binc features with the following EXCEPTIONS:
// - only integers up to 64 bits of precision are supported.
// big integers are unsupported.
// - Only IEEE 754 binary32 and binary64 floats are supported (ie Go float32 and float64 types).
// extended precision and decimal IEEE 754 floats are unsupported.
// - Only UTF-8 strings supported.
// Unicode_Other Binc types (UTF16, UTF32) are currently unsupported.
//
//Note that these EXCEPTIONS are temporary and full support is possible and may happen soon.
// Note that these EXCEPTIONS are temporary and full support is possible and may happen soon.
type BincHandle struct {
BasicHandle
binaryEncodingType
Expand Down Expand Up @@ -1046,50 +1046,49 @@ func (d *bincDecDriver) reset() {
//
// Format Description
//
// A timestamp is composed of 3 components:
// A timestamp is composed of 3 components:
//
// - secs: signed integer representing seconds since unix epoch
// - nsces: unsigned integer representing fractional seconds as a
// nanosecond offset within secs, in the range 0 <= nsecs < 1e9
// - tz: signed integer representing timezone offset in minutes east of UTC,
// and a dst (daylight savings time) flag
// - secs: signed integer representing seconds since unix epoch
// - nsces: unsigned integer representing fractional seconds as a
// nanosecond offset within secs, in the range 0 <= nsecs < 1e9
// - tz: signed integer representing timezone offset in minutes east of UTC,
// and a dst (daylight savings time) flag
//
// When encoding a timestamp, the first byte is the descriptor, which
// defines which components are encoded and how many bytes are used to
// encode secs and nsecs components. *If secs/nsecs is 0 or tz is UTC, it
// is not encoded in the byte array explicitly*.
// When encoding a timestamp, the first byte is the descriptor, which
// defines which components are encoded and how many bytes are used to
// encode secs and nsecs components. *If secs/nsecs is 0 or tz is UTC, it
// is not encoded in the byte array explicitly*.
//
// Descriptor 8 bits are of the form `A B C DDD EE`:
// A: Is secs component encoded? 1 = true
// B: Is nsecs component encoded? 1 = true
// C: Is tz component encoded? 1 = true
// DDD: Number of extra bytes for secs (range 0-7).
// If A = 1, secs encoded in DDD+1 bytes.
// If A = 0, secs is not encoded, and is assumed to be 0.
// If A = 1, then we need at least 1 byte to encode secs.
// DDD says the number of extra bytes beyond that 1.
// E.g. if DDD=0, then secs is represented in 1 byte.
// if DDD=2, then secs is represented in 3 bytes.
// EE: Number of extra bytes for nsecs (range 0-3).
// If B = 1, nsecs encoded in EE+1 bytes (similar to secs/DDD above)
// Descriptor 8 bits are of the form `A B C DDD EE`:
// A: Is secs component encoded? 1 = true
// B: Is nsecs component encoded? 1 = true
// C: Is tz component encoded? 1 = true
// DDD: Number of extra bytes for secs (range 0-7).
// If A = 1, secs encoded in DDD+1 bytes.
// If A = 0, secs is not encoded, and is assumed to be 0.
// If A = 1, then we need at least 1 byte to encode secs.
// DDD says the number of extra bytes beyond that 1.
// E.g. if DDD=0, then secs is represented in 1 byte.
// if DDD=2, then secs is represented in 3 bytes.
// EE: Number of extra bytes for nsecs (range 0-3).
// If B = 1, nsecs encoded in EE+1 bytes (similar to secs/DDD above)
//
// Following the descriptor bytes, subsequent bytes are:
// Following the descriptor bytes, subsequent bytes are:
//
// secs component encoded in `DDD + 1` bytes (if A == 1)
// nsecs component encoded in `EE + 1` bytes (if B == 1)
// tz component encoded in 2 bytes (if C == 1)
// secs component encoded in `DDD + 1` bytes (if A == 1)
// nsecs component encoded in `EE + 1` bytes (if B == 1)
// tz component encoded in 2 bytes (if C == 1)
//
// secs and nsecs components are integers encoded in a BigEndian
// 2-complement encoding format.
// secs and nsecs components are integers encoded in a BigEndian
// 2-complement encoding format.
//
// tz component is encoded as 2 bytes (16 bits). Most significant bit 15 to
// Least significant bit 0 are described below:
//
// Timezone offset has a range of -12:00 to +14:00 (ie -720 to +840 minutes).
// Bit 15 = have\_dst: set to 1 if we set the dst flag.
// Bit 14 = dst\_on: set to 1 if dst is in effect at the time, or 0 if not.
// Bits 13..0 = timezone offset in minutes. It is a signed integer in Big Endian format.
// tz component is encoded as 2 bytes (16 bits). Most significant bit 15 to
// Least significant bit 0 are described below:
//
// Timezone offset has a range of -12:00 to +14:00 (ie -720 to +840 minutes).
// Bit 15 = have\_dst: set to 1 if we set the dst flag.
// Bit 14 = dst\_on: set to 1 if dst is in effect at the time, or 0 if not.
// Bits 13..0 = timezone offset in minutes. It is a signed integer in Big Endian format.
func bincEncodeTime(t time.Time) []byte {
//t := rv.Interface().(time.Time)
tsecs, tnsecs := t.Unix(), t.Nanosecond()
Expand Down
46 changes: 21 additions & 25 deletions codec/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,7 @@

_tests() {
local gover=$( go version | cut -f 3 -d ' ' )
# note that codecgen requires fastpath, so you cannot do "codecgen notfastpath"
local a=( "" "safe" "notfastpath" "notfastpath safe" "codecgen" "codecgen safe" )
local a=( "" "codecgen" )
for i in "${a[@]}"
do
echo ">>>> TAGS: $i"
Expand All @@ -17,7 +16,7 @@ _tests() {
*) go vet -printfuncs "errorf" "$@" &&
go test ${zargs[*]} -vet off -tags "alltests $i" -run "Suite" -coverprofile "${i2// /-}.cov.out" "$@" ;;
esac
if [[ "$?" != 0 ]]; then return 1; fi
if [[ "$?" != 0 ]]; then return 1; fi
done
echo "++++++++ TEST SUITES ALL PASSED ++++++++"
}
Expand All @@ -26,10 +25,10 @@ _tests() {
# is a generation needed?
_ng() {
local a="$1"
if [[ ! -e "$a" ]]; then echo 1; return; fi
if [[ ! -e "$a" ]]; then echo 1; return; fi
for i in `ls -1 *.go.tmpl gen.go values_test.go`
do
if [[ "$a" -ot "$i" ]]; then echo 1; return; fi
if [[ "$a" -ot "$i" ]]; then echo 1; return; fi
done
}

Expand All @@ -42,19 +41,18 @@ EOF
rm -f ${1}
}

# _build generates fast-path.go and gen-helper.go.
# _build generates gen-helper.go.
_build() {
if ! [[ "${zforce}" || $(_ng "fast-path.generated.go") || $(_ng "gen-helper.generated.go") || $(_ng "gen.generated.go") ]]; then return 0; fi
if ! [[ "${zforce}" || $(_ng "gen-helper.generated.go") || $(_ng "gen.generated.go") ]]; then return 0; fi

if [ "${zbak}" ]; then
_zts=`date '+%m%d%Y_%H%M%S'`
_gg=".generated.go"
[ -e "gen-helper${_gg}" ] && mv gen-helper${_gg} gen-helper${_gg}__${_zts}.bak
[ -e "fast-path${_gg}" ] && mv fast-path${_gg} fast-path${_gg}__${_zts}.bak
[ -e "gen${_gg}" ] && mv gen${_gg} gen${_gg}__${_zts}.bak
fi
rm -f gen-helper.generated.go fast-path.generated.go gen.generated.go \
*safe.generated.go *_generated_test.go *.generated_ffjson_expose.go
fi
rm -f gen-helper.generated.go gen.generated.go \
*_generated_test.go *.generated_ffjson_expose.go

cat > gen.generated.go <<EOF
// +build codecgen.exec
Expand Down Expand Up @@ -85,7 +83,7 @@ EOF
\`
EOF
cat > gen-from-tmpl.codec.generated.go <<EOF
package codec
package codec
import "io"
func GenInternalGoFile(r io.Reader, w io.Writer) error {
return genInternalGoFile(r, w)
Expand All @@ -112,18 +110,17 @@ if err != nil { panic(err) }
}
func main() {
run("fast-path.go.tmpl", "fast-path.generated.go")
run("gen-helper.go.tmpl", "gen-helper.generated.go")
run("mammoth-test.go.tmpl", "mammoth_generated_test.go")
run("mammoth2-test.go.tmpl", "mammoth2_generated_test.go")
}
EOF

sed -e 's+// __DO_NOT_REMOVE__NEEDED_FOR_REPLACING__IMPORT_PATH__FOR_CODEC_BENCH__+import . "github.com/ugorji/go/codec"+' \
sed -e 's+// __DO_NOT_REMOVE__NEEDED_FOR_REPLACING__IMPORT_PATH__FOR_CODEC_BENCH__+import . "github.com/hashicorp/go-msgpack/v2/codec"+' \
shared_test.go > bench/shared_test.go

# explicitly return 0 if this passes, else return 1
go run -tags "notfastpath safe codecgen.exec" gen-from-tmpl.generated.go &&
go run -tags "codecgen.exec" gen-from-tmpl.generated.go &&
rm -f gen-from-tmpl.*generated.go &&
return 0
return 1
Expand All @@ -136,7 +133,7 @@ _codegenerators() {
local c9="codecgen-scratch.go"

if ! [[ $zforce || $(_ng "values_codecgen${c5}") ]]; then return 0; fi

# Note: ensure you run the codecgen for this codebase/directory i.e. ./codecgen/codecgen
true &&
echo "codecgen ... " &&
Expand All @@ -145,20 +142,20 @@ _codegenerators() {
fi &&
$c8 -rt codecgen -t 'codecgen generated' -o values_codecgen${c5} -d 19780 $zfin $zfin2 &&
cp mammoth2_generated_test.go $c9 &&
$c8 -t '!notfastpath' -o mammoth2_codecgen${c5} -d 19781 mammoth2_generated_test.go &&
$c8 -o mammoth2_codecgen${c5} -d 19781 mammoth2_generated_test.go &&
rm -f $c9 &&
echo "generators done!"
echo "generators done!"
}

_prebuild() {
echo "prebuild: zforce: $zforce"
local d="$PWD"
zfin="test_values.generated.go"
zfin2="test_values_flex.generated.go"
zpkg="github.com/ugorji/go/codec"
zpkg="github.com/hashicorp/go-msgpack/v2/codec"
# zpkg=${d##*/src/}
# zgobase=${d%%/src/*}
# rm -f *_generated_test.go
# rm -f *_generated_test.go
rm -f codecgen-*.go &&
_build &&
cp $d/values_test.go $d/$zfin &&
Expand Down Expand Up @@ -221,7 +218,7 @@ EOF

_usage() {
cat <<EOF
primary usage: $0
primary usage: $0
-[tmpfxnld] -> [tests, make, prebuild (force) (external), inlining diagnostics, mid-stack inlining, race detector]
-v -> verbose
EOF
Expand Down Expand Up @@ -263,5 +260,4 @@ _main() {
unset zforce zargs zbenchflags
}

[ "." = `dirname $0` ] && _main "$@"

[ "." = `dirname $0` ] && _main "$@"
1 change: 1 addition & 0 deletions codec/codecgen.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
//go:build codecgen || generated
// +build codecgen generated

package codec
Expand Down
Loading

0 comments on commit 948591c

Please sign in to comment.