diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 770571d..8bcee64 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2024-05-27T12:37:27Z by kres b5844f8. +# Generated on 2024-12-09T17:41:43Z by kres 8183c20. name: default concurrency: @@ -77,7 +77,7 @@ jobs: run: | make unit-tests-race - name: coverage - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@v5 with: files: _out/coverage-unit-tests.txt token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/slack-notify.yaml b/.github/workflows/slack-notify.yaml index c0dca32..10e3411 100644 --- a/.github/workflows/slack-notify.yaml +++ b/.github/workflows/slack-notify.yaml @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2024-03-11T19:57:58Z by kres latest. +# Generated on 2024-12-09T17:41:43Z by kres 8183c20. name: slack-notify "on": @@ -24,11 +24,12 @@ jobs: run: | echo pull_request_number=$(gh pr view -R ${{ github.repository }} ${{ github.event.workflow_run.head_repository.owner.login }}:${{ github.event.workflow_run.head_branch }} --json number --jq .number) >> $GITHUB_OUTPUT - name: Slack Notify - uses: slackapi/slack-github-action@v1 + uses: slackapi/slack-github-action@v2 with: - channel-id: proj-talos-maintainers + method: chat.postMessage payload: | { + "channel": "proj-talos-maintainers", "attachments": [ { "color": "${{ github.event.workflow_run.conclusion == 'success' && '#2EB886' || github.event.workflow_run.conclusion == 'failure' && '#A30002' || '#FFCC00' }}", @@ -88,5 +89,4 @@ jobs: } ] } - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + token: ${{ secrets.SLACK_BOT_TOKEN }} diff --git a/.golangci.yml b/.golangci.yml index 9705cf7..889e80e 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2024-10-17T09:56:58Z by kres 34e72ac. +# Generated on 2024-12-09T17:41:43Z by kres 8183c20. # options for analysis running run: @@ -116,7 +116,6 @@ linters: - gochecknoglobals - gochecknoinits - godox - - gomnd - gomoddirectives - gosec - inamedparam diff --git a/Dockerfile b/Dockerfile index 9189780..54dbf0c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ -# syntax = docker/dockerfile-upstream:1.10.0-labs +# syntax = docker/dockerfile-upstream:1.12.0-labs # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2024-10-23T16:30:37Z by kres 6d3cad4. +# Generated on 2024-12-09T17:41:43Z by kres 8183c20. ARG TOOLCHAIN @@ -10,9 +10,9 @@ ARG TOOLCHAIN FROM scratch AS generate # runs markdownlint -FROM docker.io/oven/bun:1.1.32-alpine AS lint-markdown +FROM docker.io/oven/bun:1.1.38-alpine AS lint-markdown WORKDIR /src -RUN bun i markdownlint-cli@0.42.0 sentences-per-line@0.2.1 +RUN bun i markdownlint-cli@0.43.0 sentences-per-line@0.2.1 COPY .markdownlint.json . COPY ./README.md ./README.md RUN bunx markdownlint --ignore "CHANGELOG.md" --ignore "**/node_modules/**" --ignore '**/hack/chglog/**' --rules node_modules/sentences-per-line/index.js . diff --git a/Makefile b/Makefile index 213545e..fe5cb0f 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2024-10-23T16:30:37Z by kres 6d3cad4. +# Generated on 2024-12-09T17:41:43Z by kres 8183c20. # common variables @@ -17,15 +17,15 @@ WITH_RACE ?= false REGISTRY ?= ghcr.io USERNAME ?= siderolabs REGISTRY_AND_USERNAME ?= $(REGISTRY)/$(USERNAME) -PROTOBUF_GO_VERSION ?= 1.35.1 +PROTOBUF_GO_VERSION ?= 1.35.2 GRPC_GO_VERSION ?= 1.5.1 -GRPC_GATEWAY_VERSION ?= 2.22.0 +GRPC_GATEWAY_VERSION ?= 2.24.0 VTPROTOBUF_VERSION ?= 0.6.0 -GOIMPORTS_VERSION ?= 0.26.0 +GOIMPORTS_VERSION ?= 0.28.0 DEEPCOPY_VERSION ?= v0.5.6 -GOLANGCILINT_VERSION ?= v1.61.0 +GOLANGCILINT_VERSION ?= v1.62.2 GOFUMPT_VERSION ?= v0.7.0 -GO_VERSION ?= 1.23.2 +GO_VERSION ?= 1.23.4 GO_BUILDFLAGS ?= GO_LDFLAGS ?= CGO_ENABLED ?= 0 @@ -41,11 +41,13 @@ PLATFORM ?= linux/amd64 PROGRESS ?= auto PUSH ?= false CI_ARGS ?= +BUILDKIT_MULTI_PLATFORM ?= 1 COMMON_ARGS = --file=Dockerfile COMMON_ARGS += --provenance=false COMMON_ARGS += --progress=$(PROGRESS) COMMON_ARGS += --platform=$(PLATFORM) COMMON_ARGS += --push=$(PUSH) +COMMON_ARGS += --build-arg=BUILDKIT_MULTI_PLATFORM=$(BUILDKIT_MULTI_PLATFORM) COMMON_ARGS += --build-arg=ARTIFACTS="$(ARTIFACTS)" COMMON_ARGS += --build-arg=SHA="$(SHA)" COMMON_ARGS += --build-arg=TAG="$(TAG)" @@ -145,6 +147,15 @@ target-%: ## Builds the specified target defined in the Dockerfile. The build r local-%: ## Builds the specified target defined in the Dockerfile using the local output type. The build result will be output to the specified local destination. @$(MAKE) target-$* TARGET_ARGS="--output=type=local,dest=$(DEST) $(TARGET_ARGS)" + @PLATFORM=$(PLATFORM) DEST=$(DEST) bash -c '\ + for platform in $$(tr "," "\n" <<< "$$PLATFORM"); do \ + echo $$platform; \ + directory="$${platform//\//_}"; \ + if [[ -d "$$DEST/$$directory" ]]; then \ + mv -f "$$DEST/$$directory/"* $$DEST; \ + rmdir "$$DEST/$$directory/"; \ + fi; \ + done' lint-golangci-lint: ## Runs golangci-lint linter. @$(MAKE) target-$@ diff --git a/concurrent/export_test.go b/concurrent/export_test.go new file mode 100644 index 0000000..b5dc727 --- /dev/null +++ b/concurrent/export_test.go @@ -0,0 +1,47 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package concurrent + +import ( + "unsafe" +) + +// NewBadHashTrieMap creates a new HashTrieMap for the provided key and value +// but with an intentionally bad hash function. +func NewBadHashTrieMap[K, V comparable]() *HashTrieMap[K, V] { + // Stub out the good hash function with a terrible one. + // Everything should still work as expected. + var m HashTrieMap[K, V] + + m.init() + + m.keyHash = func(_ unsafe.Pointer, _ uintptr) uintptr { + return 0 + } + + return &m +} + +// NewTruncHashTrieMap creates a new HashTrieMap for the provided key and value +// but with an intentionally bad hash function. +func NewTruncHashTrieMap[K, V comparable]() *HashTrieMap[K, V] { + // Stub out the good hash function with a terrible one. + // Everything should still work as expected. + var ( + m HashTrieMap[K, V] + mx map[string]int + ) + + hasher := efaceMapOf(mx)._type.Hasher + m.keyHash = func(p unsafe.Pointer, n uintptr) uintptr { + return hasher(p, n) & ((uintptr(1) << 4) - 1) + } + + return &m +} diff --git a/concurrent/go122.go b/concurrent/go122.go index 0df49d8..30ee9d6 100644 --- a/concurrent/go122.go +++ b/concurrent/go122.go @@ -8,28 +8,15 @@ //go:build go1.22 && !go1.24 && !nomaptypehash -//nolint:revive,govet,stylecheck,nlreturn,wsl package concurrent import ( - "math/rand/v2" "unsafe" ) -// NewHashTrieMap creates a new HashTrieMap for the provided key and value. -func NewHashTrieMap[K, V comparable]() *HashTrieMap[K, V] { - var m map[K]V - - mapType := efaceMapOf(m) - ht := &HashTrieMap[K, V]{ - root: newIndirectNode[K, V](nil), - keyHash: mapType._type.Hasher, - seed: uintptr(rand.Uint64()), - } - return ht -} - // _MapType is runtime.maptype from runtime/type.go. +// +//nolint:govet type _MapType struct { _Type Key *_Type @@ -44,6 +31,8 @@ type _MapType struct { } // _Type is runtime._type from runtime/type.go. +// +//nolint:govet,revive type _Type struct { Size_ uintptr PtrBytes uintptr // number of (prefix) bytes in the type that can contain pointers diff --git a/concurrent/go122_bench_test.go b/concurrent/go122_bench_test.go deleted file mode 100644 index 7c9573b..0000000 --- a/concurrent/go122_bench_test.go +++ /dev/null @@ -1,71 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at http://mozilla.org/MPL/2.0/. - -// Copyright 2024 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build go1.22 && !go1.24 && !nomaptypehash - -//nolint:wsl,testpackage -package concurrent - -import ( - "testing" -) - -func BenchmarkHashTrieMapLoadSmall(b *testing.B) { - benchmarkHashTrieMapLoad(b, testDataSmall[:]) -} - -func BenchmarkHashTrieMapLoad(b *testing.B) { - benchmarkHashTrieMapLoad(b, testData[:]) -} - -func BenchmarkHashTrieMapLoadLarge(b *testing.B) { - benchmarkHashTrieMapLoad(b, testDataLarge[:]) -} - -func benchmarkHashTrieMapLoad(b *testing.B, data []string) { - b.ReportAllocs() - m := NewHashTrieMap[string, int]() - for i := range data { - m.LoadOrStore(data[i], i) - } - b.ResetTimer() - b.RunParallel(func(pb *testing.PB) { - i := 0 - for pb.Next() { - _, _ = m.Load(data[i]) - i++ - if i >= len(data) { - i = 0 - } - } - }) -} - -func BenchmarkHashTrieMapLoadOrStore(b *testing.B) { - benchmarkHashTrieMapLoadOrStore(b, testData[:]) -} - -func BenchmarkHashTrieMapLoadOrStoreLarge(b *testing.B) { - benchmarkHashTrieMapLoadOrStore(b, testDataLarge[:]) -} - -func benchmarkHashTrieMapLoadOrStore(b *testing.B, data []string) { - b.ReportAllocs() - m := NewHashTrieMap[string, int]() - - b.RunParallel(func(pb *testing.PB) { - i := 0 - for pb.Next() { - _, _ = m.LoadOrStore(data[i], i) - i++ - if i >= len(data) { - i = 0 - } - } - }) -} diff --git a/concurrent/go122_test.go b/concurrent/go122_test.go deleted file mode 100644 index ba00179..0000000 --- a/concurrent/go122_test.go +++ /dev/null @@ -1,35 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at http://mozilla.org/MPL/2.0/. - -// Copyright 2024 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build go1.22 && !go1.24 && !nomaptypehash - -//nolint:nlreturn,wsl,testpackage -package concurrent - -import ( - "testing" - "unsafe" -) - -func TestHashTrieMap(t *testing.T) { - testHashTrieMap(t, func() *HashTrieMap[string, int] { - return NewHashTrieMap[string, int]() - }) -} - -func TestHashTrieMapBadHash(t *testing.T) { - testHashTrieMap(t, func() *HashTrieMap[string, int] { - // Stub out the good hash function with a terrible one. - // Everything should still work as expected. - m := NewHashTrieMap[string, int]() - m.keyHash = func(_ unsafe.Pointer, _ uintptr) uintptr { - return 0 - } - return m - }) -} diff --git a/concurrent/go124.go b/concurrent/go124.go new file mode 100644 index 0000000..912df98 --- /dev/null +++ b/concurrent/go124.go @@ -0,0 +1,81 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.24 && !go1.25 && !nomaptypehash + +package concurrent + +import ( + "unsafe" +) + +// _MapType is runtime.maptype [abi.SwissMapType] from runtime/type.go. +// +//nolint:govet +type _MapType struct { + _Type + Key *_Type + Elem *_Type + Group *_Type // internal type representing a slot group + // function for hashing keys (ptr to key, seed) -> hash + Hasher func(unsafe.Pointer, uintptr) uintptr + GroupSize uintptr // == Group.Size_ + SlotSize uintptr // size of key/elem slot + ElemOff uintptr // offset of elem in key/elem slot + Flags uint32 +} + +// _Type is runtime._type from runtime/type.go. +// +//nolint:govet,revive +type _Type struct { + Size_ uintptr + PtrBytes uintptr // number of (prefix) bytes in the type that can contain pointers + Hash uint32 // hash of type; avoids computation in hash tables + TFlag _TFlag // extra type information flags + Align_ uint8 // alignment of variable with this type + FieldAlign_ uint8 // alignment of struct field with this type + Kind_ _Kind // enumeration for C + // function for comparing objects of this type + // (ptr to object A, ptr to object B) -> ==? + Equal func(unsafe.Pointer, unsafe.Pointer) bool + // GCData stores the GC type data for the garbage collector. + // Normally, GCData points to a bitmask that describes the + // ptr/nonptr fields of the type. The bitmask will have at + // least PtrBytes/ptrSize bits. + // If the TFlagGCMaskOnDemand bit is set, GCData is instead a + // **byte and the pointer to the bitmask is one dereference away. + // The runtime will build the bitmask if needed. + // (See runtime/type.go:getGCMask.) + // Note: multiple types may have the same value of GCData, + // including when TFlagGCMaskOnDemand is set. The types will, of course, + // have the same pointer layout (but not necessarily the same size). + GCData *byte + Str _NameOff // string form + PtrToThis _TypeOff // type for pointer to this type, may be zero +} + +// _TypeOff is the offset to a type from moduledata.types. See resolveTypeOff in runtime. +type _TypeOff int32 + +// _NameOff is the offset to a name from moduledata.types. See resolveNameOff in runtime. +type _NameOff int32 + +type _Kind uint8 + +type _TFlag uint8 + +// efaceMap is runtime.eface from runtime/runtime2.go. +type efaceMap struct { + _type *_MapType + data unsafe.Pointer +} + +func efaceMapOf(ep any) *efaceMap { + return (*efaceMap)(unsafe.Pointer(&ep)) +} diff --git a/concurrent/hashtriemap.go b/concurrent/hashtriemap.go index 1c6b0f3..206f021 100644 --- a/concurrent/hashtriemap.go +++ b/concurrent/hashtriemap.go @@ -6,7 +6,9 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//nolint:revive,govet,nakedret,nlreturn,stylecheck,wsl,staticcheck,unused +// Package concurrent provides hash-trie implementation for concurrent use. +// +//nolint:govet,nakedret,nlreturn,predeclared,revive,staticcheck,unused,wastedassign,wsl package concurrent import ( @@ -17,27 +19,51 @@ import ( "unsafe" ) +// NewHashTrieMap creates a new HashTrieMap for the provided key and value. +func NewHashTrieMap[K, V comparable]() *HashTrieMap[K, V] { + return &HashTrieMap[K, V]{} +} + // HashTrieMap is an implementation of a concurrent hash-trie. The implementation // is designed around frequent loads, but offers decent performance for stores -// and deletes as well, especially if the map is larger. It's primary use-case is +// and deletes as well, especially if the map is larger. Its primary use-case is // the unique package, but can be used elsewhere as well. +// +// The zero HashTrieMap is empty and ready to use. +// It must not be copied after first use. type HashTrieMap[K, V comparable] struct { - root *indirect[K, V] + inited atomic.Uint32 + initMu sync.Mutex + root atomic.Pointer[indirect[K, V]] keyHash hashFunc seed uintptr } -// NewHashTrieMapHasher creates a new HashTrieMap for the provided key and value and uses -// the provided hasher function to hash keys. -func NewHashTrieMapHasher[K, V comparable](hasher func(*K, maphash.Seed) uintptr) *HashTrieMap[K, V] { - ht := &HashTrieMap[K, V]{ - root: newIndirectNode[K, V](nil), - keyHash: func(pointer unsafe.Pointer, u uintptr) uintptr { - return hasher((*K)(pointer), seedToSeed(u)) - }, - seed: uintptr(rand.Uint64()), +func (ht *HashTrieMap[K, V]) init() { + if ht.inited.Load() == 0 { + ht.initSlow() } - return ht +} + +//go:noinline +func (ht *HashTrieMap[K, V]) initSlow() { + ht.initMu.Lock() + defer ht.initMu.Unlock() + + if ht.inited.Load() != 0 { + // Someone got to it while we were waiting. + return + } + + // Set up root node, derive the hash function for the key, and the + // equal function for the value, if any. + var m map[K]V + mapType := efaceMapOf(m)._type + ht.root.Store(newIndirectNode[K, V](nil)) + ht.keyHash = mapType.Hasher + ht.seed = uintptr(rand.Uint64()) + + ht.inited.Store(1) } type hashFunc func(unsafe.Pointer, uintptr) uintptr @@ -46,9 +72,10 @@ type hashFunc func(unsafe.Pointer, uintptr) uintptr // value is present. // The ok result indicates whether value was found in the map. func (ht *HashTrieMap[K, V]) Load(key K) (value V, ok bool) { + ht.init() hash := ht.keyHash(noescape(unsafe.Pointer(&key)), ht.seed) - i := ht.root + i := ht.root.Load() hashShift := 8 * ptrSize for hashShift != 0 { hashShift -= nChildrenLog2 @@ -69,6 +96,7 @@ func (ht *HashTrieMap[K, V]) Load(key K) (value V, ok bool) { // Otherwise, it stores and returns the given value. // The loaded result is true if the value was loaded, false if stored. func (ht *HashTrieMap[K, V]) LoadOrStore(key K, value V) (result V, loaded bool) { + ht.init() hash := ht.keyHash(noescape(unsafe.Pointer(&key)), ht.seed) var i *indirect[K, V] var hashShift uint @@ -76,8 +104,9 @@ func (ht *HashTrieMap[K, V]) LoadOrStore(key K, value V) (result V, loaded bool) var n *node[K, V] for { // Find the key or a candidate location for insertion. - i = ht.root + i = ht.root.Load() hashShift = 8 * ptrSize + haveInsertPoint := false for hashShift != 0 { hashShift -= nChildrenLog2 @@ -85,6 +114,7 @@ func (ht *HashTrieMap[K, V]) LoadOrStore(key K, value V) (result V, loaded bool) n = slot.Load() if n == nil { // We found a nil slot which is a candidate for insertion. + haveInsertPoint = true break } if n.isEntry { @@ -94,11 +124,12 @@ func (ht *HashTrieMap[K, V]) LoadOrStore(key K, value V) (result V, loaded bool) if v, ok := n.entry().lookup(key); ok { return v, true } + haveInsertPoint = true break } i = n.indirect() } - if hashShift == 0 { + if !haveInsertPoint { panic("internal/concurrent.HashMapTrie: ran out of hash bits while iterating") } @@ -174,61 +205,200 @@ func (ht *HashTrieMap[K, V]) expand(oldEntry, newEntry *entry[K, V], newHash uin return &top.node } -// CompareAndDelete deletes the entry for key if its value is equal to old. -// -// If there is no current value for key in the map, CompareAndDelete returns false -// (even if the old value is the nil interface value). -func (ht *HashTrieMap[K, V]) CompareAndDelete(key K, old V) (deleted bool) { +// Store sets the value for a key. +func (ht *HashTrieMap[K, V]) Store(key K, old V) { + _, _ = ht.Swap(key, old) +} + +// Swap swaps the value for a key and returns the previous value if any. +// The loaded result reports whether the key was present. +func (ht *HashTrieMap[K, V]) Swap(key K, new V) (previous V, loaded bool) { + ht.init() hash := ht.keyHash(noescape(unsafe.Pointer(&key)), ht.seed) var i *indirect[K, V] var hashShift uint var slot *atomic.Pointer[node[K, V]] var n *node[K, V] for { - // Find the key or return when there's nothing to delete. - i = ht.root + // Find the key or a candidate location for insertion. + i = ht.root.Load() hashShift = 8 * ptrSize + haveInsertPoint := false for hashShift != 0 { hashShift -= nChildrenLog2 slot = &i.children[(hash>>hashShift)&nChildrenMask] n = slot.Load() if n == nil { - // Nothing to delete. Give up. - return + // We found a nil slot which is a candidate for insertion, + // or an existing entry that we'll replace. + haveInsertPoint = true + break } if n.isEntry { - // We found an entry. Check if it matches. - if _, ok := n.entry().lookup(key); !ok { - // No match, nothing to delete. - return + // Swap if the keys compare. + old, swapped := n.entry().swap(key, new) + if swapped { + return old, true } - // We've got something to delete. + // If we fail, that means we should try to insert. + haveInsertPoint = true break } i = n.indirect() } - if hashShift == 0 { + if !haveInsertPoint { panic("internal/concurrent.HashMapTrie: ran out of hash bits while iterating") } // Grab the lock and double-check what we saw. i.mu.Lock() n = slot.Load() - if !i.dead.Load() { + if (n == nil || n.isEntry) && !i.dead.Load() { + // What we saw is still true, so we can continue with the insert. + break + } + // We have to start over. + i.mu.Unlock() + } + // N.B. This lock is held from when we broke out of the outer loop above. + // We specifically break this out so that we can use defer here safely. + // One option is to break this out into a new function instead, but + // there's so much local iteration state used below that this turns out + // to be cleaner. + defer i.mu.Unlock() + + var zero V + var oldEntry *entry[K, V] + if n != nil { + // Between before and now, something got inserted. Swap if the keys compare. + oldEntry = n.entry() + old, swapped := oldEntry.swap(key, new) + if swapped { + return old, true + } + } + // The keys didn't compare, so we're doing an insertion. + newEntry := newEntryNode(key, new) + if oldEntry == nil { + // Easy case: create a new entry and store it. + slot.Store(&newEntry.node) + } else { + // We possibly need to expand the entry already there into one or more new nodes. + // + // Publish the node last, which will make both oldEntry and newEntry visible. We + // don't want readers to be able to observe that oldEntry isn't in the tree. + slot.Store(ht.expand(oldEntry, newEntry, hash, hashShift, i)) + } + return zero, false +} + +// CompareAndSwap swaps the old and new values for key +// if the value stored in the map is equal to old. +// The value type must be of a comparable type, otherwise CompareAndSwap will panic. +func (ht *HashTrieMap[K, V]) CompareAndSwap(key K, old, new V) (swapped bool) { + ht.init() + hash := ht.keyHash(noescape(unsafe.Pointer(&key)), ht.seed) + for { + // Find the key or return if it's not there. + i := ht.root.Load() + hashShift := 8 * ptrSize + found := false + for hashShift != 0 { + hashShift -= nChildrenLog2 + + slot := &i.children[(hash>>hashShift)&nChildrenMask] + n := slot.Load() if n == nil { - // Valid node that doesn't contain what we need. Nothing to delete. - i.mu.Unlock() - return + // Nothing to compare with. Give up. + return false } if n.isEntry { - // What we saw is still true, so we can continue with the delete. - break + // We found an entry. Try to compare and swap directly. + return n.entry().compareAndSwap(key, old, new) } + i = n.indirect() } - // We have to start over. + if !found { + panic("internal/concurrent.HashMapTrie: ran out of hash bits while iterating") + } + } +} + +// LoadAndDelete deletes the value for a key, returning the previous value if any. +// The loaded result reports whether the key was present. +func (ht *HashTrieMap[K, V]) LoadAndDelete(key K) (value V, loaded bool) { + ht.init() + hash := ht.keyHash(noescape(unsafe.Pointer(&key)), ht.seed) + + // Find a node with the key and compare with it. n != nil if we found the node. + i, hashShift, slot, n := ht.find(key, hash) + if n == nil { + if i != nil { + i.mu.Unlock() + } + return *new(V), false + } + + // Try to delete the entry. + v, e, loaded := n.entry().loadAndDelete(key) + if !loaded { + // Nothing was actually deleted, which means the node is no longer there. + i.mu.Unlock() + return *new(V), false + } + if e != nil { + // We didn't actually delete the whole entry, just one entry in the chain. + // Nothing else to do, since the parent is definitely not empty. + slot.Store(&e.node) + i.mu.Unlock() + return v, true + } + // Delete the entry. + slot.Store(nil) + + // Check if the node is now empty (and isn't the root), and delete it if able. + for i.parent != nil && i.empty() { + if hashShift == 8*ptrSize { + panic("internal/concurrent.HashMapTrie: ran out of hash bits while iterating") + } + hashShift += nChildrenLog2 + + // Delete the current node in the parent. + parent := i.parent + parent.mu.Lock() + i.dead.Store(true) + parent.children[(hash>>hashShift)&nChildrenMask].Store(nil) i.mu.Unlock() + i = parent } + i.mu.Unlock() + return v, true +} + +// Delete deletes the value for a key. +func (ht *HashTrieMap[K, V]) Delete(key K) { + _, _ = ht.LoadAndDelete(key) +} + +// CompareAndDelete deletes the entry for key if its value is equal to old. +// The value type must be comparable, otherwise this CompareAndDelete will panic. +// +// If there is no current value for key in the map, CompareAndDelete returns false +// (even if the old value is the nil interface value). +func (ht *HashTrieMap[K, V]) CompareAndDelete(key K, old V) (deleted bool) { + ht.init() + hash := ht.keyHash(noescape(unsafe.Pointer(&key)), ht.seed) + + // Find a node with the key. n != nil if we found the node. + i, hashShift, slot, n := ht.find(key, hash) + if n == nil { + if i != nil { + i.mu.Unlock() + } + return false + } + // Try to delete the entry. e, deleted := n.entry().compareAndDelete(key, old) if !deleted { @@ -248,7 +418,7 @@ func (ht *HashTrieMap[K, V]) CompareAndDelete(key K, old V) (deleted bool) { // Check if the node is now empty (and isn't the root), and delete it if able. for i.parent != nil && i.empty() { - if hashShift == 64 { + if hashShift == 8*ptrSize { panic("internal/concurrent.HashMapTrie: ran out of hash bits while iterating") } hashShift += nChildrenLog2 @@ -265,13 +435,82 @@ func (ht *HashTrieMap[K, V]) CompareAndDelete(key K, old V) (deleted bool) { return true } -// Enumerate produces all key-value pairs in the map. The enumeration does -// not represent any consistent snapshot of the map, but is guaranteed -// to visit each unique key-value pair only once. It is safe to operate -// on the tree during iteration. No particular enumeration order is -// guaranteed. -func (ht *HashTrieMap[K, V]) Enumerate(yield func(key K, value V) bool) { - ht.iter(ht.root, yield) +// find searches the tree for a node that contains key (hash must be the hash of key). +// If valEqual != nil, then it will also enforce that the values are equal as well. +// +// Returns a non-nil node, which will always be an entry, if found. +// +// If i != nil then i.mu is locked, and it is the caller's responsibility to unlock it. +func (ht *HashTrieMap[K, V]) find(key K, hash uintptr) (i *indirect[K, V], hashShift uint, slot *atomic.Pointer[node[K, V]], n *node[K, V]) { + for { + // Find the key or return if it's not there. + i = ht.root.Load() + hashShift = 8 * ptrSize + found := false + for hashShift != 0 { + hashShift -= nChildrenLog2 + + slot = &i.children[(hash>>hashShift)&nChildrenMask] + n = slot.Load() + if n == nil { + // Nothing to compare with. Give up. + i = nil + return + } + if n.isEntry { + // We found an entry. Check if it matches. + if _, ok := n.entry().lookupWithValue(key); !ok { + // No match, comparison failed. + i = nil + n = nil + return + } + // We've got a match. Prepare to perform an operation on the key. + found = true + break + } + i = n.indirect() + } + if !found { + panic("internal/concurrent.HashMapTrie: ran out of hash bits while iterating") + } + + // Grab the lock and double-check what we saw. + i.mu.Lock() + n = slot.Load() + if !i.dead.Load() && (n == nil || n.isEntry) { + // Either we've got a valid node or the node is now nil under the lock. + // In either case, we're done here. + return + } + // We have to start over. + i.mu.Unlock() + } +} + +// All returns an iterator over each key and value present in the map. +// +// The iterator does not necessarily correspond to any consistent snapshot of the +// HashTrieMap's contents: no key will be visited more than once, but if the value +// for any key is stored or deleted concurrently (including by yield), the iterator +// may reflect any mapping for that key from any point during iteration. The iterator +// does not block other methods on the receiver; even yield itself may call any +// method on the HashTrieMap. +func (ht *HashTrieMap[K, V]) All() func(yield func(K, V) bool) { + ht.init() + return func(yield func(key K, value V) bool) { + ht.iter(ht.root.Load(), yield) + } +} + +// Range calls f sequentially for each key and value present in the map. +// If f returns false, range stops the iteration. +// +// This exists for compatibility with sync.Map; All should be preferred. +// It provides the same guarantees as sync.Map, and All. +func (ht *HashTrieMap[K, V]) Range(yield func(K, V) bool) { + ht.init() + ht.iter(ht.root.Load(), yield) } func (ht *HashTrieMap[K, V]) iter(i *indirect[K, V], yield func(key K, value V) bool) bool { @@ -288,7 +527,7 @@ func (ht *HashTrieMap[K, V]) iter(i *indirect[K, V], yield func(key K, value V) } e := n.entry() for e != nil { - if !yield(e.key, e.value) { + if !yield(e.key, *e.value.Load()) { return false } e = e.overflow.Load() @@ -297,11 +536,20 @@ func (ht *HashTrieMap[K, V]) iter(i *indirect[K, V], yield func(key K, value V) return true } +// Clear deletes all the entries, resulting in an empty HashTrieMap. +func (ht *HashTrieMap[K, V]) Clear() { + ht.init() + + // It's sufficient to just drop the root on the floor, but the root + // must always be non-nil. + ht.root.Store(newIndirectNode[K, V](nil)) +} + const ( // 16 children. This seems to be the sweet spot for // load performance: any smaller and we lose out on // 50% or more in CPU performance. Any larger and the - // returns are miniscule (~1% improvement for 32 children). + // returns are minuscule (~1% improvement for 32 children). nChildrenLog2 = 4 nChildren = 1 << nChildrenLog2 nChildrenMask = nChildren - 1 @@ -335,40 +583,148 @@ type entry[K, V comparable] struct { node[K, V] overflow atomic.Pointer[entry[K, V]] // Overflow for hash collisions. key K - value V + value atomic.Pointer[V] } func newEntryNode[K, V comparable](key K, value V) *entry[K, V] { - return &entry[K, V]{ - node: node[K, V]{isEntry: true}, - key: key, - value: value, + e := &entry[K, V]{ + node: node[K, V]{isEntry: true}, + key: key, } + e.value.Store(&value) + return e } func (e *entry[K, V]) lookup(key K) (V, bool) { for e != nil { if e.key == key { - return e.value, true + return *e.value.Load(), true } e = e.overflow.Load() } return *new(V), false } +func (e *entry[K, V]) lookupWithValue(key K) (V, bool) { + for e != nil { + oldp := e.value.Load() + if e.key == key { + return *oldp, true + } + e = e.overflow.Load() + } + return *new(V), false +} + +// swap replaces a value in the overflow chain if keys compare equal. +// Returns the old value, and whether or not anything was swapped. +// +// swap must be called under the mutex of the indirect node which e is a child of. +func (head *entry[K, V]) swap(key K, newv V) (V, bool) { + if head.key == key { + vp := new(V) + *vp = newv + oldp := head.value.Swap(vp) + return *oldp, true + } + i := &head.overflow + e := i.Load() + for e != nil { + if e.key == key { + vp := new(V) + *vp = newv + oldp := e.value.Swap(vp) + return *oldp, true + } + i = &e.overflow + e = e.overflow.Load() + } + var zero V + return zero, false +} + +// compareAndSwap replaces a value for a matching key and existing value in the overflow chain. +// Returns whether or not anything was swapped. +// +// compareAndSwap must be called under the mutex of the indirect node which e is a child of. +func (head *entry[K, V]) compareAndSwap(key K, oldv, newv V) bool { + var vbox *V +outerLoop: + for { + oldvp := head.value.Load() + if head.key == key && *oldvp == oldv { + // Return the new head of the list. + if vbox == nil { + // Delay explicit creation of a new value to hold newv. If we just pass &newv + // to CompareAndSwap, then newv will unconditionally escape, even if the CAS fails. + vbox = new(V) + *vbox = newv + } + if head.value.CompareAndSwap(oldvp, vbox) { + return true + } + // We need to restart from the head of the overflow list in case, due to a removal, a node + // is moved up the list and we miss it. + continue outerLoop + } + i := &head.overflow + e := i.Load() + for e != nil { + oldvp := e.value.Load() + if e.key == key && *oldvp == oldv { + if vbox == nil { + // Delay explicit creation of a new value to hold newv. If we just pass &newv + // to CompareAndSwap, then newv will unconditionally escape, even if the CAS fails. + vbox = new(V) + *vbox = newv + } + if e.value.CompareAndSwap(oldvp, vbox) { + return true + } + continue outerLoop + } + i = &e.overflow + e = e.overflow.Load() + } + return false + } +} + +// loadAndDelete deletes an entry in the overflow chain by key. Returns the value for the key, the new +// entry chain and whether or not anything was loaded (and deleted). +// +// loadAndDelete must be called under the mutex of the indirect node which e is a child of. +func (head *entry[K, V]) loadAndDelete(key K) (V, *entry[K, V], bool) { + if head.key == key { + // Drop the head of the list. + return *head.value.Load(), head.overflow.Load(), true + } + i := &head.overflow + e := i.Load() + for e != nil { + if e.key == key { + i.Store(e.overflow.Load()) + return *e.value.Load(), head, true + } + i = &e.overflow + e = e.overflow.Load() + } + return *new(V), head, false +} + // compareAndDelete deletes an entry in the overflow chain if both the key and value compare // equal. Returns the new entry chain and whether or not anything was deleted. // // compareAndDelete must be called under the mutex of the indirect node which e is a child of. func (head *entry[K, V]) compareAndDelete(key K, value V) (*entry[K, V], bool) { - if head.key == key && head.value == value { + if head.key == key && *head.value.Load() == value { // Drop the head of the list. return head.overflow.Load(), true } i := &head.overflow e := i.Load() for e != nil { - if e.key == key && e.value == value { + if e.key == key && *e.value.Load() == value { i.Store(e.overflow.Load()) return head, true } @@ -422,7 +778,3 @@ func init() { panic("concurrent: seedType is not maphash.Seed") } } - -func seedToSeed(val uintptr) maphash.Seed { - return *(*maphash.Seed)(unsafe.Pointer(&val)) -} diff --git a/concurrent/hashtriemap_bench_test.go b/concurrent/hashtriemap_bench_test.go index 9fd5b31..ac6f82d 100644 --- a/concurrent/hashtriemap_bench_test.go +++ b/concurrent/hashtriemap_bench_test.go @@ -6,28 +6,30 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//nolint:wsl,testpackage -package concurrent +//nolint:wsl +package concurrent_test import ( "testing" + + cnc "github.com/siderolabs/gen/concurrent" ) -func BenchmarkHashTrieMapHasherLoadSmall(b *testing.B) { - benchmarkHashTrieMapHasherLoad(b, testDataSmall[:]) +func BenchmarkHashTrieMapLoadSmall(b *testing.B) { + benchmarkHashTrieMapLoad(b, testDataSmall[:]) } -func BenchmarkHashTrieMapHasherLoad(b *testing.B) { - benchmarkHashTrieMapHasherLoad(b, testData[:]) +func BenchmarkHashTrieMapLoad(b *testing.B) { + benchmarkHashTrieMapLoad(b, testData[:]) } -func BenchmarkHashTrieMapHasherLoadLarge(b *testing.B) { - benchmarkHashTrieMapHasherLoad(b, testDataLarge[:]) +func BenchmarkHashTrieMapLoadLarge(b *testing.B) { + benchmarkHashTrieMapLoad(b, testDataLarge[:]) } -func benchmarkHashTrieMapHasherLoad(b *testing.B, data []string) { +func benchmarkHashTrieMapLoad(b *testing.B, data []string) { b.ReportAllocs() - m := NewHashTrieMapHasher[string, int](stringHasher) + var m cnc.HashTrieMap[string, int] for i := range data { m.LoadOrStore(data[i], i) } @@ -44,17 +46,17 @@ func benchmarkHashTrieMapHasherLoad(b *testing.B, data []string) { }) } -func BenchmarkHashTrieMapHasherLoadOrStore(b *testing.B) { - benchmarkHashTrieMapHasherLoadOrStore(b, testData[:]) +func BenchmarkHashTrieMapLoadOrStore(b *testing.B) { + benchmarkHashTrieMapLoadOrStore(b, testData[:]) } -func BenchmarkHashTrieMapHasherLoadOrStoreLarge(b *testing.B) { - benchmarkHashTrieMapHasherLoadOrStore(b, testDataLarge[:]) +func BenchmarkHashTrieMapLoadOrStoreLarge(b *testing.B) { + benchmarkHashTrieMapLoadOrStore(b, testDataLarge[:]) } -func benchmarkHashTrieMapHasherLoadOrStore(b *testing.B, data []string) { +func benchmarkHashTrieMapLoadOrStore(b *testing.B, data []string) { b.ReportAllocs() - m := NewHashTrieMapHasher[string, int](stringHasher) + var m cnc.HashTrieMap[string, int] b.RunParallel(func(pb *testing.PB) { i := 0 diff --git a/concurrent/hashtriemap_test.go b/concurrent/hashtriemap_test.go index 84f7b55..677b8f6 100644 --- a/concurrent/hashtriemap_test.go +++ b/concurrent/hashtriemap_test.go @@ -6,51 +6,43 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//nolint:nlreturn,wsl,gocyclo,unparam,unused,cyclop,testpackage,errcheck,revive -package concurrent +//nolint:cyclop,gocognit,gocyclo,maintidx,nlreturn,predeclared,revive,unparam,wsl +package concurrent_test import ( "fmt" - "hash/maphash" "math" "runtime" "strconv" - "strings" "sync" "testing" - "unsafe" -) - -func stringHasher(val *string, seed maphash.Seed) uintptr { - var hm maphash.Hash - hm.SetSeed(seed) + cnc "github.com/siderolabs/gen/concurrent" +) - hm.WriteString(*val) +func TestHashTrieMap(t *testing.T) { + t.Log(runtime.Version()) - return uintptr(hm.Sum64()) + testHashTrieMap(t, cnc.NewHashTrieMap[string, int]) } -func TestHashTrieMapHasher(t *testing.T) { - testHashTrieMap(t, func() *HashTrieMap[string, int] { - return NewHashTrieMapHasher[string, int](stringHasher) +func TestHashTrieMapBadHash(t *testing.T) { + testHashTrieMap(t, func() *cnc.HashTrieMap[string, int] { + return cnc.NewBadHashTrieMap[string, int]() }) } -func TestNewHashTrieMapHasherBadHash(t *testing.T) { - testHashTrieMap(t, func() *HashTrieMap[string, int] { - // Stub out the good hash function with a terrible one. - // Everything should still work as expected. - m := NewHashTrieMap[string, int]() - m.keyHash = func(_ unsafe.Pointer, _ uintptr) uintptr { - return 0 - } - return m +func TestHashTrieMapTruncHash(t *testing.T) { + testHashTrieMap(t, func() *cnc.HashTrieMap[string, int] { + // Stub out the good hash function with a different terrible one + // (truncated hash). Everything should still work as expected. + // This is useful to test independently to catch issues with + // near collisions, where only the last few bits of the hash differ. + return cnc.NewTruncHashTrieMap[string, int]() }) } -//nolint:gocognit -func testHashTrieMap(t *testing.T, newMap func() *HashTrieMap[string, int]) { +func testHashTrieMap(t *testing.T, newMap func() *cnc.HashTrieMap[string, int]) { t.Run("LoadEmpty", func(t *testing.T) { m := newMap() @@ -72,157 +64,671 @@ func testHashTrieMap(t *testing.T, newMap func() *HashTrieMap[string, int]) { expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) } }) - t.Run("CompareAndDeleteAll", func(t *testing.T) { + t.Run("All", func(t *testing.T) { m := newMap() - for range 3 { + testAll(t, m, testDataMap(testData[:]), func(_ string, _ int) bool { + return true + }) + }) + t.Run("Clear", func(t *testing.T) { + t.Run("Simple", func(t *testing.T) { + m := newMap() + for i, s := range testData { expectMissing(t, s, 0)(m.Load(s)) expectStored(t, s, i)(m.LoadOrStore(s, i)) expectPresent(t, s, i)(m.Load(s)) expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) } + m.Clear() + for _, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + } + }) + t.Run("Concurrent", func(t *testing.T) { + m := newMap() + + // Load up the map. for i, s := range testData { - expectPresent(t, s, i)(m.Load(s)) - expectNotDeleted(t, s, math.MaxInt)(m.CompareAndDelete(s, math.MaxInt)) - expectDeleted(t, s, i)(m.CompareAndDelete(s, i)) - expectNotDeleted(t, s, i)(m.CompareAndDelete(s, i)) expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + } + gmp := runtime.GOMAXPROCS(-1) + var wg sync.WaitGroup + for i := range gmp { + wg.Add(1) + go func(id int) { + defer wg.Done() + + for _, s := range testData { + // Try a couple things to interfere with the clear. + expectNotDeleted(t, s, math.MaxInt)(m.CompareAndDelete(s, math.MaxInt)) + m.CompareAndSwap(s, i, i+1) // May succeed or fail; we don't care. + } + }(i) } + + // Concurrently clear the map. + runtime.Gosched() + m.Clear() + + // Wait for workers to finish. + wg.Wait() + + // It should all be empty now. for _, s := range testData { expectMissing(t, s, 0)(m.Load(s)) } - } + }) }) - t.Run("CompareAndDeleteOne", func(t *testing.T) { - m := newMap() + t.Run("CompareAndDelete", func(t *testing.T) { + t.Run("All", func(t *testing.T) { + m := newMap() - for i, s := range testData { - expectMissing(t, s, 0)(m.Load(s)) - expectStored(t, s, i)(m.LoadOrStore(s, i)) - expectPresent(t, s, i)(m.Load(s)) - expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) - } - expectNotDeleted(t, testData[15], math.MaxInt)(m.CompareAndDelete(testData[15], math.MaxInt)) - expectDeleted(t, testData[15], 15)(m.CompareAndDelete(testData[15], 15)) - expectNotDeleted(t, testData[15], 15)(m.CompareAndDelete(testData[15], 15)) - for i, s := range testData { - if i == 15 { + for range 3 { + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) + } + for i, s := range testData { + expectPresent(t, s, i)(m.Load(s)) + expectNotDeleted(t, s, math.MaxInt)(m.CompareAndDelete(s, math.MaxInt)) + expectDeleted(t, s, i)(m.CompareAndDelete(s, i)) + expectNotDeleted(t, s, i)(m.CompareAndDelete(s, i)) + expectMissing(t, s, 0)(m.Load(s)) + } + for _, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + } + } + }) + t.Run("One", func(t *testing.T) { + m := newMap() + + for i, s := range testData { expectMissing(t, s, 0)(m.Load(s)) - } else { + expectStored(t, s, i)(m.LoadOrStore(s, i)) expectPresent(t, s, i)(m.Load(s)) + expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) } - } - }) - t.Run("DeleteMultiple", func(t *testing.T) { - m := newMap() + expectNotDeleted(t, testData[15], math.MaxInt)(m.CompareAndDelete(testData[15], math.MaxInt)) + expectDeleted(t, testData[15], 15)(m.CompareAndDelete(testData[15], 15)) + expectNotDeleted(t, testData[15], 15)(m.CompareAndDelete(testData[15], 15)) + for i, s := range testData { + if i == 15 { + expectMissing(t, s, 0)(m.Load(s)) + } else { + expectPresent(t, s, i)(m.Load(s)) + } + } + }) + t.Run("Multiple", func(t *testing.T) { + m := newMap() - for i, s := range testData { - expectMissing(t, s, 0)(m.Load(s)) - expectStored(t, s, i)(m.LoadOrStore(s, i)) - expectPresent(t, s, i)(m.Load(s)) - expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) - } - for _, i := range []int{1, 105, 6, 85} { - expectNotDeleted(t, testData[i], math.MaxInt)(m.CompareAndDelete(testData[i], math.MaxInt)) - expectDeleted(t, testData[i], i)(m.CompareAndDelete(testData[i], i)) - expectNotDeleted(t, testData[i], i)(m.CompareAndDelete(testData[i], i)) - } - for i, s := range testData { - if i == 1 || i == 105 || i == 6 || i == 85 { + for i, s := range testData { expectMissing(t, s, 0)(m.Load(s)) - } else { + expectStored(t, s, i)(m.LoadOrStore(s, i)) expectPresent(t, s, i)(m.Load(s)) + expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) } - } - }) - t.Run("Enumerate", func(t *testing.T) { - m := newMap() + for _, i := range []int{1, 105, 6, 85} { + expectNotDeleted(t, testData[i], math.MaxInt)(m.CompareAndDelete(testData[i], math.MaxInt)) + expectDeleted(t, testData[i], i)(m.CompareAndDelete(testData[i], i)) + expectNotDeleted(t, testData[i], i)(m.CompareAndDelete(testData[i], i)) + } + for i, s := range testData { + if i == 1 || i == 105 || i == 6 || i == 85 { + expectMissing(t, s, 0)(m.Load(s)) + } else { + expectPresent(t, s, i)(m.Load(s)) + } + } + }) + t.Run("Iterate", func(t *testing.T) { + m := newMap() - testEnumerate(t, m, testDataMap(testData[:]), func(_ string, _ int) bool { - return true + testAll(t, m, testDataMap(testData[:]), func(s string, i int) bool { + expectDeleted(t, s, i)(m.CompareAndDelete(s, i)) + return true + }) + for _, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + } }) - }) - t.Run("EnumerateDelete", func(t *testing.T) { - m := newMap() + t.Run("ConcurrentUnsharedKeys", func(t *testing.T) { + m := newMap() + + gmp := runtime.GOMAXPROCS(-1) + var wg sync.WaitGroup + for i := range gmp { + wg.Add(1) + go func(id int) { + defer wg.Done() + + makeKey := func(s string) string { + return s + "-" + strconv.Itoa(id) + } + for _, s := range testData { + key := makeKey(s) + expectMissing(t, key, 0)(m.Load(key)) + expectStored(t, key, id)(m.LoadOrStore(key, id)) + expectPresent(t, key, id)(m.Load(key)) + expectLoaded(t, key, id)(m.LoadOrStore(key, 0)) + } + for _, s := range testData { + key := makeKey(s) + expectPresent(t, key, id)(m.Load(key)) + expectDeleted(t, key, id)(m.CompareAndDelete(key, id)) + expectMissing(t, key, 0)(m.Load(key)) + } + for _, s := range testData { + key := makeKey(s) + expectMissing(t, key, 0)(m.Load(key)) + } + }(i) + } + wg.Wait() + }) + t.Run("ConcurrentSharedKeys", func(t *testing.T) { + m := newMap() - testEnumerate(t, m, testDataMap(testData[:]), func(s string, i int) bool { - expectDeleted(t, s, i)(m.CompareAndDelete(s, i)) - return true + // Load up the map. + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + } + gmp := runtime.GOMAXPROCS(-1) + var wg sync.WaitGroup + for i := range gmp { + wg.Add(1) + go func(id int) { + defer wg.Done() + + for i, s := range testData { + expectNotDeleted(t, s, math.MaxInt)(m.CompareAndDelete(s, math.MaxInt)) + m.CompareAndDelete(s, i) + expectMissing(t, s, 0)(m.Load(s)) + } + for _, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + } + }(i) + } + wg.Wait() }) - for _, s := range testData { - expectMissing(t, s, 0)(m.Load(s)) - } }) - t.Run("ConcurrentLifecycleUnsharedKeys", func(t *testing.T) { - m := newMap() + t.Run("CompareAndSwap", func(t *testing.T) { + t.Run("All", func(t *testing.T) { + m := newMap() - gmp := runtime.GOMAXPROCS(-1) - var wg sync.WaitGroup - for i := range gmp { - wg.Add(1) - go func(id int) { - defer wg.Done() + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) + } + for j := range 3 { + for i, s := range testData { + expectPresent(t, s, i+j)(m.Load(s)) + expectNotSwapped(t, s, math.MaxInt, i+j+1)(m.CompareAndSwap(s, math.MaxInt, i+j+1)) + expectSwapped(t, s, i, i+j+1)(m.CompareAndSwap(s, i+j, i+j+1)) + expectNotSwapped(t, s, i+j, i+j+1)(m.CompareAndSwap(s, i+j, i+j+1)) + expectPresent(t, s, i+j+1)(m.Load(s)) + } + } + for i, s := range testData { + expectPresent(t, s, i+3)(m.Load(s)) + } + }) + t.Run("One", func(t *testing.T) { + m := newMap() - makeKey := func(s string) string { - return s + "-" + strconv.Itoa(id) + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) + } + expectNotSwapped(t, testData[15], math.MaxInt, 16)(m.CompareAndSwap(testData[15], math.MaxInt, 16)) + expectSwapped(t, testData[15], 15, 16)(m.CompareAndSwap(testData[15], 15, 16)) + expectNotSwapped(t, testData[15], 15, 16)(m.CompareAndSwap(testData[15], 15, 16)) + for i, s := range testData { + if i == 15 { + expectPresent(t, s, 16)(m.Load(s)) + } else { + expectPresent(t, s, i)(m.Load(s)) } - for _, s := range testData { - key := makeKey(s) - expectMissing(t, key, 0)(m.Load(key)) - expectStored(t, key, id)(m.LoadOrStore(key, id)) - expectPresent(t, key, id)(m.Load(key)) - expectLoaded(t, key, id)(m.LoadOrStore(key, 0)) + } + }) + t.Run("Multiple", func(t *testing.T) { + m := newMap() + + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) + } + for _, i := range []int{1, 105, 6, 85} { + expectNotSwapped(t, testData[i], math.MaxInt, i+1)(m.CompareAndSwap(testData[i], math.MaxInt, i+1)) + expectSwapped(t, testData[i], i, i+1)(m.CompareAndSwap(testData[i], i, i+1)) + expectNotSwapped(t, testData[i], i, i+1)(m.CompareAndSwap(testData[i], i, i+1)) + } + for i, s := range testData { + if i == 1 || i == 105 || i == 6 || i == 85 { + expectPresent(t, s, i+1)(m.Load(s)) + } else { + expectPresent(t, s, i)(m.Load(s)) } - for _, s := range testData { - key := makeKey(s) - expectPresent(t, key, id)(m.Load(key)) - expectDeleted(t, key, id)(m.CompareAndDelete(key, id)) - expectMissing(t, key, 0)(m.Load(key)) + } + }) + + t.Run("ConcurrentUnsharedKeys", func(t *testing.T) { + m := newMap() + + gmp := runtime.GOMAXPROCS(-1) + var wg sync.WaitGroup + for i := range gmp { + wg.Add(1) + go func(id int) { + defer wg.Done() + + makeKey := func(s string) string { + return s + "-" + strconv.Itoa(id) + } + for _, s := range testData { + key := makeKey(s) + expectMissing(t, key, 0)(m.Load(key)) + expectStored(t, key, id)(m.LoadOrStore(key, id)) + expectPresent(t, key, id)(m.Load(key)) + expectLoaded(t, key, id)(m.LoadOrStore(key, 0)) + } + for _, s := range testData { + key := makeKey(s) + expectPresent(t, key, id)(m.Load(key)) + expectSwapped(t, key, id, id+1)(m.CompareAndSwap(key, id, id+1)) + expectPresent(t, key, id+1)(m.Load(key)) + } + for _, s := range testData { + key := makeKey(s) + expectPresent(t, key, id+1)(m.Load(key)) + } + }(i) + } + wg.Wait() + }) + t.Run("ConcurrentUnsharedKeysWithDelete", func(t *testing.T) { + m := newMap() + + gmp := runtime.GOMAXPROCS(-1) + var wg sync.WaitGroup + for i := range gmp { + wg.Add(1) + go func(id int) { + defer wg.Done() + + makeKey := func(s string) string { + return s + "-" + strconv.Itoa(id) + } + for _, s := range testData { + key := makeKey(s) + expectMissing(t, key, 0)(m.Load(key)) + expectStored(t, key, id)(m.LoadOrStore(key, id)) + expectPresent(t, key, id)(m.Load(key)) + expectLoaded(t, key, id)(m.LoadOrStore(key, 0)) + } + for _, s := range testData { + key := makeKey(s) + expectPresent(t, key, id)(m.Load(key)) + expectSwapped(t, key, id, id+1)(m.CompareAndSwap(key, id, id+1)) + expectPresent(t, key, id+1)(m.Load(key)) + expectDeleted(t, key, id+1)(m.CompareAndDelete(key, id+1)) + expectNotSwapped(t, key, id+1, id+2)(m.CompareAndSwap(key, id+1, id+2)) + expectNotDeleted(t, key, id+1)(m.CompareAndDelete(key, id+1)) + expectMissing(t, key, 0)(m.Load(key)) + } + for _, s := range testData { + key := makeKey(s) + expectMissing(t, key, 0)(m.Load(key)) + } + }(i) + } + wg.Wait() + }) + t.Run("ConcurrentSharedKeys", func(t *testing.T) { + m := newMap() + + // Load up the map. + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + } + gmp := runtime.GOMAXPROCS(-1) + var wg sync.WaitGroup + for i := range gmp { + wg.Add(1) + go func(id int) { + defer wg.Done() + + for i, s := range testData { + expectNotSwapped(t, s, math.MaxInt, i+1)(m.CompareAndSwap(s, math.MaxInt, i+1)) + m.CompareAndSwap(s, i, i+1) + expectPresent(t, s, i+1)(m.Load(s)) + } + for i, s := range testData { + expectPresent(t, s, i+1)(m.Load(s)) + } + }(i) + } + wg.Wait() + }) + }) + t.Run("Swap", func(t *testing.T) { + t.Run("All", func(t *testing.T) { + m := newMap() + + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectNotLoadedFromSwap(t, s, i)(m.Swap(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoadedFromSwap(t, s, i, i)(m.Swap(s, i)) + } + for j := range 3 { + for i, s := range testData { + expectPresent(t, s, i+j)(m.Load(s)) + expectLoadedFromSwap(t, s, i+j, i+j+1)(m.Swap(s, i+j+1)) + expectPresent(t, s, i+j+1)(m.Load(s)) } - for _, s := range testData { - key := makeKey(s) - expectMissing(t, key, 0)(m.Load(key)) + } + for i, s := range testData { + expectLoadedFromSwap(t, s, i+3, i+3)(m.Swap(s, i+3)) + } + }) + t.Run("One", func(t *testing.T) { + m := newMap() + + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectNotLoadedFromSwap(t, s, i)(m.Swap(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoadedFromSwap(t, s, i, i)(m.Swap(s, i)) + } + expectLoadedFromSwap(t, testData[15], 15, 16)(m.Swap(testData[15], 16)) + for i, s := range testData { + if i == 15 { + expectPresent(t, s, 16)(m.Load(s)) + } else { + expectPresent(t, s, i)(m.Load(s)) } - }(i) - } - wg.Wait() - }) - t.Run("ConcurrentDeleteSharedKeys", func(t *testing.T) { - m := newMap() + } + }) + t.Run("Multiple", func(t *testing.T) { + m := newMap() - // Load up the map. - for i, s := range testData { - expectMissing(t, s, 0)(m.Load(s)) - expectStored(t, s, i)(m.LoadOrStore(s, i)) - } - gmp := runtime.GOMAXPROCS(-1) - var wg sync.WaitGroup - for i := range gmp { - wg.Add(1) - go func(id int) { - defer wg.Done() + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectNotLoadedFromSwap(t, s, i)(m.Swap(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoadedFromSwap(t, s, i, i)(m.Swap(s, i)) + } + for _, i := range []int{1, 105, 6, 85} { + expectLoadedFromSwap(t, testData[i], i, i+1)(m.Swap(testData[i], i+1)) + } + for i, s := range testData { + if i == 1 || i == 105 || i == 6 || i == 85 { + expectPresent(t, s, i+1)(m.Load(s)) + } else { + expectPresent(t, s, i)(m.Load(s)) + } + } + }) + t.Run("ConcurrentUnsharedKeys", func(t *testing.T) { + m := newMap() + + gmp := runtime.GOMAXPROCS(-1) + var wg sync.WaitGroup + for i := range gmp { + wg.Add(1) + go func(id int) { + defer wg.Done() + + makeKey := func(s string) string { + return s + "-" + strconv.Itoa(id) + } + for _, s := range testData { + key := makeKey(s) + expectMissing(t, key, 0)(m.Load(key)) + expectNotLoadedFromSwap(t, key, id)(m.Swap(key, id)) + expectPresent(t, key, id)(m.Load(key)) + expectLoadedFromSwap(t, key, id, id)(m.Swap(key, id)) + } + for _, s := range testData { + key := makeKey(s) + expectPresent(t, key, id)(m.Load(key)) + expectLoadedFromSwap(t, key, id, id+1)(m.Swap(key, id+1)) + expectPresent(t, key, id+1)(m.Load(key)) + } + for _, s := range testData { + key := makeKey(s) + expectPresent(t, key, id+1)(m.Load(key)) + } + }(i) + } + wg.Wait() + }) + t.Run("ConcurrentUnsharedKeysWithDelete", func(t *testing.T) { + m := newMap() + + gmp := runtime.GOMAXPROCS(-1) + var wg sync.WaitGroup + for i := range gmp { + wg.Add(1) + go func(id int) { + defer wg.Done() + + makeKey := func(s string) string { + return s + "-" + strconv.Itoa(id) + } + for _, s := range testData { + key := makeKey(s) + expectMissing(t, key, 0)(m.Load(key)) + expectNotLoadedFromSwap(t, key, id)(m.Swap(key, id)) + expectPresent(t, key, id)(m.Load(key)) + expectLoadedFromSwap(t, key, id, id)(m.Swap(key, id)) + } + for _, s := range testData { + key := makeKey(s) + expectPresent(t, key, id)(m.Load(key)) + expectLoadedFromSwap(t, key, id, id+1)(m.Swap(key, id+1)) + expectPresent(t, key, id+1)(m.Load(key)) + expectDeleted(t, key, id+1)(m.CompareAndDelete(key, id+1)) + expectNotLoadedFromSwap(t, key, id+2)(m.Swap(key, id+2)) + expectPresent(t, key, id+2)(m.Load(key)) + } + for _, s := range testData { + key := makeKey(s) + expectPresent(t, key, id+2)(m.Load(key)) + } + }(i) + } + wg.Wait() + }) + t.Run("ConcurrentSharedKeys", func(t *testing.T) { + m := newMap() + // Load up the map. + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + } + gmp := runtime.GOMAXPROCS(-1) + var wg sync.WaitGroup + for i := range gmp { + wg.Add(1) + go func(id int) { + defer wg.Done() + + for i, s := range testData { + m.Swap(s, i+1) + expectPresent(t, s, i+1)(m.Load(s)) + } + for i, s := range testData { + expectPresent(t, s, i+1)(m.Load(s)) + } + }(i) + } + wg.Wait() + }) + }) + t.Run("LoadAndDelete", func(t *testing.T) { + t.Run("All", func(t *testing.T) { + m := newMap() + + for range 3 { for i, s := range testData { - expectNotDeleted(t, s, math.MaxInt)(m.CompareAndDelete(s, math.MaxInt)) - m.CompareAndDelete(s, i) expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) + } + for i, s := range testData { + expectPresent(t, s, i)(m.Load(s)) + expectLoadedFromDelete(t, s, i)(m.LoadAndDelete(s)) + expectMissing(t, s, 0)(m.Load(s)) + expectNotLoadedFromDelete(t, s, 0)(m.LoadAndDelete(s)) } for _, s := range testData { expectMissing(t, s, 0)(m.Load(s)) } - }(i) - } - wg.Wait() + } + }) + t.Run("One", func(t *testing.T) { + m := newMap() + + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) + } + expectPresent(t, testData[15], 15)(m.Load(testData[15])) + expectLoadedFromDelete(t, testData[15], 15)(m.LoadAndDelete(testData[15])) + expectMissing(t, testData[15], 0)(m.Load(testData[15])) + expectNotLoadedFromDelete(t, testData[15], 0)(m.LoadAndDelete(testData[15])) + for i, s := range testData { + if i == 15 { + expectMissing(t, s, 0)(m.Load(s)) + } else { + expectPresent(t, s, i)(m.Load(s)) + } + } + }) + t.Run("Multiple", func(t *testing.T) { + m := newMap() + + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) + } + for _, i := range []int{1, 105, 6, 85} { + expectPresent(t, testData[i], i)(m.Load(testData[i])) + expectLoadedFromDelete(t, testData[i], i)(m.LoadAndDelete(testData[i])) + expectMissing(t, testData[i], 0)(m.Load(testData[i])) + expectNotLoadedFromDelete(t, testData[i], 0)(m.LoadAndDelete(testData[i])) + } + for i, s := range testData { + if i == 1 || i == 105 || i == 6 || i == 85 { + expectMissing(t, s, 0)(m.Load(s)) + } else { + expectPresent(t, s, i)(m.Load(s)) + } + } + }) + t.Run("Iterate", func(t *testing.T) { + m := newMap() + + testAll(t, m, testDataMap(testData[:]), func(s string, i int) bool { + expectLoadedFromDelete(t, s, i)(m.LoadAndDelete(s)) + return true + }) + for _, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + } + }) + t.Run("ConcurrentUnsharedKeys", func(t *testing.T) { + m := newMap() + + gmp := runtime.GOMAXPROCS(-1) + var wg sync.WaitGroup + for i := range gmp { + wg.Add(1) + go func(id int) { + defer wg.Done() + + makeKey := func(s string) string { + return s + "-" + strconv.Itoa(id) + } + for _, s := range testData { + key := makeKey(s) + expectMissing(t, key, 0)(m.Load(key)) + expectStored(t, key, id)(m.LoadOrStore(key, id)) + expectPresent(t, key, id)(m.Load(key)) + expectLoaded(t, key, id)(m.LoadOrStore(key, 0)) + } + for _, s := range testData { + key := makeKey(s) + expectPresent(t, key, id)(m.Load(key)) + expectLoadedFromDelete(t, key, id)(m.LoadAndDelete(key)) + expectMissing(t, key, 0)(m.Load(key)) + } + for _, s := range testData { + key := makeKey(s) + expectMissing(t, key, 0)(m.Load(key)) + } + }(i) + } + wg.Wait() + }) + t.Run("ConcurrentSharedKeys", func(t *testing.T) { + m := newMap() + + // Load up the map. + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + } + gmp := runtime.GOMAXPROCS(-1) + var wg sync.WaitGroup + for i := range gmp { + wg.Add(1) + go func(id int) { + defer wg.Done() + + for _, s := range testData { + m.LoadAndDelete(s) + expectMissing(t, s, 0)(m.Load(s)) + } + for _, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + } + }(i) + } + wg.Wait() + }) }) } -func testEnumerate[K, V comparable](t *testing.T, m *HashTrieMap[K, V], testData map[K]V, yield func(K, V) bool) { +func testAll[K, V comparable](t *testing.T, m *cnc.HashTrieMap[K, V], testData map[K]V, yield func(K, V) bool) { for k, v := range testData { expectStored(t, k, v)(m.LoadOrStore(k, v)) } visited := make(map[K]int) - m.Enumerate(func(key K, got V) bool { + m.All()(func(key K, got V) bool { want, ok := testData[key] if !ok { t.Errorf("unexpected key %v in map", key) @@ -325,6 +831,76 @@ func expectNotDeleted[K, V comparable](t *testing.T, key K, old V) func(deleted } } +func expectSwapped[K, V comparable](t *testing.T, key K, old, new V) func(swapped bool) { + t.Helper() + return func(swapped bool) { + t.Helper() + + if !swapped { + t.Errorf("expected key %v with value %v to be in map and swapped for %v", key, old, new) + } + } +} + +func expectNotSwapped[K, V comparable](t *testing.T, key K, old, new V) func(swapped bool) { + t.Helper() + return func(swapped bool) { + t.Helper() + + if swapped { + t.Errorf("expected key %v with value %v to not be in map or not swapped for %v", key, old, new) + } + } +} + +func expectLoadedFromSwap[K, V comparable](t *testing.T, key K, want, new V) func(got V, loaded bool) { + t.Helper() + return func(got V, loaded bool) { + t.Helper() + + if !loaded { + t.Errorf("expected key %v to be in map and for %v to have been swapped for %v", key, want, new) + } else if want != got { + t.Errorf("key %v had its value %v swapped for %v, but expected it to have value %v", key, got, new, want) + } + } +} + +func expectNotLoadedFromSwap[K, V comparable](t *testing.T, key K, new V) func(old V, loaded bool) { + t.Helper() + return func(old V, loaded bool) { + t.Helper() + + if loaded { + t.Errorf("expected key %v to not be in map, but found value %v for it", key, old) + } + } +} + +func expectLoadedFromDelete[K, V comparable](t *testing.T, key K, want V) func(got V, loaded bool) { + t.Helper() + return func(got V, loaded bool) { + t.Helper() + + if !loaded { + t.Errorf("expected key %v to be in map to be deleted", key) + } else if want != got { + t.Errorf("key %v was deleted with value %v, but expected it to have value %v", key, got, want) + } + } +} + +func expectNotLoadedFromDelete[K, V comparable](t *testing.T, key K, _ V) func(old V, loaded bool) { + t.Helper() + return func(old V, loaded bool) { + t.Helper() + + if loaded { + t.Errorf("expected key %v to not be in map, but found value %v for it", key, old) + } + } +} + func testDataMap(data []string) map[string]int { m := make(map[string]int) for i, s := range data { @@ -350,39 +926,3 @@ func init() { testDataLarge[i] = fmt.Sprintf("%b", i) } } - -func dumpMap[K, V comparable](ht *HashTrieMap[K, V]) { - dumpNode(ht, &ht.root.node, 0) -} - -func dumpNode[K, V comparable](ht *HashTrieMap[K, V], n *node[K, V], depth int) { - var sb strings.Builder - for range depth { - fmt.Fprintf(&sb, "\t") - } - prefix := sb.String() - if n.isEntry { - e := n.entry() - for e != nil { - fmt.Printf("%s%p [Entry Key=%v Value=%v Overflow=%p, Hash=%016x]\n", prefix, e, e.key, e.value, e.overflow.Load(), ht.keyHash(unsafe.Pointer(&e.key), ht.seed)) - e = e.overflow.Load() - } - return - } - i := n.indirect() - fmt.Printf("%s%p [Indirect Parent=%p Dead=%t Children=[", prefix, i, i.parent, i.dead.Load()) - for j := range i.children { - c := i.children[j].Load() - fmt.Printf("%p", c) - if j != len(i.children)-1 { - fmt.Printf(", ") - } - } - fmt.Printf("]]\n") - for j := range i.children { - c := i.children[j].Load() - if c != nil { - dumpNode(ht, c, depth+1) - } - } -} diff --git a/containers/syncmap.go b/containers/syncmap.go index 005e2b6..032f221 100644 --- a/containers/syncmap.go +++ b/containers/syncmap.go @@ -85,5 +85,5 @@ func castOrZero[T any](val any) T { return zero } - return val.(T) //nolint:forcetypeassert + return val.(T) //nolint:errcheck,forcetypeassert }