runtime: spurious SIGSEGV in systemstack() #68632

mrwonko · 2024-07-29T15:21:27Z

Go version

go version go1.22.5 linux/amd64

Output of `go env` in your module/workspace:

GO111MODULE='on'
GOARCH='amd64'
GOBIN=''
GOCACHE='/home/AzDevOps_azpcontainer/.cache/go-build'
GOENV='/home/AzDevOps_azpcontainer/.config/go/env'
GOEXE=''
GOEXPERIMENT=''
GOFLAGS=''
GOHOSTARCH='amd64'
GOHOSTOS='linux'
GOINSECURE=''
GOMODCACHE='/home/AzDevOps_azpcontainer/go/pkg/mod'
GONOPROXY=''
GONOSUMDB=''
GOOS='linux'
GOPATH='/home/AzDevOps_azpcontainer/go'
GOPRIVATE=''
GOPROXY='https://***:***@redacted.jfrog.io/artifactory/api/go/golang'
GOROOT='/usr'
GOSUMDB='sum.golang.org'
GOTMPDIR=''
GOTOOLCHAIN='auto'
GOTOOLDIR='/usr/pkg/tool/linux_amd64'
GOVCS=''
GOVERSION='go1.22.5'
GCCGO='gccgo'
GOAMD64='v1'
AR='ar'
CC='gcc'
CXX='g++'
CGO_ENABLED='1'
GOMOD='/__w/1/s/go.mod'
GOWORK='/__w/1/s/go.work'
CGO_CFLAGS='-O2 -g'
CGO_CPPFLAGS=''
CGO_CXXFLAGS='-O2 -g'
CGO_FFLAGS='-O2 -g'
CGO_LDFLAGS='-O2 -g'
PKG_CONFIG='pkg-config'
GOGCCFLAGS='-fPIC -m64 -pthread -Wl,--no-gc-sections -fmessage-length=0 -ffile-prefix-map=/tmp/go-build3329120536=/tmp/go-build -gno-record-gcc-switches'

What did you do?

Our build pipeline ran a test that usually succeeds. Simplified, it looks like this:

package main

import (
	"fmt"
	"testing"

	"github.com/google/go-cmp/cmp"
)

func Test_getAllBatches(t *testing.T) {
	// originally this tests a concurrent function, but the tests themselves run sequentially
	t.Run("Concurrency Test", func(t *testing.T) { // corresponds to util_test.go:1229 in the stacktrace
		for i := 0; i < 100; i++ {
			t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { // corresponds to util_test.go:1232 in the stacktrace
				// complicated concurrent calculation omitted as it appears to be irrelevant
				got := []int{1, 2, 3}
				want := []int{1, 2, 3}
				if d := cmp.Diff(got, want); d != "" { // corresponds to util_test.go:1251 in the stacktrace
					t.Errorf("got an unexpected result with diff from the expected: %v", d)
				}
			})
		}
	})
}

and was essentially run like this

go test -timeout=5m -cover -coverprofile=cover.out ./domains/til/services/transformer/main ./unrelated-package-also-being-tested

What did you see happen?

<a bunch of unrelated logs printed by code being tested omitted for brevity>

SIGSEGV: segmentation violation
PC=0x41c1db m=7 sigcode=1 addr=0x1

goroutine 0 gp=0xc000007a40 m=7 mp=0xc0000c2008 [idle]:
runtime.readUintptr(...)
	/usr/src/runtime/mbitmap.go:466
runtime.(*mspan).typePointersOfUnchecked(0xc000cfa900?, 0xc0002f2460?)
	/usr/src/runtime/mbitmap_allocheaders.go:203 +0x3b fp=0x7854337fdc88 sp=0x7854337fdc68 pc=0x41c1db
runtime.scanobject(0xc00006b268?, 0xc00006b268)
	/usr/src/runtime/mgcmark.go:1446 +0xb5 fp=0x7854337fdd18 sp=0x7854337fdc88 pc=0x427b75
runtime.gcDrain(0xc00006b268, 0x7)
	/usr/src/runtime/mgcmark.go:1242 +0x1f4 fp=0x7854337fdd80 sp=0x7854337fdd18 pc=0x4274d4
runtime.gcDrainMarkWorkerIdle(...)
	/usr/src/runtime/mgcmark.go:1114
runtime.gcBgMarkWorker.func2()
	/usr/src/runtime/mgc.go:1406 +0x6f fp=0x7854337fddd0 sp=0x7854337fdd80 pc=0x423b0f
runtime.systemstack(0x800000)
	/usr/src/runtime/asm_amd64.s:509 +0x4a fp=0x7854337fdde0 sp=0x7854337fddd0 pc=0x479a8a

goroutine 6 gp=0xc0000d88c0 m=7 mp=0xc0000c2008 [GC worker (active)]:
runtime.systemstack_switch()
	/usr/src/runtime/asm_amd64.s:474 +0x8 fp=0xc00007e750 sp=0xc00007e740 pc=0x479a28
runtime.gcBgMarkWorker()
	/usr/src/runtime/mgc.go:1370 +0x1f2 fp=0xc00007e7e0 sp=0xc00007e750 pc=0x4237d2
runtime.goexit({})
	/usr/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc00007e7e8 sp=0xc00007e7e0 pc=0x47ba41
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/src/runtime/mgc.go:1234 +0x1c

goroutine 1 gp=0xc0000061c0 m=nil [chan receive]:
runtime.gopark(0xc00109bc70?, 0xc00109bcf8?, 0x70?, 0xbc?, 0x1072440?)
	/usr/src/runtime/proc.go:402 +0xce fp=0xc0005519b0 sp=0xc000551990 pc=0x4433ce
runtime.chanrecv(0xc000a25340, 0xc000551a97, 0x1)
	/usr/src/runtime/chan.go:583 +0x3bf fp=0xc000551a28 sp=0xc0005519b0 pc=0x40cf7f
runtime.chanrecv1(0x205f1c0?, 0xfb6f40?)
	/usr/src/runtime/chan.go:442 +0x12 fp=0xc000551a50 sp=0xc000551a28 pc=0x40cb92
testing.(*T).Run(0xc0001489c0, {0x1269059?, 0x0?}, 0x135b9e0)
	/usr/src/testing/testing.go:1750 +0x3ab fp=0xc000551b10 sp=0xc000551a50 pc=0x595b6b
testing.runTests.func1(0xc0001489c0)
	/usr/src/testing/testing.go:2161 +0x37 fp=0xc000551b50 sp=0xc000551b10 pc=0x597cb7
testing.tRunner(0xc0001489c0, 0xc0003dfc60)
	/usr/src/testing/testing.go:1689 +0xfb fp=0xc000551ba0 sp=0xc000551b50 pc=0x594c9b
testing.runTests(0xc000012288, {0x1efad40, 0xc5, 0xc5}, {0x1?, 0x527a0e?, 0x20609e0?})
	/usr/src/testing/testing.go:2159 +0x445 fp=0xc000551c90 sp=0xc000551ba0 pc=0x597ba5
testing.(*M).Run(0xc0002fa8c0)
	/usr/src/testing/testing.go:2027 +0x68b fp=0xc000551ec0 sp=0xc000551c90 pc=0x5965ab
main.main()
	_testmain.go:473 +0x193 fp=0xc000551f50 sp=0xc000551ec0 pc=0xf0b113
runtime.main()
	/usr/src/runtime/proc.go:271 +0x29d fp=0xc000551fe0 sp=0xc000551f50 pc=0x442f7d
runtime.goexit({})
	/usr/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc000551fe8 sp=0xc000551fe0 pc=0x47ba41

goroutine 2 gp=0xc000006c40 m=nil [force gc (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/src/runtime/proc.go:402 +0xce fp=0xc00007cfa8 sp=0xc00007cf88 pc=0x4433ce
runtime.goparkunlock(...)
	/usr/src/runtime/proc.go:408
runtime.forcegchelper()
	/usr/src/runtime/proc.go:326 +0xb3 fp=0xc00007cfe0 sp=0xc00007cfa8 pc=0x443233
runtime.goexit({})
	/usr/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc00007cfe8 sp=0xc00007cfe0 pc=0x47ba41
created by runtime.init.6 in goroutine 1
	/usr/src/runtime/proc.go:314 +0x1a

goroutine 3 gp=0xc000007180 m=nil [GC sweep wait]:
runtime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/src/runtime/proc.go:402 +0xce fp=0xc00007d780 sp=0xc00007d760 pc=0x4433ce
runtime.goparkunlock(...)
	/usr/src/runtime/proc.go:408
runtime.bgsweep(0xc000050310)
	/usr/src/runtime/mgcsweep.go:318 +0xdf fp=0xc00007d7c8 sp=0xc00007d780 pc=0x42ccff
runtime.gcenable.gowrap1()
	/usr/src/runtime/mgc.go:203 +0x25 fp=0xc00007d7e0 sp=0xc00007d7c8 pc=0x4215e5
runtime.goexit({})
	/usr/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc00007d7e8 sp=0xc00007d7e0 pc=0x47ba41
created by runtime.gcenable in goroutine 1
	/usr/src/runtime/mgc.go:203 +0x66

goroutine 4 gp=0xc000007340 m=nil [GC scavenge wait]:
runtime.gopark(0x4647742?, 0x3b9aca00?, 0x0?, 0x0?, 0x0?)
	/usr/src/runtime/proc.go:402 +0xce fp=0xc00007df78 sp=0xc00007df58 pc=0x4433ce
runtime.goparkunlock(...)
	/usr/src/runtime/proc.go:408
runtime.(*scavengerState).park(0x20613a0)
	/usr/src/runtime/mgcscavenge.go:425 +0x49 fp=0xc00007dfa8 sp=0xc00007df78 pc=0x42a689
runtime.bgscavenge(0xc000050310)
	/usr/src/runtime/mgcscavenge.go:658 +0x59 fp=0xc00007dfc8 sp=0xc00007dfa8 pc=0x42ac39
runtime.gcenable.gowrap2()
	/usr/src/runtime/mgc.go:204 +0x25 fp=0xc00007dfe0 sp=0xc00007dfc8 pc=0x421585
runtime.goexit({})
	/usr/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc00007dfe8 sp=0xc00007dfe0 pc=0x47ba41
created by runtime.gcenable in goroutine 1
	/usr/src/runtime/mgc.go:204 +0xa5

goroutine 18 gp=0xc000106380 m=nil [finalizer wait]:
runtime.gopark(0x0?, 0x135bd40?, 0x0?, 0xa0?, 0x2000000020?)
	/usr/src/runtime/proc.go:402 +0xce fp=0xc000092e20 sp=0xc000092e00 pc=0x4433ce
runtime.runfinq()
	/usr/src/runtime/mfinal.go:194 +0x107 fp=0xc000092fe0 sp=0xc000092e20 pc=0x420627
runtime.goexit({})
	/usr/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc000092fe8 sp=0xc000092fe0 pc=0x47ba41
created by runtime.createfing in goroutine 1
	/usr/src/runtime/mfinal.go:164 +0x3d

goroutine 19 gp=0xc0001fdc00 m=nil [select]:
runtime.gopark(0xc000078590?, 0x2?, 0x0?, 0x0?, 0xc000078564?)
	/usr/src/runtime/proc.go:402 +0xce fp=0xc000091c10 sp=0xc000091bf0 pc=0x4433ce
runtime.selectgo(0xc000091d90, 0xc000078560, 0x0?, 0x0, 0xc0000785b8?, 0x1)
	/usr/src/runtime/select.go:327 +0x725 fp=0xc000091d30 sp=0xc000091c10 pc=0x455205
io.(*pipe).read(0xc0002884e0, {0xc0000b2000, 0x10000, 0x0?})
	/usr/src/io/pipe.go:57 +0xa5 fp=0xc000091dc0 sp=0xc000091d30 pc=0x4c6505
io.(*PipeReader).Read(0xc000078608?, {0xc0000b2000?, 0x20d5660?, 0xc000078690?})
	/usr/src/io/pipe.go:134 +0x1a fp=0xc000091df0 sp=0xc000091dc0 pc=0x4c6c5a
bufio.(*Scanner).Scan(0xc000091f28)
	/usr/src/bufio/scan.go:219 +0x81e fp=0xc000091ec8 sp=0xc000091df0 pc=0x52b7de
github.com/sirupsen/logrus.(*Entry).writerScanner(0xc000304bd0, 0xc0002884e0, 0xc0002d30a0)
	/home/AzDevOps_azpcontainer/go/pkg/mod/github.com/sirupsen/[email protected]/writer.go:86 +0x11d fp=0xc000091fb8 sp=0xc000091ec8 pc=0x986d7d
github.com/sirupsen/logrus.(*Entry).WriterLevel.gowrap1()
	/home/AzDevOps_azpcontainer/go/pkg/mod/github.com/sirupsen/[email protected]/writer.go:57 +0x28 fp=0xc000091fe0 sp=0xc000091fb8 pc=0x986c28
runtime.goexit({})
	/usr/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc000091fe8 sp=0xc000091fe0 pc=0x47ba41
created by github.com/sirupsen/logrus.(*Entry).WriterLevel in goroutine 1
	/home/AzDevOps_azpcontainer/go/pkg/mod/github.com/sirupsen/[email protected]/writer.go:57 +0x31f

goroutine 20 gp=0xc00033a700 m=nil [select, locked to thread]:
runtime.gopark(0xc000078fa8?, 0x2?, 0x69?, 0x36?, 0xc000078f94?)
	/usr/src/runtime/proc.go:402 +0xce fp=0xc000078e38 sp=0xc000078e18 pc=0x4433ce
runtime.selectgo(0xc000078fa8, 0xc000078f90, 0x0?, 0x0, 0x0?, 0x1)
	/usr/src/runtime/select.go:327 +0x725 fp=0xc000078f58 sp=0xc000078e38 pc=0x455205
runtime.ensureSigM.func1()
	/usr/src/runtime/signal_unix.go:1034 +0x19f fp=0xc000078fe0 sp=0xc000078f58 pc=0x47219f
runtime.goexit({})
	/usr/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc000078fe8 sp=0xc000078fe0 pc=0x47ba41
created by runtime.ensureSigM in goroutine 1
	/usr/src/runtime/signal_unix.go:1017 +0xc8

goroutine 21 gp=0xc00033ac40 m=6 mp=0xc0003bc808 [syscall]:
runtime.notetsleepg(0x20d49c0, 0xffffffffffffffff)
	/usr/src/runtime/lock_futex.go:246 +0x29 fp=0xc0000797a0 sp=0xc000079778 pc=0x4132e9
os/signal.signal_recv()
	/usr/src/runtime/sigqueue.go:152 +0x29 fp=0xc0000797c0 sp=0xc0000797a0 pc=0x477b29
os/signal.loop()
	/usr/src/os/signal/signal_unix.go:23 +0x13 fp=0xc0000797e0 sp=0xc0000797c0 pc=0x5db9d3
runtime.goexit({})
	/usr/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc0000797e8 sp=0xc0000797e0 pc=0x47ba41
created by os/signal.Notify.func1.1 in goroutine 1
	/usr/src/os/signal/signal.go:151 +0x1f

goroutine 5 gp=0xc0000d8700 m=nil [GC worker (idle)]:
runtime.gopark(0x4b6ed77b9d6?, 0x0?, 0x0?, 0x20?, 0x10000?)
	/usr/src/runtime/proc.go:402 +0xce fp=0xc000078750 sp=0xc000078730 pc=0x4433ce
runtime.gcBgMarkWorker()
	/usr/src/runtime/mgc.go:1310 +0xe5 fp=0xc0000787e0 sp=0xc000078750 pc=0x4236c5
runtime.goexit({})
	/usr/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc0000787e8 sp=0xc0000787e0 pc=0x47ba41
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/src/runtime/mgc.go:1234 +0x1c

goroutine 3436 gp=0xc000501180 m=nil [chan receive]:
runtime.gopark(0xc0005f6341?, 0xc0005f63d0?, 0x40?, 0x63?, 0x1072440?)
	/usr/src/runtime/proc.go:402 +0xce fp=0xc000555458 sp=0xc000555438 pc=0x4433ce
runtime.chanrecv(0xc000a25730, 0xc00055553f, 0x1)
	/usr/src/runtime/chan.go:583 +0x3bf fp=0xc0005554d0 sp=0xc000555458 pc=0x40cf7f
runtime.chanrecv1(0x205f1c0?, 0xfb6f40?)
	/usr/src/runtime/chan.go:442 +0x12 fp=0xc0005554f8 sp=0xc0005554d0 pc=0x40cb92
testing.(*T).Run(0xc000d61380, {0x1265abb?, 0x1032340?}, 0xc000dd4030)
	/usr/src/testing/testing.go:1750 +0x3ab fp=0xc0005555b8 sp=0xc0005554f8 pc=0x595b6b
<redacted>/domains/til/services/transformer/main.Test_getAllBatches(0xc000d61380)
	/__w/1/s/domains/til/services/transformer/main/util_test.go:1229 +0x484b fp=0xc000555f70 sp=0xc0005555b8 pc=0xe9b6cb
testing.tRunner(0xc000d61380, 0x135b9e0)
	/usr/src/testing/testing.go:1689 +0xfb fp=0xc000555fc0 sp=0xc000555f70 pc=0x594c9b
testing.(*T).Run.gowrap1()
	/usr/src/testing/testing.go:1742 +0x25 fp=0xc000555fe0 sp=0xc000555fc0 pc=0x595cc5
runtime.goexit({})
	/usr/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc000555fe8 sp=0xc000555fe0 pc=0x47ba41
created by testing.(*T).Run in goroutine 1
	/usr/src/testing/testing.go:1742 +0x390

goroutine 25423 gp=0xc000c2efc0 m=nil [runnable]:
runtime.(*_panic).nextDefer(0xc0003e87c8?)
	/usr/src/runtime/panic.go:831 +0x1fa fp=0xc0003e87b0 sp=0xc0003e87a8 pc=0x43feda
runtime.deferreturn()
	/usr/src/runtime/panic.go:598 +0x68 fp=0xc0003e8840 sp=0xc0003e87b0 pc=0x43f4a8
github.com/google/go-cmp/cmp.(*state).compareAny(0xc000c5a000, {0x153d150, 0xc000e16000})
	/home/AzDevOps_azpcontainer/go/pkg/mod/github.com/google/[email protected]/cmp/compare.go:262 +0xd4a fp=0xc0003e8a00 sp=0xc0003e8840 pc=0xcc00aa
github.com/google/go-cmp/cmp.(*state).statelessCompare(0xc000c5a000, {0x153d150?, 0xc000e16000?})
	/home/AzDevOps_azpcontainer/go/pkg/mod/github.com/google/[email protected]/cmp/compare.go:232 +0x7a fp=0xc0003e8a50 sp=0xc0003e8a00 pc=0xcbf2ba
github.com/google/go-cmp/cmp.(*state).compareSlice(0xc000c5a000, {0x1550d08, 0xfa2a20}, {0xfa2a20?, 0xc00011a390?, 0xc000d23f80?}, {0xfa2a20?, 0xc00011a3a8?, 0x30?})
	/home/AzDevOps_azpcontainer/go/pkg/mod/github.com/google/[email protected]/cmp/compare.go:464 +0x56c fp=0xc0003e8c50 sp=0xc0003e8a50 pc=0xcc22cc
github.com/google/go-cmp/cmp.(*state).compareAny(0xc000c5a000, {0x153d240, 0xc000e080c0})
	/home/AzDevOps_azpcontainer/go/pkg/mod/github.com/google/[email protected]/cmp/compare.go:291 +0xbbb fp=0xc0003e8e10 sp=0xc0003e8c50 pc=0xcbff1b
github.com/google/go-cmp/cmp.Diff({0xfa2a20, 0xc00011a390}, {0xfa2a20, 0xc00011a3a8}, {0x0?, 0xc0007a5998?, 0x1543460?})
	/home/AzDevOps_azpcontainer/go/pkg/mod/github.com/google/[email protected]/cmp/compare.go:122 +0x75 fp=0xc0003e8e70 sp=0xc0003e8e10 pc=0xcbe4b5
<redacted>/domains/til/services/transformer/main.Test_getAllBatches.func12.1(0xc000dcc340)
	/__w/1/s/domains/til/services/transformer/main/util_test.go:1251 +0x271 fp=0xc0003e8f70 sp=0xc0003e8e70 pc=0xe9bb51
testing.tRunner(0xc000dcc340, 0xc0007a58a8)
	/usr/src/testing/testing.go:1689 +0xfb fp=0xc0003e8fc0 sp=0xc0003e8f70 pc=0x594c9b
testing.(*T).Run.gowrap1()
	/usr/src/testing/testing.go:1742 +0x25 fp=0xc0003e8fe0 sp=0xc0003e8fc0 pc=0x595cc5
runtime.goexit({})
	/usr/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc0003e8fe8 sp=0xc0003e8fe0 pc=0x47ba41
created by testing.(*T).Run in goroutine 3459
	/usr/src/testing/testing.go:1742 +0x390

goroutine 3459 gp=0xc000c2f180 m=nil [chan receive]:
runtime.gopark(0xc0010dc0d2?, 0xc0010dc168?, 0xd0?, 0xc0?, 0x1072440?)
	/usr/src/runtime/proc.go:402 +0xce fp=0xc0006c3cd8 sp=0xc0006c3cb8 pc=0x4433ce
runtime.chanrecv(0xc000a25260, 0xc0006c3dbf, 0x1)
	/usr/src/runtime/chan.go:583 +0x3bf fp=0xc0006c3d50 sp=0xc0006c3cd8 pc=0x40cf7f
runtime.chanrecv1(0x205f1c0?, 0xfb6f40?)
	/usr/src/runtime/chan.go:442 +0x12 fp=0xc0006c3d78 sp=0xc0006c3d50 pc=0x40cb92
testing.(*T).Run(0xc000dcc680, {0xc00100a0f0?, 0x603?}, 0xc0007a58a8)
	/usr/src/testing/testing.go:1750 +0x3ab fp=0xc0006c3e38 sp=0xc0006c3d78 pc=0x595b6b
<redacted>/domains/til/services/transformer/main.Test_getAllBatches.func12(0xc000dcc680)
	/__w/1/s/domains/til/services/transformer/main/util_test.go:1232 +0x95 fp=0xc0006c3f70 sp=0xc0006c3e38 pc=0xe9b795
testing.tRunner(0xc000dcc680, 0xc000dd4030)
	/usr/src/testing/testing.go:1689 +0xfb fp=0xc0006c3fc0 sp=0xc0006c3f70 pc=0x594c9b
testing.(*T).Run.gowrap1()
	/usr/src/testing/testing.go:1742 +0x25 fp=0xc0006c3fe0 sp=0xc0006c3fc0 pc=0x595cc5
runtime.goexit({})
	/usr/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc0006c3fe8 sp=0xc0006c3fe0 pc=0x47ba41
created by testing.(*T).Run in goroutine 3436
	/usr/src/testing/testing.go:1742 +0x390

goroutine 1292 gp=0xc000c2fa40 m=nil [select]:
runtime.gopark(0xc000611f88?, 0x2?, 0x60?, 0x0?, 0xc000611f84?)
	/usr/src/runtime/proc.go:402 +0xce fp=0xc000611e30 sp=0xc000611e10 pc=0x4433ce
runtime.selectgo(0xc000611f88, 0xc000611f80, 0x0?, 0x0, 0x0?, 0x1)
	/usr/src/runtime/select.go:327 +0x725 fp=0xc000611f50 sp=0xc000611e30 pc=0x455205
database/sql.(*DB).connectionOpener(0xc0010dd450, {0x153ef20, 0xc0010a04b0})
	/usr/src/database/sql/sql.go:1246 +0x87 fp=0xc000611fb8 sp=0xc000611f50 pc=0xac50a7
database/sql.OpenDB.gowrap1()
	/usr/src/database/sql/sql.go:824 +0x28 fp=0xc000611fe0 sp=0xc000611fb8 pc=0xac33c8
runtime.goexit({})
	/usr/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc000611fe8 sp=0xc000611fe0 pc=0x47ba41
created by database/sql.OpenDB in goroutine 1251
	/usr/src/database/sql/sql.go:824 +0x14c

goroutine 1256 gp=0xc000c2fdc0 m=nil [select]:
runtime.gopark(0xc000726788?, 0x2?, 0x60?, 0x0?, 0xc000726784?)
	/usr/src/runtime/proc.go:402 +0xce fp=0xc000726630 sp=0xc000726610 pc=0x4433ce
runtime.selectgo(0xc000726788, 0xc000726780, 0x125a25a?, 0x0, 0x0?, 0x1)
	/usr/src/runtime/select.go:327 +0x725 fp=0xc000726750 sp=0xc000726630 pc=0x455205
database/sql.(*DB).connectionOpener(0xc00030e270, {0x153ef20, 0xc000604050})
	/usr/src/database/sql/sql.go:1246 +0x87 fp=0xc0007267b8 sp=0xc000726750 pc=0xac50a7
database/sql.OpenDB.gowrap1()
	/usr/src/database/sql/sql.go:824 +0x28 fp=0xc0007267e0 sp=0xc0007267b8 pc=0xac33c8
runtime.goexit({})
	/usr/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc0007267e8 sp=0xc0007267e0 pc=0x47ba41
created by database/sql.OpenDB in goroutine 1251
	/usr/src/database/sql/sql.go:824 +0x14c

rax    0x785431ef2398
rbx    0xc000cfa908
rcx    0xc000cfa900
rdx    0xc0010a1450
rdi    0x42
rsi    0x1
rbp    0x7854337fdc78
rsp    0x7854337fdc68
r8     0xc0000d88c0
r9     0xc00006a008
r10    0xc000af9000
r11    0x9
r12    0x7854337fdd08
r13    0x1
r14    0xc000007a40
r15    0x1
rip    0x41c1db
rflags 0x10202
cs     0x33
fs     0x0
gs     0x0
FAIL	<redacted>/domains/til/services/transformer/main	41.253s
FAIL
make: Leaving directory '/__w/1/s/domains/til/services/transformer'

If I'm reading this right, GC was forced upon completion of go-cmp.compareAny(), and crashed.

What did you expect to see?

ok  	<redacted>/domains/til/services/transformer/main	3.456s	coverage: 69% of statements

Usually, that is what we see. I'm seeing the SIGSEGV for the first time today, and have not been able to reproduce it quickly.

The text was updated successfully, but these errors were encountered:

gabyhelp · 2024-07-29T15:23:22Z

Related Issues and Documentation

_{(Emoji vote if this was helpful or unhelpful; more detailed feedback welcome in this discussion.)}

prattmic · 2024-07-29T17:24:55Z

Does that example test crash exactly as written? The only required dependency is go-cmp?

prattmic · 2024-07-29T17:29:31Z

cc @mknyszek @golang/runtime

mrwonko · 2024-07-30T07:15:14Z

Like I wrote, I have been unable to reproduce it, neither using the original code, nor the simplified version I included here. There's a chance that somewhere in the 13000 omitted lines something happens that later impacts GC, but none of them show up in the goroutine dump, and I'm not at liberty to share them.

prattmic · 2024-07-30T15:05:25Z

Ah, OK. The crash here is an invalid pointer in the type referenced by a type header of a heap allocation. The most likely cause of this is your application causing some form of memory corruption, which would usually come from bad cgo or unsafe code someone in the application.

mknyszek · 2024-07-30T17:25:43Z

Unfortunately crashes like this, whatever the root cause, are usually the result of a sequence of events that happened long before the actual crash. I don't think we can make much progress here without a way to reproduce, without a core dump, or anything else more in-depth. If the tests you were running include packages that make frequent use of unsafe or C code, then it's also hard to rule out a bug in a dependency. If you know for a fact that all your tests are pure Go code (for example, they run with CGO_ENABLED=0) and don't make frequent use of unsafe then that would be helpful as that more strongly implicates the standard library, compiler, or runtime.

gopherbot added the compiler/runtime Issues related to the Go compiler and/or runtime. label Jul 29, 2024

prattmic added the NeedsInvestigation Someone must examine and confirm this is a valid issue and not a duplicate of an existing one. label Jul 29, 2024

prattmic added this to the Backlog milestone Jul 29, 2024

mknyszek added this to Go Compiler / Runtime Jul 31, 2024

mknyszek moved this to Todo in Go Compiler / Runtime Jul 31, 2024

mknyszek self-assigned this Jul 31, 2024

This was referenced Aug 28, 2024

maps: segmentation violation in maps.Clone #69110

Closed

runtime: SIGSEGV: segmentation violation #69247

Closed

gabyhelp mentioned this issue Sep 10, 2024

x/sys/unix: mkerrors.sh broken on AIX #69365

Closed

gabyhelp mentioned this issue Sep 26, 2024

runtime: fpTracebackPartialExpand SIGSEGV on amd64 with deep inlining #69629

Closed

gabyhelp mentioned this issue Oct 23, 2024

runtime: cgoCheckPointer fatal error: can't happen #70016

Closed

msbutler mentioned this issue Nov 1, 2024

ccl/backupccl: TestRestoreDatabaseVersusTable failed cockroachdb/cockroach#134020

Closed

gabyhelp mentioned this issue Nov 5, 2024

go1.22.8 runtime error: SIGSEGV: segmentation violation #70194

Closed

gabyhelp mentioned this issue Nov 21, 2024

runtime: AIX build broken with SIGSEGV #70483

Closed

gabyhelp mentioned this issue Jan 7, 2025

runtime: CGo thread stacks stay in physical memory #71150

Open

gabyhelp mentioned this issue Jan 22, 2025

runtime: -msan / -asan stack corruption with CPU profiling and SetCgoTraceback context callback #71395

Open

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

runtime: spurious SIGSEGV in systemstack() #68632

runtime: spurious SIGSEGV in systemstack() #68632

mrwonko commented Jul 29, 2024

gabyhelp commented Jul 29, 2024

prattmic commented Jul 29, 2024

prattmic commented Jul 29, 2024

mrwonko commented Jul 30, 2024

prattmic commented Jul 30, 2024

mknyszek commented Jul 30, 2024

runtime: spurious SIGSEGV in systemstack() #68632

runtime: spurious SIGSEGV in systemstack() #68632

Comments

mrwonko commented Jul 29, 2024

Go version

Output of go env in your module/workspace:

What did you do?

What did you see happen?

What did you expect to see?

gabyhelp commented Jul 29, 2024

prattmic commented Jul 29, 2024

prattmic commented Jul 29, 2024

mrwonko commented Jul 30, 2024

prattmic commented Jul 30, 2024

mknyszek commented Jul 30, 2024

Output of `go env` in your module/workspace: