diff --git a/Makefile b/Makefile index 9a8c8b15fb3..a31bcfa48fb 100644 --- a/Makefile +++ b/Makefile @@ -12,8 +12,8 @@ GOCHECKER := awk '{ print } END { if (NR > 0) { exit 1 } }' RETOOL := ./scripts/retool OVERALLS := overalls -GOFAIL_ENABLE := $$(find $$PWD/ -type d | grep -vE "(\.git|\.retools)" | xargs ./scripts/retool do gofail enable) -GOFAIL_DISABLE := $$(find $$PWD/ -type d | grep -vE "(\.git|\.retools)" | xargs ./scripts/retool do gofail disable) +FAILPOINT_ENABLE := $$(find $$PWD/ -type d | grep -vE "(\.git|\.retools)" | xargs ./scripts/retool do failpoint-ctl enable) +FAILPOINT_DISABLE := $$(find $$PWD/ -type d | grep -vE "(\.git|\.retools)" | xargs ./scripts/retool do failpoint-ctl disable) LDFLAGS += -X "$(PD_PKG)/server.PDReleaseVersion=$(shell git describe --tags --dirty)" LDFLAGS += -X "$(PD_PKG)/server.PDBuildTS=$(shell date -u '+%Y-%m-%d %I:%M:%S')" @@ -48,14 +48,14 @@ endif test: retool-setup # testing.. - @$(GOFAIL_ENABLE) - CGO_ENABLED=1 GO111MODULE=on go test -race -cover $(TEST_PKGS) || { $(GOFAIL_DISABLE); exit 1; } - @$(GOFAIL_DISABLE) + @$(FAILPOINT_ENABLE) + CGO_ENABLED=1 GO111MODULE=on go test -race -cover $(TEST_PKGS) || { $(FAILPOINT_DISABLE); exit 1; } + @$(FAILPOINT_DISABLE) basic-test: - @$(GOFAIL_ENABLE) - GO111MODULE=on go test $(BASIC_TEST_PKGS) || { $(GOFAIL_DISABLE); exit 1; } - @$(GOFAIL_DISABLE) + @$(FAILPOINT_ENABLE) + GO111MODULE=on go test $(BASIC_TEST_PKGS) || { $(FAILPOINT_DISABLE); exit 1; } + @$(FAILPOINT_DISABLE) # These need to be fixed before they can be ran regularly check-fail: @@ -98,9 +98,9 @@ tidy: travis_coverage: export GO111MODULE=on travis_coverage: ifeq ("$(TRAVIS_COVERAGE)", "1") - @$(GOFAIL_ENABLE) - CGO_ENABLED=1 ./scripts/retool do $(OVERALLS) -project=github.com/pingcap/pd -covermode=count -ignore='.git,vendor' -- -coverpkg=./... || { $(GOFAIL_DISABLE); exit 1; } - @$(GOFAIL_DISABLE) + @$(FAILPOINT_ENABLE) + CGO_ENABLED=1 ./scripts/retool do $(OVERALLS) -project=github.com/pingcap/pd -covermode=count -ignore='.git,vendor' -- -coverpkg=./... || { $(FAILPOINT_DISABLE); exit 1; } + @$(FAILPOINT_DISABLE) else @echo "coverage only runs in travis." endif @@ -114,12 +114,12 @@ clean-test: rm -rf /tmp/pd-tests* rm -rf /tmp/test_etcd* -gofail-enable: - # Converting gofail failpoints... - @$(GOFAIL_ENABLE) +failpoint-enable: + # Converting failpoints... + @$(FAILPOINT_ENABLE) -gofail-disable: - # Restoring gofail failpoints... - @$(GOFAIL_DISABLE) +failpoint-disable: + # Restoring failpoints... + @$(FAILPOINT_DISABLE) .PHONY: all ci vendor clean-test tidy diff --git a/go.mod b/go.mod index 55930166716..8e643602226 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 github.com/pingcap/errcode v0.0.0-20180921232412-a1a7271709d9 github.com/pingcap/errors v0.10.1 // indirect - github.com/pingcap/gofail v0.0.0-20181217135706-6a951c1e42c3 + github.com/pingcap/failpoint v0.0.0-20190512135322-30cc7431d99c github.com/pingcap/kvproto v0.0.0-20190327032727-3d8cb3a30d5d github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7 github.com/pkg/errors v0.8.1 diff --git a/go.sum b/go.sum index 37ac9eb6bdf..59c8c800ee7 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,6 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= @@ -30,6 +31,7 @@ github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 h1:Mn26/9ZMNWSw9C9ERFA1PUxfmGpolnw2v0bKOREu5ew= github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32/go.mod h1:GIjDIg/heH5DOkXY3YJ/wNhfHsQHoXGjl8G8amsYQ1I= +github.com/go-playground/overalls v0.0.0-20180201144345-22ec1a223b7c/go.mod h1:UqxAgEOt89sCiXlrc/ycnx00LVvUO/eS8tMUkWX4R7w= github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.0.0 h1:2jyBKDKU/8v3v2xVR2PtiWQviFUyiaGk2rpfyFT8rTM= github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -46,6 +48,7 @@ github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8l github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE= github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f h1:9oNbS1z4rVpbnkHBdPZU4jo9bSmrLpII768arSyMFgk= @@ -75,8 +78,11 @@ github.com/kr/pty v1.0.0/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-runewidth v0.0.2 h1:UnlwIPBGaTZfPQ6T1IGzPI0EkYAQmT9fAEJ/poFC63o= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-shellwords v1.0.3 h1:K/VxK7SZ+cvuPgFSLKi5QPI9Vr/ipOf4C1gN+ntueUk= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= @@ -84,6 +90,7 @@ github.com/matttproud/golang_protobuf_extensions v1.0.0 h1:YNOwxxSJzSUARoD9KRZLz github.com/matttproud/golang_protobuf_extensions v1.0.0/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/montanaflynn/stats v0.0.0-20151014174947-eeaced052adb h1:bsjNADsjHq0gjU7KO7zwoX5k3HtFdf6TDzB3ncl5iUs= github.com/montanaflynn/stats v0.0.0-20151014174947-eeaced052adb/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -91,14 +98,15 @@ github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I= github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pelletier/go-toml v1.3.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 h1:USx2/E1bX46VG32FIw034Au6seQ2fY9NEILmNh/UlQg= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= github.com/pingcap/errcode v0.0.0-20180921232412-a1a7271709d9 h1:KH4f4Si9XK6/IW50HtoaiLIFHGkapOM6w83za47UYik= github.com/pingcap/errcode v0.0.0-20180921232412-a1a7271709d9/go.mod h1:4b2X8xSqxIroj/IZ9MX/VGZhAwc11wB9wRIzHvz6SeM= github.com/pingcap/errors v0.10.1 h1:fGVuPMtwNcxbzQ3aoRyyi6kxvXKMkEsceP81f3b8wsk= github.com/pingcap/errors v0.10.1/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= -github.com/pingcap/gofail v0.0.0-20181217135706-6a951c1e42c3 h1:04yuCf5NMvLU8rB2m4Qs3rynH7EYpMno3lHkewIOdMo= -github.com/pingcap/gofail v0.0.0-20181217135706-6a951c1e42c3/go.mod h1:DazNTg0PTldtpsQiT9I5tVJwV1onHMKBBgXzmJUlMns= +github.com/pingcap/failpoint v0.0.0-20190512135322-30cc7431d99c h1:hvQd3aOLKLF7xvRV6DzvPkKY4QXzfVbjU1BhW0d9yL8= +github.com/pingcap/failpoint v0.0.0-20190512135322-30cc7431d99c/go.mod h1:DNS3Qg7bEDhU6EXNHF+XSv/PGznQaMJ5FWvctpm6pQI= github.com/pingcap/kvproto v0.0.0-20190327032727-3d8cb3a30d5d h1:LJYJl+cBhkkTWD79n+n9Bp4agQ85SdF9YKY4zEtL9Kw= github.com/pingcap/kvproto v0.0.0-20190327032727-3d8cb3a30d5d/go.mod h1:QMdbTAXCHzzygQzqcG9uVUgU2fKeSN1GmfMiykdSzzY= github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7 h1:kOHAMalwF69bJrtWrOdVaCSvZjLucrJhP4NQKIu6uM4= @@ -117,6 +125,7 @@ github.com/prometheus/common v0.0.0-20180518154759-7600349dcfe1 h1:osmNoEW2SCW3L github.com/prometheus/common v0.0.0-20180518154759-7600349dcfe1/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/procfs v0.0.0-20180612222113-7d6f385de8be h1:MoyXp/VjXUwM0GyDcdwT7Ubea2gxOSHpPaFo3qV+Y2A= github.com/prometheus/procfs v0.0.0-20180612222113-7d6f385de8be/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sirupsen/logrus v1.0.5 h1:8c8b5uO0zS4X6RPl/sd1ENwSkIc0/H2PaHxE3udaE8I= github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= @@ -145,6 +154,7 @@ github.com/urfave/negroni v0.3.0 h1:PaXOb61mWeZJxc1Ji2xJjpVg9QfPo0rrB+lHyBxGNSU= github.com/urfave/negroni v0.3.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/yookoala/realpath v1.0.0/go.mod h1:gJJMA9wuX7AcqLy1+ffPatSCySA1FQ2S8Ya9AIoYBpE= go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20190320044326-77d4b742cdbf h1:rmttwKPEgG/l4UscTDYtaJgeUsedKPKSyFfNQLI6q+I= @@ -178,6 +188,8 @@ google.golang.org/grpc v1.14.0 h1:ArxJuB1NWfPY6r9Gp9gqwplT0Ge7nqv9msgu03lHLmo= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= +gopkg.in/alecthomas/gometalinter.v2 v2.0.12/go.mod h1:NDRytsqEZyolNuAgTzJkZMkSQM7FIKyzVzGhjB/qfYo= +gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20180810215634-df19058c872c/go.mod h1:3HH7i1SgMqlzxCcBmUHW657sD4Kvv9sC3HpL3YukzwA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= diff --git a/scripts/retool-install.sh b/scripts/retool-install.sh index 2fa1afeb96f..c72e09af658 100755 --- a/scripts/retool-install.sh +++ b/scripts/retool-install.sh @@ -22,4 +22,4 @@ which retool >/dev/null || go get github.com/twitchtv/retool ./scripts/retool add github.com/mgechev/revive 7773f47324c2bf1c8f7a5500aff2b6c01d3ed73b ./scripts/retool add github.com/securego/gosec/cmd/gosec 1.0.0 # go fail -./scripts/retool add github.com/pingcap/gofail master +./scripts/retool add github.com/pingcap/failpoint/failpoint-ctl master diff --git a/server/cluster.go b/server/cluster.go index 76d1e9e04ff..0784d4f7032 100644 --- a/server/cluster.go +++ b/server/cluster.go @@ -21,6 +21,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pingcap/errcode" + "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" log "github.com/pingcap/log" @@ -120,10 +121,9 @@ func (c *RaftCluster) start() error { c.wg.Add(3) go c.runCoordinator() - // gofail: var highFrequencyClusterJobs bool - // if highFrequencyClusterJobs { - // backgroundJobInterval = 100 * time.Microsecond - // } + failpoint.Inject("highFrequencyClusterJobs", func() { + backgroundJobInterval = 100 * time.Microsecond + }) go c.runBackgroundJobs(backgroundJobInterval) go c.syncRegions() c.running = true diff --git a/server/cluster_test.go b/server/cluster_test.go index 1a20705cc7c..78c4dde1c58 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -21,7 +21,7 @@ import ( "time" . "github.com/pingcap/check" - gofail "github.com/pingcap/gofail/runtime" + "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/pd/server/core" @@ -452,7 +452,7 @@ func (s *testClusterSuite) TestRaftClusterMultipleRestart(c *C) { c.Assert(cluster, NotNil) // let the job run at small interval - gofail.Enable("github.com/pingcap/pd/server/highFrequencyClusterJobs", `return(true)`) + c.Assert(failpoint.Enable("github.com/pingcap/pd/server/highFrequencyClusterJobs", `return(true)`), IsNil) for i := 0; i < 100; i++ { err = s.svr.createRaftCluster() c.Assert(err, IsNil) diff --git a/server/leader.go b/server/leader.go index fcc03d401eb..3f78de0fdc3 100644 --- a/server/leader.go +++ b/server/leader.go @@ -20,6 +20,7 @@ import ( "strings" "time" + "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/pdpb" log "github.com/pingcap/log" "github.com/pingcap/pd/pkg/etcdutil" @@ -321,7 +322,7 @@ func (s *Server) watchLeader(leader *pdpb.Member, revision int64) { // If the revision is compacted, will meet required revision has been compacted error. // In this case, use the compact revision to re-watch the key. for { - // gofail: var delayWatcher struct{} + failpoint.Inject("delayWatcher", nil) rch := watcher.Watch(ctx, s.getLeaderPath(), clientv3.WithRev(revision)) for wresp := range rch { // meet compacted error, use the compact revision. diff --git a/server/tso.go b/server/tso.go index 54cdeca2c35..17d2ebaad18 100644 --- a/server/tso.go +++ b/server/tso.go @@ -18,6 +18,7 @@ import ( "sync/atomic" "time" + "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/pdpb" log "github.com/pingcap/log" "github.com/pkg/errors" @@ -84,10 +85,9 @@ func (s *Server) syncTimestamp() error { } next := time.Now() - // gofail: var fallBackSync bool - // if fallBackSync { - // next = next.Add(time.Hour) - // } + failpoint.Inject("fallBackSync", func() { + next = next.Add(time.Hour) + }) // If the current system time minus the saved etcd timestamp is less than `updateTimestampGuard`, // the timestamp allocation will start from the saved etcd timestamp temporarily. @@ -126,10 +126,9 @@ func (s *Server) updateTimestamp() error { prev := s.ts.Load().(*atomicObject) now := time.Now() - // gofail: var fallBackUpdate bool - // if fallBackUpdate { - // now = now.Add(time.Hour) - // } + failpoint.Inject("fallBackUpdate", func() { + now = now.Add(time.Hour) + }) tsoCounter.WithLabelValues("save").Inc() diff --git a/server/tso_test.go b/server/tso_test.go index fd4b69ec569..7084b3e16ad 100644 --- a/server/tso_test.go +++ b/server/tso_test.go @@ -19,7 +19,7 @@ import ( "time" . "github.com/pingcap/check" - gofail "github.com/pingcap/gofail/runtime" + "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/pdpb" "go.etcd.io/etcd/clientv3" ) @@ -113,15 +113,15 @@ type testTimeFallBackSuite struct { } func (s *testTimeFallBackSuite) SetUpSuite(c *C) { - gofail.Enable("github.com/pingcap/pd/server/fallBackSync", `return(true)`) - gofail.Enable("github.com/pingcap/pd/server/fallBackUpadte", `return(true)`) + c.Assert(failpoint.Enable("github.com/pingcap/pd/server/fallBackSync", `return(true)`), IsNil) + c.Assert(failpoint.Enable("github.com/pingcap/pd/server/fallBackUpdate", `return(true)`), IsNil) s.svr, s.cleanup = mustRunTestServer(c) s.client = s.svr.client mustWaitLeader(c, []*Server{s.svr}) s.grpcPDClient = mustNewGrpcClient(c, s.svr.GetAddr()) s.svr.Close() - gofail.Disable("github.com/pingcap/pd/server/fallBackSync") - gofail.Disable("github.com/pingcap/pd/server/fallBackUpdate") + failpoint.Disable("github.com/pingcap/pd/server/fallBackSync") + failpoint.Disable("github.com/pingcap/pd/server/fallBackUpdate") err := s.svr.Run(context.TODO()) c.Assert(err, IsNil) mustWaitLeader(c, []*Server{s.svr}) diff --git a/tests/server/leader_watch_test.go b/tests/server/leader_watch_test.go index 860ad34a27c..3a48db7af78 100644 --- a/tests/server/leader_watch_test.go +++ b/tests/server/leader_watch_test.go @@ -18,7 +18,7 @@ import ( "time" . "github.com/pingcap/check" - gofail "github.com/pingcap/gofail/runtime" + "github.com/pingcap/failpoint" "github.com/pingcap/pd/pkg/testutil" "github.com/pingcap/pd/server" "github.com/pingcap/pd/tests" @@ -45,7 +45,7 @@ func (s *serverTestSuite) TestWatcher(c *C) { time.Sleep(5 * time.Second) pd3, err := cluster.Join() c.Assert(err, IsNil) - gofail.Enable("github.com/pingcap/pd/server/delayWatcher", `pause`) + c.Assert(failpoint.Enable("github.com/pingcap/pd/server/delayWatcher", `pause`), IsNil) err = pd3.Run(context.Background()) c.Assert(err, IsNil) time.Sleep(200 * time.Millisecond) @@ -54,7 +54,7 @@ func (s *serverTestSuite) TestWatcher(c *C) { c.Assert(err, IsNil) cluster.WaitLeader() c.Assert(pd2.GetLeader().GetName(), Equals, pd2.GetConfig().Name) - gofail.Disable("github.com/pingcap/pd/server/delayWatcher") + failpoint.Disable("github.com/pingcap/pd/server/delayWatcher") testutil.WaitUntil(c, func(c *C) bool { return c.Check(pd3.GetLeader().GetName(), Equals, pd2.GetConfig().Name) }) diff --git a/tools.json b/tools.json index e7359a9bd34..4e3d043e673 100644 --- a/tools.json +++ b/tools.json @@ -9,8 +9,8 @@ "Commit": "4a96d43e39d340b63daa8bc5576985aa599885f6" }, { - "Repository": "github.com/pingcap/gofail", - "Commit": "c3f835e5a7d8cedf2f61cc55f961d4d120ae90eb" + "Repository": "github.com/pingcap/failpoint/failpoint-ctl", + "Commit": "30cc7431d99c6a7f2836387d4bb255a3bd6a5e0a" }, { "Repository": "github.com/go-playground/overalls",