diff --git a/pkg/server/BUILD.bazel b/pkg/server/BUILD.bazel
index cc6391de1ddc8..b9466e7c4582d 100644
--- a/pkg/server/BUILD.bazel
+++ b/pkg/server/BUILD.bazel
@@ -16,7 +16,6 @@ go_library(
         "rpc_server.go",
         "server.go",
         "stat.go",
-        "tokenlimiter.go",
     ],
     importpath = "github.com/pingcap/tidb/pkg/server",
     visibility = ["//visibility:public"],
diff --git a/pkg/server/server.go b/pkg/server/server.go
index 448dd39750f67..57b46ae96cd79 100644
--- a/pkg/server/server.go
+++ b/pkg/server/server.go
@@ -119,7 +119,7 @@ type Server struct {
 	driver            IDriver
 	listener          net.Listener
 	socket            net.Listener
-	concurrentLimiter *TokenLimiter
+	concurrentLimiter *util.TokenLimiter
 
 	rwlock  sync.RWMutex
 	clients map[uint64]*clientConn
@@ -201,7 +201,7 @@ func (s *Server) ConnectionCount() int {
 	return cnt
 }
 
-func (s *Server) getToken() *Token {
+func (s *Server) getToken() *util.Token {
 	start := time.Now()
 	tok := s.concurrentLimiter.Get()
 	metrics.TokenGauge.Inc()
@@ -210,7 +210,7 @@ func (s *Server) getToken() *Token {
 	return tok
 }
 
-func (s *Server) releaseToken(token *Token) {
+func (s *Server) releaseToken(token *util.Token) {
 	s.concurrentLimiter.Put(token)
 	metrics.TokenGauge.Dec()
 }
@@ -242,7 +242,7 @@ func NewServer(cfg *config.Config, driver IDriver) (*Server, error) {
 	s := &Server{
 		cfg:               cfg,
 		driver:            driver,
-		concurrentLimiter: NewTokenLimiter(cfg.TokenLimit),
+		concurrentLimiter: util.NewTokenLimiter(cfg.TokenLimit),
 		clients:           make(map[uint64]*clientConn),
 		internalSessions:  make(map[any]struct{}, 100),
 		health:            uatomic.NewBool(false),
diff --git a/pkg/util/BUILD.bazel b/pkg/util/BUILD.bazel
index 315ae7180d24c..0af7f3ae6e948 100644
--- a/pkg/util/BUILD.bazel
+++ b/pkg/util/BUILD.bazel
@@ -16,6 +16,7 @@ go_library(
         "rlimit_other.go",
         "rlimit_windows.go",
         "security.go",
+        "tokenlimiter.go",
         "urls.go",
         "util.go",
         "wait_group_wrapper.go",
diff --git a/pkg/server/tokenlimiter.go b/pkg/util/tokenlimiter.go
similarity index 98%
rename from pkg/server/tokenlimiter.go
rename to pkg/util/tokenlimiter.go
index 57f917479fae2..2e7924638a886 100644
--- a/pkg/server/tokenlimiter.go
+++ b/pkg/util/tokenlimiter.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package server
+package util
 
 // Token is used as a permission to keep on running.
 type Token struct {