diff --git a/pkg/statistics/handle/bootstrap.go b/pkg/statistics/handle/bootstrap.go
index 33f2d3c279fb8..d238c620fec8d 100644
--- a/pkg/statistics/handle/bootstrap.go
+++ b/pkg/statistics/handle/bootstrap.go
@@ -751,11 +751,8 @@ func (h *Handle) InitStats(is infoschema.InfoSchema) (err error) {
 	for _, table := range cache.Values() {
 		for _, col := range table.Columns {
 			if col.StatsAvailable() {
-				if mysql.HasPriKeyFlag(col.Info.GetFlag()) {
-					col.StatsLoadedStatus = statistics.NewStatsFullLoadStatus()
-				} else {
-					col.StatsLoadedStatus = statistics.NewStatsAllEvictedStatus()
-				}
+				// primary key column has no stats info, because primary key's is_index is false. so it cannot load the topn
+				col.StatsLoadedStatus = statistics.NewStatsAllEvictedStatus()
 			}
 		}
 	}
diff --git a/pkg/statistics/handle/handletest/statstest/BUILD.bazel b/pkg/statistics/handle/handletest/statstest/BUILD.bazel
index f29409c070428..9a2c6ceb69313 100644
--- a/pkg/statistics/handle/handletest/statstest/BUILD.bazel
+++ b/pkg/statistics/handle/handletest/statstest/BUILD.bazel
@@ -13,6 +13,7 @@ go_test(
     deps = [
         "//pkg/config",
         "//pkg/parser/model",
+        "//pkg/parser/mysql",
         "//pkg/statistics/handle/internal",
         "//pkg/testkit",
         "//pkg/testkit/testsetup",
diff --git a/pkg/statistics/handle/handletest/statstest/stats_test.go b/pkg/statistics/handle/handletest/statstest/stats_test.go
index 55f0dbf2822ad..3d9a0c1066635 100644
--- a/pkg/statistics/handle/handletest/statstest/stats_test.go
+++ b/pkg/statistics/handle/handletest/statstest/stats_test.go
@@ -22,6 +22,7 @@ import (
 	"github.com/pingcap/failpoint"
 	"github.com/pingcap/tidb/pkg/config"
 	"github.com/pingcap/tidb/pkg/parser/model"
+	"github.com/pingcap/tidb/pkg/parser/mysql"
 	"github.com/pingcap/tidb/pkg/statistics/handle/internal"
 	"github.com/pingcap/tidb/pkg/testkit"
 	"github.com/stretchr/testify/require"
@@ -314,6 +315,16 @@ func TestInitStats51358(t *testing.T) {
 		require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/statistics/handle/cache/StatsCacheGetNil"))
 	}()
 	require.NoError(t, h.InitStats(is))
+	tbl, err := dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
+	require.NoError(t, err)
+	stats := h.GetTableStats(tbl.Meta())
+	for _, column := range stats.Columns {
+		if mysql.HasPriKeyFlag(column.Info.GetFlag()) {
+			// primary key column has no stats info, because primary key's is_index is false. so it cannot load the topn
+			require.Nil(t, column.TopN)
+		}
+		require.False(t, column.IsFullLoad())
+	}
 }
 
 func TestInitStatsVer2(t *testing.T) {