Release v2020.07.15.1 (#692)

* ui: Fix TiFlash log searching (#680)
* ui: Add a whitelist in i18next to fix the issue #675 (#677)
* docs: add ericsyh as a contributor (#681)
* ui: refine code and error handle (#662)
* *: Rename disable_telemetry to enable_telemetry (#684)
* ui: Update space behaviour in statement / slow log search (#682)
* ui: No need to generate ui library (#687)
* forwarder: evict invalid current picked remote (#689)
* keyviz: support to merge cold logical range (#674)
* Update release version

.all-contributorsrc

@@ -121,6 +121,15 @@
       "contributions": [
         "code"
       ]
+    },
+    {
+      "login": "ericsyh",
+      "name": "Eric Shen",
+      "avatar_url": "https://avatars3.githubusercontent.com/u/10498732?v=4",
+      "profile": "https://github.com/ericsyh",
+      "contributions": [
+        "code"
+      ]
     }
   ],
   "contributorsPerLine": 10,

README.md

@@ -59,6 +59,7 @@ Thank you to all the people who already contributed to TiDB Dashboard!
     <td align="center"><a href="https://github.com/yikeke"><img src="https://avatars1.githubusercontent.com/u/40977455?v=4" width="50px;" alt=""/></a></td>
     <td align="center"><a href="https://github.com/qxhy123"><img src="https://avatars2.githubusercontent.com/u/518969?v=4" width="50px;" alt=""/></a></td>
     <td align="center"><a href="http://www.rustin.cn"><img src="https://avatars0.githubusercontent.com/u/29879298?v=4" width="50px;" alt=""/></a></td>
+    <td align="center"><a href="https://github.com/ericsyh"><img src="https://avatars3.githubusercontent.com/u/10498732?v=4" width="50px;" alt=""/></a></td>
   </tr>
 </table>
 

cmd/tidb-dashboard/main.go

@@ -71,7 +71,7 @@ func NewCLIConfig() *DashboardCLIConfig {
 	flag.StringVar(&cfg.CoreConfig.DataDir, "data-dir", "/tmp/dashboard-data", "path to the Dashboard Server data directory")
 	flag.StringVar(&cfg.CoreConfig.PublicPathPrefix, "path-prefix", config.DefaultPublicPathPrefix, "public URL path prefix for reverse proxies")
 	flag.StringVar(&cfg.CoreConfig.PDEndPoint, "pd", "http://127.0.0.1:2379", "PD endpoint address that Dashboard Server connects to")
-	flag.BoolVar(&cfg.CoreConfig.DisableTelemetry, "disable-telemetry", false, "disable client to report data")
+	flag.BoolVar(&cfg.CoreConfig.EnableTelemetry, "enable-telemetry", true, "enable client to report data for analysis")
 
 	showVersion := flag.BoolP("version", "v", false, "print version information and exit")
 

pkg/apiserver/info/info.go

@@ -47,9 +47,8 @@ func Register(r *gin.RouterGroup, auth *user.AuthService, s *Service) {
 }
 
 type InfoResponse struct { //nolint:golint
-	Version          *version.Info `json:"version"`
-	PDEndPoint       string        `json:"pd_end_point"`
-	DisableTelemetry bool          `json:"disable_telemetry"`
+	Version         *version.Info `json:"version"`
+	EnableTelemetry bool          `json:"enable_telemetry"`
 }
 
 // @Summary Dashboard info
@@ -62,9 +61,8 @@ type InfoResponse struct { //nolint:golint
 // @Failure 401 {object} utils.APIError "Unauthorized failure"
 func (s *Service) infoHandler(c *gin.Context) {
 	resp := InfoResponse{
-		Version:          version.GetInfo(),
-		PDEndPoint:       s.config.PDEndPoint,
-		DisableTelemetry: s.config.DisableTelemetry,
+		Version:         version.GetInfo(),
+		EnableTelemetry: s.config.EnableTelemetry,
 	}
 	c.JSON(http.StatusOK, resp)
 }

pkg/apiserver/logsearch/service.go

@@ -74,16 +74,20 @@ func NewService(lc fx.Lifecycle, config *config.Config, db *dbstore.DB) *Service
 
 func Register(r *gin.RouterGroup, auth *user.AuthService, s *Service) {
 	endpoint := r.Group("/logs")
-
-	endpoint.GET("/download", s.DownloadLogs)
-	endpoint.GET("/download/acquire_token", auth.MWAuthRequired(), s.GetDownloadToken)
-	endpoint.PUT("/taskgroup", auth.MWAuthRequired(), s.CreateTaskGroup)
-	endpoint.GET("/taskgroups", auth.MWAuthRequired(), s.GetAllTaskGroups)
-	endpoint.GET("/taskgroups/:id", auth.MWAuthRequired(), s.GetTaskGroup)
-	endpoint.GET("/taskgroups/:id/preview", auth.MWAuthRequired(), s.GetTaskGroupPreview)
-	endpoint.POST("/taskgroups/:id/retry", auth.MWAuthRequired(), s.RetryTask)
-	endpoint.POST("/taskgroups/:id/cancel", auth.MWAuthRequired(), s.CancelTask)
-	endpoint.DELETE("/taskgroups/:id", auth.MWAuthRequired(), s.DeleteTaskGroup)
+	{
+		endpoint.GET("/download", s.DownloadLogs)
+		endpoint.Use(auth.MWAuthRequired())
+		{
+			endpoint.GET("/download/acquire_token", s.GetDownloadToken)
+			endpoint.PUT("/taskgroup", s.CreateTaskGroup)
+			endpoint.GET("/taskgroups", s.GetAllTaskGroups)
+			endpoint.GET("/taskgroups/:id", s.GetTaskGroup)
+			endpoint.GET("/taskgroups/:id/preview", s.GetTaskGroupPreview)
+			endpoint.POST("/taskgroups/:id/retry", s.RetryTask)
+			endpoint.POST("/taskgroups/:id/cancel", s.CancelTask)
+			endpoint.DELETE("/taskgroups/:id", s.DeleteTaskGroup)
+		}
+	}
 }
 
 type CreateTaskGroupRequest struct {

pkg/apiserver/slowquery/queries.go

@@ -150,12 +150,16 @@ func QuerySlowLogList(db *gorm.DB, req *GetListRequest) ([]Base, error) {
 
 	if req.Text != "" {
 		lowerStr := strings.ToLower(req.Text)
-		tx = tx.Where("txn_start_ts REGEXP ? OR LOWER(digest) REGEXP ? OR LOWER(CONVERT(prev_stmt USING utf8)) REGEXP ? OR LOWER(CONVERT(query USING utf8)) REGEXP ?",
-			lowerStr,
-			lowerStr,
-			lowerStr,
-			lowerStr,
-		)
+		arr := strings.Fields(lowerStr)
+		for _, v := range arr {
+			tx = tx.Where(
+				`txn_start_ts REGEXP ?
+				 OR LOWER(digest) REGEXP ?
+				 OR LOWER(CONVERT(prev_stmt USING utf8)) REGEXP ?
+				 OR LOWER(CONVERT(query USING utf8)) REGEXP ?`,
+				v, v, v, v,
+			)
+		}
 	}
 
 	if len(req.DB) > 0 {

pkg/apiserver/statement/queries.go

@@ -176,14 +176,17 @@ func QueryStatementsOverview(
 
 	if len(text) > 0 {
 		lowerText := strings.ToLower(text)
-		query = query.Where(
-			`LOWER(digest_text) REGEXP ?
-			 OR LOWER(digest) REGEXP ?
-			 OR LOWER(schema_name) REGEXP ?
-			 OR LOWER(table_names) REGEXP ?
-			 OR LOWER(plan) REGEXP ?`,
-			lowerText, lowerText, lowerText, lowerText, lowerText,
-		)
+		arr := strings.Fields(lowerText)
+		for _, v := range arr {
+			query = query.Where(
+				`LOWER(digest_text) REGEXP ?
+				 OR LOWER(digest) REGEXP ?
+				 OR LOWER(schema_name) REGEXP ?
+				 OR LOWER(table_names) REGEXP ?
+				 OR LOWER(plan) REGEXP ?`,
+				v, v, v, v, v,
+			)
+		}
 	}
 
 	err = query.Find(&result).Error

pkg/config/config.go

@@ -39,8 +39,8 @@ type Config struct {
 	// TLS config for mTLS authentication between TiDB and MySQL client.
 	TiDBTLSConfig *tls.Config
 
-	// Disable client to report data for analysis
-	DisableTelemetry bool
+	// Enable client to report data for analysis
+	EnableTelemetry bool
 }
 
 func (c *Config) NormalizePDEndPoint() error {

pkg/http/http.go

@@ -26,7 +26,7 @@ import (
 )
 
 const (
-	Timeout = time.Second * 3
+	Timeout = time.Second * 10
 )
 
 func NewHTTPClientWithConf(lc fx.Lifecycle, config *config.Config) *http.Client {

pkg/keyvisual/decorator/decorator.go

@@ -29,7 +29,10 @@ type LabelKey struct {
 // LabelStrategy requires cross-border determination and key decoration scheme.
 type LabelStrategy interface {
 	ReloadConfig(cfg *config.KeyVisualConfig)
+	// CrossBorder determines whether two keys not belong to the same logical range.
 	CrossBorder(startKey, endKey string) bool
+	// Label returns the Label information of the key.
+	// Note: When the key is "", need to use LabelGlobalStart or LabelGlobalEnd.
 	Label(key string) LabelKey
 	LabelGlobalStart() LabelKey
 	LabelGlobalEnd() LabelKey

pkg/keyvisual/matrix/axis.go

@@ -83,7 +83,7 @@ func (axis *Axis) Focus(strategy Strategy, threshold uint64, ratio int, target i
 	}
 
 	baseChunk := createChunk(axis.Keys, axis.ValuesList[0])
-	newChunk := baseChunk.Focus(strategy, threshold, ratio, target)
+	newChunk := baseChunk.Focus(strategy, threshold, ratio, target, MergeColdLogicalRange)
 	valuesListLen := len(axis.ValuesList)
 	newValuesList := make([][]uint64, valuesListLen)
 	newValuesList[0] = newChunk.Values
@@ -102,7 +102,7 @@ func (axis *Axis) Divide(strategy Strategy, target int) Axis {
 	}
 
 	baseChunk := createChunk(axis.Keys, axis.ValuesList[0])
-	newChunk := baseChunk.Divide(strategy, target)
+	newChunk := baseChunk.Divide(strategy, target, MergeColdLogicalRange)
 	valuesListLen := len(axis.ValuesList)
 	newValuesList := make([][]uint64, valuesListLen)
 	newValuesList[0] = newChunk.Values
@@ -113,6 +113,13 @@ func (axis *Axis) Divide(strategy Strategy, target int) Axis {
 	return CreateAxis(newChunk.Keys, newValuesList)
 }
 
+type FocusMode int
+
+const (
+	NotMergeLogicalRange FocusMode = iota
+	MergeColdLogicalRange
+)
+
 type chunk struct {
 	// Keys and ValuesList[i] from Axis
 	Keys   []string
@@ -209,7 +216,7 @@ func (c *chunk) GetFocusRows(threshold uint64) (count int) {
 // Given a `threshold`, merge the rows with less traffic,
 // and merge the most `ratio` rows at a time.
 // `target` is the estimated final number of rows.
-func (c *chunk) Focus(strategy Strategy, threshold uint64, ratio int, target int) chunk {
+func (c *chunk) Focus(strategy Strategy, threshold uint64, ratio int, target int, mode FocusMode) chunk {
 	newKeys := make([]string, 0, target)
 	newValues := make([]uint64, 0, target)
 	newKeys = append(newKeys, c.Keys[0])
@@ -236,11 +243,66 @@ func (c *chunk) Focus(strategy Strategy, threshold uint64, ratio int, target int
 	}
 	generateBucket(len(c.Values))
 
+	newChunk := createChunk(newKeys, newValues)
+	if mode == MergeColdLogicalRange && len(newValues) >= target {
+		newChunk = newChunk.MergeColdLogicalRange(strategy, threshold, target)
+	}
+	return newChunk
+}
+
+func (c *chunk) MergeColdLogicalRange(strategy Strategy, threshold uint64, target int) chunk {
+	threshold /= 4 // TODO: This var can be adjusted
+
+	newKeys := make([]string, 0, target)
+	newValues := make([]uint64, 0, target)
+	newKeys = append(newKeys, c.Keys[0])
+
+	coldStart := 0
+	coldEnd := 0
+	var coldRangeSum uint64 = 0
+	mergeColdRange := func() {
+		if coldEnd <= coldStart {
+			return
+		}
+		newKeys = append(newKeys, c.Keys[coldEnd])
+		newValues = append(newValues, coldRangeSum)
+		coldStart = coldEnd
+		coldRangeSum = 0
+	}
+	generateRange := func(end int) {
+		if end <= coldEnd {
+			return
+		}
+		var rangeSum uint64 = 0
+		for i := coldEnd; i < end; i++ {
+			rangeSum += c.Values[i]
+		}
+		if coldRangeSum > threshold || rangeSum > threshold {
+			mergeColdRange()
+		}
+		if rangeSum > threshold {
+			newKeys = append(newKeys, c.Keys[coldEnd+1:end+1]...)
+			newValues = append(newValues, c.Values[coldEnd:end]...)
+			coldStart = end
+		} else {
+			coldRangeSum += rangeSum
+		}
+		coldEnd = end
+	}
+
+	for i := range c.Values {
+		if strategy.CrossBorder(c.Keys[i], c.Keys[i+1]) {
+			generateRange(i + 1)
+		}
+	}
+	generateRange(len(c.Values))
+	mergeColdRange()
+
 	return createChunk(newKeys, newValues)
 }
 
 // Divide uses binary search to find a suitable threshold, which can reduce the number of buckets of Axis to near the target.
-func (c *chunk) Divide(strategy Strategy, target int) chunk {
+func (c *chunk) Divide(strategy Strategy, target int, mode FocusMode) chunk {
 	if target >= len(c.Values) {
 		return *c
 	}
@@ -264,5 +326,5 @@ func (c *chunk) Divide(strategy Strategy, target int) chunk {
 	threshold := lowerThreshold
 	focusRows := c.GetFocusRows(threshold)
 	ratio := len(c.Values)/(target-focusRows) + 1
-	return c.Focus(strategy, threshold, ratio, target)
+	return c.Focus(strategy, threshold, ratio, target, mode)
 }

pkg/keyvisual/matrix/plane.go

@@ -75,7 +75,7 @@ func (plane *Plane) Pixel(strategy Strategy, target int, displayTags []string) M
 		chunks[i] = createChunk(axis.Keys, axis.ValuesList[0])
 	}
 	compactChunk, helper := compact(strategy, chunks)
-	baseKeys := compactChunk.Divide(strategy, target).Keys
+	baseKeys := compactChunk.Divide(strategy, target, NotMergeLogicalRange).Keys
 	matrix := CreateMatrix(strategy, plane.Times, baseKeys, valuesListLen)
 
 	var wg sync.WaitGroup

pkg/tidb/forwarder.go

@@ -16,7 +16,6 @@ package tidb
 import (
 	"context"
 	"crypto/tls"
-	"encoding/json"
 	"fmt"
 	"net"
 	"net/http"
@@ -25,27 +24,18 @@ import (
 	"github.com/cenkalti/backoff/v4"
 	"github.com/go-sql-driver/mysql"
 	"github.com/joomcode/errorx"
-	"github.com/pingcap/log"
 	"go.etcd.io/etcd/clientv3"
 	"go.uber.org/fx"
-	"go.uber.org/zap"
 
 	"github.com/pingcap-incubator/tidb-dashboard/pkg/config"
-	"github.com/pingcap-incubator/tidb-dashboard/pkg/pd"
+	"github.com/pingcap-incubator/tidb-dashboard/pkg/utils/topology"
 )
 
 var (
 	ErrPDAccessFailed = ErrNS.NewType("pd_access_failed")
 	ErrNoAliveTiDB    = ErrNS.NewType("no_alive_tidb")
 )
 
-type tidbServerInfo struct {
-	ID         string `json:"ddl_id"`
-	IP         string `json:"ip"`
-	Port       int    `json:"listening_port"`
-	StatusPort uint   `json:"status_port"`
-}
-
 type ForwarderConfig struct {
 	ClusterTLSConfig    *tls.Config
 	TiDBTLSConfig       *tls.Config
@@ -104,33 +94,6 @@ func (f *Forwarder) Start(ctx context.Context) error {
 	return nil
 }
 
-func (f *Forwarder) getServerInfo() ([]*tidbServerInfo, error) {
-	ctx, cancel := context.WithTimeout(f.lifecycleCtx, f.config.TiDBRetrieveTimeout)
-	resp, err := f.etcdClient.Get(ctx, pd.TiDBServerInformationPath, clientv3.WithPrefix())
-	cancel()
-
-	if err != nil {
-		log.Warn("Fail to get TiDB server info from PD", zap.Error(err))
-		return nil, ErrPDAccessFailed.WrapWithNoMessage(err)
-	}
-
-	allTiDB := make([]*tidbServerInfo, 0, len(resp.Kvs))
-	for _, kv := range resp.Kvs {
-		var info *tidbServerInfo
-		err = json.Unmarshal(kv.Value, &info)
-		if err != nil {
-			continue
-		}
-		allTiDB = append(allTiDB, info)
-	}
-	if len(allTiDB) == 0 {
-		log.Warn("No TiDB is alive now")
-		return nil, backoff.Permanent(ErrNoAliveTiDB.NewWithNoMessage())
-	}
-
-	return allTiDB, nil
-}
-
 func (f *Forwarder) createProxy() (*proxy, error) {
 	l, err := net.Listen("tcp", "127.0.0.1:0")
 	if err != nil {
@@ -146,10 +109,10 @@ func (f *Forwarder) pollingForTiDB() {
 	bo := backoff.WithContext(ebo, f.lifecycleCtx)
 
 	for {
-		var allTiDB []*tidbServerInfo
+		var allTiDB []topology.TiDBInfo
 		err := backoff.Retry(func() error {
 			var err error
-			allTiDB, err = f.getServerInfo()
+			allTiDB, err = topology.FetchTiDBTopology(bo.Context(), f.etcdClient)
 			return err
 		}, bo)
 		if err != nil {

pkg/tidb/proxy.go

@@ -135,7 +135,7 @@ func (p *proxy) serve(in net.Conn) {
 	in.Close()
 }
 
-// pick returns an active remote. If there
+// pick returns an active remote if there is any
 func (p *proxy) pick() *remote {
 	var picked *remote
 	if p.current == "" {
@@ -157,6 +157,8 @@ func (p *proxy) pick() *remote {
 		r, ok := p.remotes.Load(p.current)
 		if ok {
 			picked = r.(*remote)
+		} else {
+			p.current = ""
 		}
 	}
 	return picked