feat: using a new progress bar SDK to sealer scp (#2136)

Signed-off-by: Lancelot <[email protected]>

go.mod

@@ -35,15 +35,16 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/pkg/sftp v1.13.0
 	github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5
+	github.com/schollz/progressbar/v3 v3.13.1
 	github.com/sirupsen/logrus v1.9.0
 	github.com/spf13/cobra v1.5.0
 	github.com/spf13/viper v1.10.0
 	github.com/stretchr/testify v1.8.0
 	github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5
 	golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e
 	golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f
-	golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8
-	golang.org/x/term v0.0.0-20220526004731-065cf7ba2467
+	golang.org/x/sys v0.6.0
+	golang.org/x/term v0.6.0
 	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
 	helm.sh/helm/v3 v3.9.4
@@ -126,11 +127,12 @@ require (
 	github.com/magiconair/properties v1.8.5 // indirect
 	github.com/mailru/easyjson v0.7.6 // indirect
 	github.com/manifoldco/promptui v0.9.0 // indirect
-	github.com/mattn/go-runewidth v0.0.13 // indirect
+	github.com/mattn/go-runewidth v0.0.14 // indirect
 	github.com/mattn/go-shellwords v1.0.12 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/miekg/pkcs11 v1.1.1 // indirect
 	github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible // indirect
+	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect

utils/progressbar/progressbar.go

@@ -0,0 +1,75 @@
+// Copyright © 2023 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package progressbar
+
+import (
+	"fmt"
+
+	"github.com/schollz/progressbar/v3"
+	"github.com/sirupsen/logrus"
+)
+
+type EasyProgressUtil struct {
+	progressbar.ProgressBar
+}
+
+var (
+	width                  = 50
+	optionEnableColorCodes = progressbar.OptionEnableColorCodes(true)
+	optionSetWidth         = progressbar.OptionSetWidth(width)
+	optionSetTheme         = progressbar.OptionSetTheme(progressbar.Theme{
+		Saucer:        "=",
+		SaucerHead:    ">",
+		SaucerPadding: " ",
+		BarStart:      "[",
+		BarEnd:        "]",
+	})
+)
+
+func NewEasyProgressUtil(total int, describe string) *EasyProgressUtil {
+	return &EasyProgressUtil{
+		*progressbar.NewOptions(total,
+			optionEnableColorCodes,
+			optionSetWidth,
+			optionSetTheme,
+			progressbar.OptionSetDescription(describe),
+			// after finish, print a new line
+			progressbar.OptionOnCompletion(func() {
+				fmt.Println()
+			}),
+		),
+	}
+}
+
+// increment add 1 to progress bar
+func (epu *EasyProgressUtil) Increment() {
+	if err := epu.Add(1); err != nil {
+		logrus.Errorf("failed to increment progress bar, err: %s", err)
+	}
+}
+
+// fail print error message
+func (epu *EasyProgressUtil) Fail(err error) {
+	if err != nil {
+		epu.Describe(err.Error())
+	}
+}
+
+// setTotal set total num of progress bar
+func (epu *EasyProgressUtil) SetTotal(num int) {
+	if num > epu.GetMax() {
+		epu.ChangeMax(num)
+	}
+}

utils/ssh/scp.go

@@ -24,66 +24,39 @@ import (
 	"strings"
 	"sync"
 
-	dockerioutils "github.com/docker/docker/pkg/ioutils"
-	"github.com/docker/docker/pkg/progress"
 	"github.com/pkg/sftp"
-	"github.com/sirupsen/logrus"
-
 	utilsnet "github.com/sealerio/sealer/utils/net"
 	osi "github.com/sealerio/sealer/utils/os"
+	progressbar "github.com/sealerio/sealer/utils/progressbar"
+	"github.com/sirupsen/logrus"
 )
 
 const (
 	Md5sumCmd = "md5sum %s | cut -d\" \" -f1"
 )
 
 var (
-	displayInitOnce sync.Once
-	reader          *io.PipeReader
-	writer          *io.PipeWriter
-	writeFlusher    *dockerioutils.WriteFlusher
-	progressChanOut progress.Output
-	epuMap          = &epuRWMap{epu: map[string]*easyProgressUtil{}}
+	epuMap = &epuRWMap{epu: map[string]*progressbar.EasyProgressUtil{}}
 )
 
-type easyProgressUtil struct {
-	output         progress.Output
-	copyID         string
-	completeNumber int
-	total          int
-}
-
 type epuRWMap struct {
 	sync.RWMutex
-	epu map[string]*easyProgressUtil
+	epu map[string]*progressbar.EasyProgressUtil
 }
 
-func (m *epuRWMap) Get(k string) (*easyProgressUtil, bool) {
+func (m *epuRWMap) Get(k string) (*progressbar.EasyProgressUtil, bool) {
 	m.RLock()
 	defer m.RUnlock()
 	v, existed := m.epu[k]
 	return v, existed
 }
 
-func (m *epuRWMap) Set(k string, v *easyProgressUtil) {
+func (m *epuRWMap) Set(k string, v *progressbar.EasyProgressUtil) {
 	m.Lock()
 	defer m.Unlock()
 	m.epu[k] = v
 }
 
-func (epu *easyProgressUtil) increment() {
-	epu.completeNumber = epu.completeNumber + 1
-	progress.Update(epu.output, epu.copyID, fmt.Sprintf("%d/%d", epu.completeNumber, epu.total))
-}
-
-func (epu *easyProgressUtil) fail(err error) {
-	progress.Update(epu.output, epu.copyID, fmt.Sprintf("failed, err: %s", err))
-}
-
-func (epu *easyProgressUtil) startMessage() {
-	progress.Update(epu.output, epu.copyID, fmt.Sprintf("%d/%d", epu.completeNumber, epu.total))
-}
-
 // CopyR scp remote file to local
 func (s *SSH) CopyR(host net.IP, localFilePath, remoteFilePath string) error {
 	if utilsnet.IsLocalIP(host, s.LocalAddress) {
@@ -130,7 +103,6 @@ func (s *SSH) CopyR(host net.IP, localFilePath, remoteFilePath string) error {
 
 // Copy file or dir to remotePath, add md5 validate
 func (s *SSH) Copy(host net.IP, localPath, remotePath string) error {
-	go displayInitOnce.Do(displayInit)
 	if utilsnet.IsLocalIP(host, s.LocalAddress) {
 		if localPath == remotePath {
 			return nil
@@ -168,32 +140,20 @@ func (s *SSH) Copy(host net.IP, localPath, remotePath string) error {
 
 	epu, ok := epuMap.Get(host.String())
 	if !ok {
-		if progressChanOut == nil {
-			logrus.Warn("call DisplayInit first")
-		}
-
-		epu = &easyProgressUtil{
-			output:         progressChanOut,
-			copyID:         "copying files to " + host.String(),
-			completeNumber: 0,
-			total:          number,
-		}
-
+		epu = progressbar.NewEasyProgressUtil(number, fmt.Sprintf("[copying files to %s]", host))
 		epuMap.Set(host.String(), epu)
 	} else {
-		epu.total += number
+		epu.SetTotal(epu.GetMax() + number)
 	}
 
-	epu.startMessage()
-
 	if f.IsDir() {
 		s.copyLocalDirToRemote(host, sftpClient, localPath, remotePath, epu)
 	} else {
 		err = s.copyLocalFileToRemote(host, sftpClient, localPath, remotePath)
 		if err != nil {
-			epu.fail(err)
+			epu.Fail(err)
 		}
-		epu.increment()
+		epu.Increment()
 	}
 	return nil
 }
@@ -207,7 +167,7 @@ func (s *SSH) remoteMd5Sum(host net.IP, remoteFilePath string) string {
 	return strings.ReplaceAll(remoteMD5, "\r", "")
 }
 
-func (s *SSH) copyLocalDirToRemote(host net.IP, sftpClient *sftp.Client, localPath, remotePath string, epu *easyProgressUtil) {
+func (s *SSH) copyLocalDirToRemote(host net.IP, sftpClient *sftp.Client, localPath, remotePath string, epu *progressbar.EasyProgressUtil) {
 	localFiles, err := os.ReadDir(localPath)
 	if err != nil {
 		logrus.Errorf("failed to read local path dir(%s) on host(%s): %s", localPath, host, err)
@@ -230,11 +190,11 @@ func (s *SSH) copyLocalDirToRemote(host net.IP, sftpClient *sftp.Client, localPa
 			err := s.copyLocalFileToRemote(host, sftpClient, lfp, rfp)
 			if err != nil {
 				errMsg := fmt.Sprintf("failed to copy local file(%s) to remote(%s) on host(%s): %v", lfp, rfp, host, err)
-				epu.fail(err)
+				epu.Fail(err)
 				logrus.Error(errMsg)
 				return
 			}
-			epu.increment()
+			epu.Increment()
 		}
 	}
 }

utils/ssh/utils.go

@@ -24,31 +24,26 @@ import (
 	"sync"
 	"time"
 
-	dockerstreams "github.com/docker/cli/cli/streams"
-	dockerioutils "github.com/docker/docker/pkg/ioutils"
-	dockerjsonmessage "github.com/docker/docker/pkg/jsonmessage"
-	"github.com/docker/docker/pkg/streamformatter"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sync/errgroup"
 
-	"github.com/sealerio/sealer/common"
 	"github.com/sealerio/sealer/utils/hash"
 )
 
-func displayInit() {
-	reader, writer = io.Pipe()
-	writeFlusher = dockerioutils.NewWriteFlusher(writer)
-	defer func() {
-		_ = reader.Close()
-		_ = writer.Close()
-		_ = writeFlusher.Close()
-	}()
-	progressChanOut = streamformatter.NewJSONProgressOutput(writeFlusher, false)
-	err := dockerjsonmessage.DisplayJSONMessagesToStream(reader, dockerstreams.NewOut(common.StdOut), nil)
-	if err != nil && err != io.ErrClosedPipe {
-		logrus.Warnf("error occurs in display progressing, err: %s", err)
-	}
-}
+//func displayInit() {
+//	reader, writer = io.Pipe()
+//	writeFlusher = dockerioutils.NewWriteFlusher(writer)
+//	defer func() {
+//		_ = reader.Close()
+//		_ = writer.Close()
+//		_ = writeFlusher.Close()
+//	}()
+//	progressChanOut = streamformatter.NewJSONProgressOutput(writeFlusher, false)
+//	err := dockerjsonmessage.DisplayJSONMessagesToStream(reader, dockerstreams.NewOut(common.StdOut), nil)
+//	if err != nil && err != io.ErrClosedPipe {
+//		logrus.Warnf("error occurs in display progressing, err: %s", err)
+//	}
+//}
 
 func localMd5Sum(localPath string) string {
 	md5, err := hash.FileMD5(localPath)