fix(system-services): when the child process exits, roller should als…

…o exit explicitly (#1124)
dymensionxyz · Nov 22, 2024 · 72119fb · 72119fb
1 parent dc0c86e
commit 72119fb
Show file tree

Hide file tree

Showing 5 changed files with 162 additions and 20 deletions.
diff --git a/cmd/da-light-client/start/start.go b/cmd/da-light-client/start/start.go
@@ -1,8 +1,10 @@
 package start
 
 import (
+	"context"
 	"errors"
 	"fmt"
+	"os"
 	"path/filepath"
 
 	"github.com/pterm/pterm"
@@ -74,6 +76,7 @@ func Cmd() *cobra.Command {
 			}
 
 			fmt.Println(startDALCCmd.String())
+			done := make(chan error, 1)
 			if rollerData.KeyringBackend == consts.SupportedKeyringBackends.OS {
 				pswFileName, err := filesystem.GetOsKeyringPswFileName(consts.Executables.Celestia)
 				if err != nil {
@@ -93,11 +96,55 @@ func Cmd() *cobra.Command {
 					"Re-enter keyring passphrase": psw,
 				}
 
+				ctx, cancel := context.WithCancel(context.Background())
+				defer cancel()
 				// nolint: errcheck
-				go bash.ExecCmdFollow(startDALCCmd, pr)
+				go func() {
+					err := bash.ExecCmdFollow(
+						done,
+						ctx,
+						startDALCCmd,
+						pr, // No need for printOutput since we configured output above
+					)
+
+					done <- err
+				}()
+
+				select {
+				case err := <-done:
+					if err != nil {
+						pterm.Error.Println("da process returned an error: ", err)
+						os.Exit(1)
+					}
+				case <-ctx.Done():
+					pterm.Error.Println("context cancelled, terminating command")
+					return
+				}
 			} else {
-				// nolint: errcheck
-				go bash.ExecCmdFollow(startDALCCmd, nil)
+				ctx, cancel := context.WithCancel(cmd.Context())
+				defer cancel()
+
+				go func() {
+					err := bash.ExecCmdFollow(
+						done,
+						ctx,
+						startDALCCmd,
+						nil,
+					)
+
+					done <- err
+				}()
+
+				select {
+				case err := <-done:
+					if err != nil {
+						pterm.Error.Println("da process returned an error: ", err)
+						os.Exit(1)
+					}
+				case <-ctx.Done():
+					pterm.Error.Println("context cancelled, terminating command")
+					return
+				}
 			}
 			select {}
 		},

diff --git a/cmd/eibc/scale/scale.go b/cmd/eibc/scale/scale.go
@@ -1,6 +1,7 @@
 package scale
 
 import (
+	"github.com/pterm/pterm"
 	"github.com/spf13/cobra"
 
 	"github.com/dymensionxyz/roller/utils/bash"
@@ -23,8 +24,9 @@ a good number to start with is 30 (default when initializing the eibc client)
 
 			c := eibcutils.GetScaleCmd(count)
 
-			err := bash.ExecCmdFollow(c, nil)
+			_, err := bash.ExecCommandWithStdout(c)
 			if err != nil {
+				pterm.Error.Println("failed to scale the number of fulfillers: ", err)
 				return
 			}
 		},

diff --git a/cmd/eibc/start/start.go b/cmd/eibc/start/start.go
@@ -1,6 +1,7 @@
 package start
 
 import (
+	"context"
 	"os"
 	"path/filepath"
 
@@ -37,10 +38,31 @@ func Cmd() *cobra.Command {
 				return
 			}
 
+			done := make(chan error, 1)
 			c := eibcutils.GetStartCmd()
-			err = bash.ExecCmdFollow(c, nil)
-			if err != nil {
-				pterm.Error.Println("failed to start the eibc client:", err)
+
+			ctx, cancel := context.WithCancel(cmd.Context())
+			defer cancel()
+
+			go func() {
+				err := bash.ExecCmdFollow(
+					done,
+					ctx,
+					c,
+					nil, // No need for printOutput since we configured output above
+				)
+
+				done <- err
+			}()
+
+			select {
+			case err := <-done:
+				if err != nil {
+					pterm.Error.Println("eibc client process returned an error: ", err)
+					return
+				}
+			case <-ctx.Done():
+				pterm.Error.Println("context cancelled, terminating command")
 				return
 			}
 		},

diff --git a/cmd/rollapp/start/start.go b/cmd/rollapp/start/start.go
@@ -1,6 +1,7 @@
 package start
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"os/exec"
@@ -95,6 +96,7 @@ Consider using 'services' if you want to run a 'systemd'(unix) or 'launchd'(mac)
 				go healthagent.Start(home, rollerLogger)
 			}
 
+			done := make(chan error, 1)
 			// nolint: errcheck
 			if rollappConfig.KeyringBackend == consts.SupportedKeyringBackends.OS {
 				pswFileName, err := filesystem.GetOsKeyringPswFileName(
@@ -117,12 +119,44 @@ Consider using 'services' if you want to run a 'systemd'(unix) or 'launchd'(mac)
 					"Re-enter keyring passphrase": psw,
 				}
 
-				go bash.ExecCmdFollow(
-					startRollappCmd,
-					pr,
-				)
+				ctx, cancel := context.WithCancel(cmd.Context())
+				defer cancel()
+				go func() {
+					err := bash.ExecCmdFollow(
+						done,
+						ctx,
+						startRollappCmd,
+						pr,
+					)
+
+					done <- err
+				}()
 			} else {
-				go bash.ExecCmdFollow(startRollappCmd, nil)
+				ctx, cancel := context.WithCancel(cmd.Context())
+				defer cancel()
+
+				go func() {
+					err := bash.ExecCmdFollow(
+						done,
+						ctx,
+						startRollappCmd,
+						nil, // No need for printOutput since we configured output above
+					)
+
+					done <- err
+				}()
+
+				select {
+				case err := <-done:
+					if err != nil {
+						pterm.Error.Println("rollapp's process returned an error: ", err)
+						os.Exit(1)
+					}
+				case <-ctx.Done():
+					pterm.Error.Println("context cancelled, terminating command")
+					return
+				}
+
 			}
 
 			select {}
@@ -245,7 +279,7 @@ func createPidFile(path string, cmd *exec.Cmd) error {
 		fmt.Println("Error creating file:", err)
 		return err
 	}
-	// nolint errcheck
+	// nolint: errcheck
 	defer file.Close()
 
 	pid := cmd.Process.Pid

diff --git a/utils/bash/bash_commands.go b/utils/bash/bash_commands.go
@@ -9,8 +9,10 @@ import (
 	"io"
 	"os"
 	"os/exec"
+	"os/signal"
 	"strings"
 	"sync"
+	"syscall"
 	"time"
 
 	"github.com/pterm/pterm"
@@ -64,6 +66,7 @@ func RunCmdAsync(
 	for _, option := range options {
 		option(cmd)
 	}
+
 	if parseError == nil {
 		parseError = func(errMsg string) string {
 			return errMsg
@@ -75,6 +78,7 @@ func RunCmdAsync(
 	if cmd.Stderr != nil {
 		mw = io.MultiWriter(&stderr, cmd.Stderr)
 	}
+
 	cmd.Stderr = mw
 	err := cmd.Start()
 	if err != nil {
@@ -138,7 +142,15 @@ func ExecCmd(cmd *exec.Cmd, options ...CommandOption) error {
 	return nil
 }
 
-func ExecCmdFollow(cmd *exec.Cmd, promptResponses map[string]string) error {
+func ExecCmdFollow(
+	doneChan chan<- error,
+	ctx context.Context,
+	cmd *exec.Cmd,
+	promptResponses map[string]string,
+) error {
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
+
 	stdout, err := cmd.StdoutPipe()
 	if err != nil {
 		return err
@@ -157,6 +169,22 @@ func ExecCmdFollow(cmd *exec.Cmd, promptResponses map[string]string) error {
 		return err
 	}
 
+	// handle signals
+	go func() {
+		for {
+			select {
+			case sig := <-sigChan:
+				pterm.Info.Println("received signal: ", sig)
+				if cmd.Process != nil {
+					_ = cmd.Process.Signal(sig)
+					doneChan <- fmt.Errorf("received signal: %s", sig)
+				}
+			case <-ctx.Done():
+				_ = cmd.Process.Signal(syscall.SIGTERM)
+			}
+		}
+	}()
+
 	// Use a WaitGroup to wait for both stdout and stderr to be processed
 	var wg sync.WaitGroup
 	wg.Add(2)
@@ -183,23 +211,32 @@ func ExecCmdFollow(cmd *exec.Cmd, promptResponses map[string]string) error {
 		for scanner.Scan() {
 			fmt.Println(scanner.Text())
 		}
+
 		if err := scanner.Err(); err != nil {
 			errChan <- err
 		}
 	}()
 
-	// Wait for both stdout and stderr goroutines to finish
 	go func() {
-		wg.Wait()
-		close(errChan)
+		<-ctx.Done()
+		if cmd.Process != nil {
+			pterm.Info.Println("killing process: ", cmd.Process.Pid)
+			err = cmd.Process.Kill()
+			if err != nil {
+				pterm.Error.Println("failed to kill process: ", err)
+			}
+		}
 	}()
 
-	// Wait for the command to finish
-	if err := cmd.Wait(); err != nil {
+	err = cmd.Wait()
+	if err != nil {
 		return err
 	}
 
-	// Check if there were any errors in the goroutines
+	wg.Wait()
+	close(errChan)
+
+	// Check for any scanning errors
 	for err := range errChan {
 		if err != nil {
 			return err