GOMAXPROCS, et al.

* revisit, add comments * with minor refactoring Signed-off-by: Alex Aizman <[email protected]>
NVIDIA · Jul 7, 2024 · e2a68fa · e2a68fa
1 parent 7b9faaf
commit e2a68fa
Show file tree

Hide file tree

Showing 4 changed files with 79 additions and 33 deletions.
diff --git a/ais/daemon.go b/ais/daemon.go
@@ -11,6 +11,7 @@ import (
 	"net/url"
 	"os"
 	"runtime"
+	"strconv"
 	"strings"
 
 	"github.com/NVIDIA/aistore/api/apc"
@@ -20,6 +21,7 @@ import (
 	"github.com/NVIDIA/aistore/cmn/debug"
 	"github.com/NVIDIA/aistore/cmn/k8s"
 	"github.com/NVIDIA/aistore/cmn/nlog"
+	"github.com/NVIDIA/aistore/core/meta"
 	"github.com/NVIDIA/aistore/fs"
 	"github.com/NVIDIA/aistore/hk"
 	"github.com/NVIDIA/aistore/space"
@@ -186,15 +188,8 @@ func initDaemon(version, buildTime string) cos.Runner {
 	}
 
 	daemon.version, daemon.buildTime = version, buildTime
-	loghdr := fmt.Sprintf("Version %s, build time %s, debug %t", version, buildTime, debug.ON())
-	cpus := sys.NumCPU()
-	if containerized := sys.Containerized(); containerized {
-		loghdr += fmt.Sprintf(", CPUs(%d, runtime=%d), containerized", cpus, runtime.NumCPU())
-	} else {
-		loghdr += fmt.Sprintf(", CPUs(%d, runtime=%d)", cpus, runtime.NumCPU())
-	}
-	nlog.Infoln(loghdr) // redundant (see below), prior to start/init
-	sys.SetMaxProcs()
+	loghdr := _loghdr()
+	sys.GoEnvMaxprocs()
 
 	daemon.rg = &rungroup{rs: make(map[string]cos.Runner, 6)}
 	hk.Init()
@@ -241,7 +236,7 @@ func initDaemon(version, buildTime string) cos.Runner {
 		xs.Xreg(true /* x-ele only */)
 		p := newProxy(co)
 		p.init(config)
-		title := "Node " + p.si.Name() + ", " + loghdr + "\n"
+		title := _loghdr2(p.si, loghdr)
 		nlog.Infoln(title)
 
 		// aux plumbing
@@ -256,7 +251,7 @@ func initDaemon(version, buildTime string) cos.Runner {
 
 	t := newTarget(co)
 	t.init(config)
-	title := "Node " + t.si.Name() + ", " + loghdr + "\n"
+	title := _loghdr2(t.si, loghdr)
 	nlog.Infoln(title)
 
 	// aux plumbing
@@ -266,6 +261,42 @@ func initDaemon(version, buildTime string) cos.Runner {
 	return t
 }
 
+func _loghdr2(si *meta.Snode, loghdr string) string {
+	var sb strings.Builder
+	sb.WriteString("Node ")
+	sb.WriteString(si.Name())
+	sb.WriteString(", ")
+	sb.WriteString(loghdr)
+	sb.WriteString("\n")
+	return sb.String()
+}
+
+func _loghdr() (loghdr string) {
+	var sb strings.Builder
+	sb.WriteString("Version ")
+	sb.WriteString(daemon.version)
+	if debug.ON() {
+		sb.WriteString(", DEBUG build ")
+	} else {
+		sb.WriteString(", build ")
+	}
+	sb.WriteString(daemon.buildTime)
+
+	cpus := sys.NumCPU()
+	sb.WriteString(", CPUs(")
+	sb.WriteString(strconv.Itoa(cpus))
+	sb.WriteString(", runtime=")
+	sb.WriteString(strconv.Itoa(runtime.NumCPU()))
+	sb.WriteByte(')')
+
+	if sys.Containerized() {
+		sb.WriteString(", containerized")
+	}
+	loghdr = sb.String()
+	nlog.Infoln(loghdr) // redundant (see below), prior to start/init
+	return loghdr
+}
+
 func newProxy(co *configOwner) *proxy {
 	p := &proxy{}
 	p.owner.config = co

diff --git a/sys/cpu.go b/sys/cpu.go
@@ -1,22 +1,26 @@
 // Package sys provides methods to read system information
 /*
- * Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
  */
 package sys
 
 import (
+	"fmt"
 	"os"
 	"runtime"
 
 	"github.com/NVIDIA/aistore/cmn/nlog"
 )
 
-const maxProcsEnvVar = "GOMAXPROCS"
-
 type LoadAvg struct {
 	One, Five, Fifteen float64
 }
 
+// TODO -- FIXME:
+// - see cpu_linux.go comment on detecting containerization
+// - blog https://www.riverphillips.dev/blog/go-cfs
+// - available "maxprocs" open-source
+
 var (
 	contCPUs      int
 	containerized bool
@@ -28,24 +32,27 @@ func init() {
 		if c, err := containerNumCPU(); err == nil {
 			contCPUs = c
 		} else {
-			nlog.Errorln(err)
+			fmt.Fprintln(os.Stderr, err) // (cannot nlog yet)
 		}
 	}
 }
 
 func Containerized() bool { return containerized }
 func NumCPU() int         { return contCPUs }
 
-// SetMaxProcs sets GOMAXPROCS = NumCPU unless already overridden via Go environment
-func SetMaxProcs() {
-	if val, exists := os.LookupEnv(maxProcsEnvVar); exists {
-		nlog.Warningf("GOMAXPROCS is set via Go environment %q: %q", maxProcsEnvVar, val)
+func GoEnvMaxprocs() {
+	if val, exists := os.LookupEnv("GOMEMLIMIT"); exists {
+		nlog.Warningln("Go environment: GOMEMLIMIT =", val) // soft memory limit for the runtime (IEC units or raw bytes)
+	}
+	if val, exists := os.LookupEnv("GOMAXPROCS"); exists {
+		nlog.Warningln("Go environment: GOMAXPROCS =", val)
 		return
 	}
+
 	maxprocs := runtime.GOMAXPROCS(0)
-	ncpu := NumCPU()
+	ncpu := NumCPU() // TODO: (see comment at the top)
 	if maxprocs > ncpu {
-		nlog.Warningf("Reducing GOMAXPROCS (%d) to %d (num CPUs)", maxprocs, ncpu)
+		nlog.Warningf("Reducing GOMAXPROCS (prev = %d) to %d", maxprocs, ncpu)
 		runtime.GOMAXPROCS(ncpu)
 	}
 }
diff --git a/sys/cpu_linux.go b/sys/cpu_linux.go
@@ -1,6 +1,6 @@
 // Package sys provides methods to read system information
 /*
- * Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
  */
 package sys
 
@@ -15,11 +15,20 @@ import (
 	"github.com/NVIDIA/aistore/cmn/nlog"
 )
 
-// isContainerized returns true if the application is running
-// inside a container(docker/lxc/k8s)
+// NOTE (July 2024):
+// Reading /proc/1/cgroup cannot be relied on - does not always produce "docker", etc. strings that indicate "containerization."
+// One plausible, albeit still somewhat hacky approach could be detecting filesystem type of the root ("/"),
+// as in:
 //
-// How to detect being inside a container:
-// https://stackoverflow.com/questions/20010199/how-to-determine-if-a-process-runs-inside-lxc-docker
+// var mi = fs.Mountpath{Path: "/"}
+// if mi.resolveFS() == nil {
+// 	if mi.FsType == "overlay" {
+// 		containerized = true
+// 		...
+// 	}
+// }
+
+// TODO: either amend, OR introduce env var and remove auto-detection altogether
 func isContainerized() (yes bool) {
 	err := cos.ReadLines(rootProcess, func(line string) error {
 		if strings.Contains(line, "docker") || strings.Contains(line, "lxc") || strings.Contains(line, "kube") {

diff --git a/sys/sys_test.go b/sys/sys_test.go
@@ -39,14 +39,13 @@ func TestLoadAvg(t *testing.T) {
 		"All load average must be positive ones")
 }
 
-func TestLimitMaxProc(t *testing.T) {
-	prev := runtime.GOMAXPROCS(0)
-	defer runtime.GOMAXPROCS(prev)
+func TestMaxProcs(t *testing.T) {
+	newval := 4
+	prev := runtime.GOMAXPROCS(newval)
+	tassert.Errorf(t, runtime.GOMAXPROCS(0) == newval, "Failed to set GOMAXPROCS to %d", newval)
 
-	ncpu := sys.NumCPU()
-	sys.SetMaxProcs()
-	curr := runtime.GOMAXPROCS(0)
-	tassert.Errorf(t, ncpu == curr, "Failed to set GOMAXPROCS to %d, current value is %d", ncpu, curr)
+	runtime.GOMAXPROCS(prev)
+	tassert.Errorf(t, runtime.GOMAXPROCS(0) == prev, "Failed to restore GOMAXPROCS to %d", prev)
 }
 
 func TestMemoryStats(t *testing.T) {