diff --git a/ais/daemon.go b/ais/daemon.go index 18508fba4b9..23cbe1b0a6f 100644 --- a/ais/daemon.go +++ b/ais/daemon.go @@ -11,6 +11,7 @@ import ( "net/url" "os" "runtime" + "strconv" "strings" "github.com/NVIDIA/aistore/api/apc" @@ -20,6 +21,7 @@ import ( "github.com/NVIDIA/aistore/cmn/debug" "github.com/NVIDIA/aistore/cmn/k8s" "github.com/NVIDIA/aistore/cmn/nlog" + "github.com/NVIDIA/aistore/core/meta" "github.com/NVIDIA/aistore/fs" "github.com/NVIDIA/aistore/hk" "github.com/NVIDIA/aistore/space" @@ -186,15 +188,8 @@ func initDaemon(version, buildTime string) cos.Runner { } daemon.version, daemon.buildTime = version, buildTime - loghdr := fmt.Sprintf("Version %s, build time %s, debug %t", version, buildTime, debug.ON()) - cpus := sys.NumCPU() - if containerized := sys.Containerized(); containerized { - loghdr += fmt.Sprintf(", CPUs(%d, runtime=%d), containerized", cpus, runtime.NumCPU()) - } else { - loghdr += fmt.Sprintf(", CPUs(%d, runtime=%d)", cpus, runtime.NumCPU()) - } - nlog.Infoln(loghdr) // redundant (see below), prior to start/init - sys.SetMaxProcs() + loghdr := _loghdr() + sys.GoEnvMaxprocs() daemon.rg = &rungroup{rs: make(map[string]cos.Runner, 6)} hk.Init() @@ -241,7 +236,7 @@ func initDaemon(version, buildTime string) cos.Runner { xs.Xreg(true /* x-ele only */) p := newProxy(co) p.init(config) - title := "Node " + p.si.Name() + ", " + loghdr + "\n" + title := _loghdr2(p.si, loghdr) nlog.Infoln(title) // aux plumbing @@ -256,7 +251,7 @@ func initDaemon(version, buildTime string) cos.Runner { t := newTarget(co) t.init(config) - title := "Node " + t.si.Name() + ", " + loghdr + "\n" + title := _loghdr2(t.si, loghdr) nlog.Infoln(title) // aux plumbing @@ -266,6 +261,42 @@ func initDaemon(version, buildTime string) cos.Runner { return t } +func _loghdr2(si *meta.Snode, loghdr string) string { + var sb strings.Builder + sb.WriteString("Node ") + sb.WriteString(si.Name()) + sb.WriteString(", ") + sb.WriteString(loghdr) + sb.WriteString("\n") + return sb.String() +} + +func _loghdr() (loghdr string) { + var sb strings.Builder + sb.WriteString("Version ") + sb.WriteString(daemon.version) + if debug.ON() { + sb.WriteString(", DEBUG build ") + } else { + sb.WriteString(", build ") + } + sb.WriteString(daemon.buildTime) + + cpus := sys.NumCPU() + sb.WriteString(", CPUs(") + sb.WriteString(strconv.Itoa(cpus)) + sb.WriteString(", runtime=") + sb.WriteString(strconv.Itoa(runtime.NumCPU())) + sb.WriteByte(')') + + if sys.Containerized() { + sb.WriteString(", containerized") + } + loghdr = sb.String() + nlog.Infoln(loghdr) // redundant (see below), prior to start/init + return loghdr +} + func newProxy(co *configOwner) *proxy { p := &proxy{} p.owner.config = co diff --git a/sys/cpu.go b/sys/cpu.go index 863f542fc2c..1009ed8cf84 100644 --- a/sys/cpu.go +++ b/sys/cpu.go @@ -1,22 +1,26 @@ // Package sys provides methods to read system information /* - * Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved. */ package sys import ( + "fmt" "os" "runtime" "github.com/NVIDIA/aistore/cmn/nlog" ) -const maxProcsEnvVar = "GOMAXPROCS" - type LoadAvg struct { One, Five, Fifteen float64 } +// TODO -- FIXME: +// - see cpu_linux.go comment on detecting containerization +// - blog https://www.riverphillips.dev/blog/go-cfs +// - available "maxprocs" open-source + var ( contCPUs int containerized bool @@ -28,7 +32,7 @@ func init() { if c, err := containerNumCPU(); err == nil { contCPUs = c } else { - nlog.Errorln(err) + fmt.Fprintln(os.Stderr, err) // (cannot nlog yet) } } } @@ -36,16 +40,19 @@ func init() { func Containerized() bool { return containerized } func NumCPU() int { return contCPUs } -// SetMaxProcs sets GOMAXPROCS = NumCPU unless already overridden via Go environment -func SetMaxProcs() { - if val, exists := os.LookupEnv(maxProcsEnvVar); exists { - nlog.Warningf("GOMAXPROCS is set via Go environment %q: %q", maxProcsEnvVar, val) +func GoEnvMaxprocs() { + if val, exists := os.LookupEnv("GOMEMLIMIT"); exists { + nlog.Warningln("Go environment: GOMEMLIMIT =", val) // soft memory limit for the runtime (IEC units or raw bytes) + } + if val, exists := os.LookupEnv("GOMAXPROCS"); exists { + nlog.Warningln("Go environment: GOMAXPROCS =", val) return } + maxprocs := runtime.GOMAXPROCS(0) - ncpu := NumCPU() + ncpu := NumCPU() // TODO: (see comment at the top) if maxprocs > ncpu { - nlog.Warningf("Reducing GOMAXPROCS (%d) to %d (num CPUs)", maxprocs, ncpu) + nlog.Warningf("Reducing GOMAXPROCS (prev = %d) to %d", maxprocs, ncpu) runtime.GOMAXPROCS(ncpu) } } diff --git a/sys/cpu_linux.go b/sys/cpu_linux.go index 387024d0fc2..efa3cff5aea 100644 --- a/sys/cpu_linux.go +++ b/sys/cpu_linux.go @@ -1,6 +1,6 @@ // Package sys provides methods to read system information /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved. */ package sys @@ -15,11 +15,20 @@ import ( "github.com/NVIDIA/aistore/cmn/nlog" ) -// isContainerized returns true if the application is running -// inside a container(docker/lxc/k8s) +// NOTE (July 2024): +// Reading /proc/1/cgroup cannot be relied on - does not always produce "docker", etc. strings that indicate "containerization." +// One plausible, albeit still somewhat hacky approach could be detecting filesystem type of the root ("/"), +// as in: // -// How to detect being inside a container: -// https://stackoverflow.com/questions/20010199/how-to-determine-if-a-process-runs-inside-lxc-docker +// var mi = fs.Mountpath{Path: "/"} +// if mi.resolveFS() == nil { +// if mi.FsType == "overlay" { +// containerized = true +// ... +// } +// } + +// TODO: either amend, OR introduce env var and remove auto-detection altogether func isContainerized() (yes bool) { err := cos.ReadLines(rootProcess, func(line string) error { if strings.Contains(line, "docker") || strings.Contains(line, "lxc") || strings.Contains(line, "kube") { diff --git a/sys/sys_test.go b/sys/sys_test.go index 3395f05dd3d..d6fe0a6e718 100644 --- a/sys/sys_test.go +++ b/sys/sys_test.go @@ -39,14 +39,13 @@ func TestLoadAvg(t *testing.T) { "All load average must be positive ones") } -func TestLimitMaxProc(t *testing.T) { - prev := runtime.GOMAXPROCS(0) - defer runtime.GOMAXPROCS(prev) +func TestMaxProcs(t *testing.T) { + newval := 4 + prev := runtime.GOMAXPROCS(newval) + tassert.Errorf(t, runtime.GOMAXPROCS(0) == newval, "Failed to set GOMAXPROCS to %d", newval) - ncpu := sys.NumCPU() - sys.SetMaxProcs() - curr := runtime.GOMAXPROCS(0) - tassert.Errorf(t, ncpu == curr, "Failed to set GOMAXPROCS to %d, current value is %d", ncpu, curr) + runtime.GOMAXPROCS(prev) + tassert.Errorf(t, runtime.GOMAXPROCS(0) == prev, "Failed to restore GOMAXPROCS to %d", prev) } func TestMemoryStats(t *testing.T) {