Add msgo to buildimages (#715)

Co-authored-by: Pierre Gimalac <[email protected]>

go.env

@@ -3,3 +3,7 @@ GO_SHA256_LINUX_AMD64=5f467d29fc67c7ae6468cb6ad5b047a274bae8180cac5e0b7ddbfeba3e
 GO_SHA256_LINUX_ARM64=5c616b32dab04bb8c4c8700478381daea0174dc70083e4026321163879278a4a
 GO_SHA256_LINUX_ARMV6L=5191e87a51a85d88edddc028ab30dfbfa2d7c37cf35d536655e7a063bfb2c9d2
 GO_SHA256_WINDOWS_AMD64=9eca39a677c6d055ed947087c63e430b2c6d5dd0dd84636cb171fa2717451ee1
+MSGO_SHA256_LINUX_AMD64=f3f812fe9ce2e6f7ba14a38ad29b1e1907153bfb5589967086cb3ce367c13565
+MSGO_SHA256_LINUX_ARM64=9d77b18a2b1413ce41d925465d8132a982368a9d937252880d298cdb49978b3a
+MSGO_SHA256_LINUX_ARMV6L=047c824a677a35e0e9ae57eb8bc9a76f0be4a507e0d465f2167bf96a1a7a104b
+MSGO_SHA256_WINDOWS_AMD64=fe045e6b19f1c513269200fea25d9ecc70c25276ce5701229fe58db2d06de6f2

linux-glibc-2.17-x64/Dockerfile

@@ -2,6 +2,7 @@ FROM ubuntu:24.04
 
 ARG GO_VERSION
 ARG GO_SHA256_LINUX_AMD64
+ARG MSGO_SHA256_LINUX_AMD64
 ARG CTNG_VERSION=1.26.0
 ARG IBM_MQ_VERSION=9.2.4.0
 ARG IBM_MQ_SHA256="d0d583eba72daf20b3762976f8831c2e23150ace90509520e12f8cda5b5bdb49"
@@ -93,12 +94,8 @@ RUN /bin/bash -l -c "gem install bundler --version $BUNDLER_VERSION --no-documen
 RUN echo 'source /usr/local/rvm/scripts/rvm' >> /root/.bashrc
 
 # Go
-RUN curl -sL -o /tmp/golang.tar.gz https://go.dev/dl/go$GO_VERSION.linux-amd64.tar.gz \
-  && echo "$GO_SHA256_LINUX_AMD64  /tmp/golang.tar.gz" | sha256sum --check \
-  && tar -C /usr/local -xzf /tmp/golang.tar.gz \
-  && rm -f /tmp/golang.tar.gz
-ENV PATH="/usr/local/go/bin:${PATH}"
-ENV PATH="${GOPATH}/bin:${PATH}"
+COPY setup_go.sh /
+RUN ./setup_go.sh
 
 # Rust is needed to compile the SDS library
 RUN curl -sSL -o rustup-init https://static.rust-lang.org/rustup/archive/${RUSTUP_VERSION}/x86_64-unknown-linux-gnu/rustup-init \

linux-glibc-2.23-arm64/Dockerfile

@@ -2,6 +2,7 @@ FROM ubuntu:24.04
 
 ARG GO_VERSION
 ARG GO_SHA256_LINUX_ARM64
+ARG MSGO_SHA256_LINUX_ARM64
 ARG CTNG_VERSION=1.26.0
 ARG RUST_VERSION=1.76.0
 ARG RUSTC_SHA256="673e336c81c65e6b16dcdede33f4cc9ed0f08bde1dbe7a935f113605292dc800"
@@ -83,11 +84,8 @@ RUN /bin/bash -l -c "gem install bundler --version $BUNDLER_VERSION --no-documen
 RUN echo 'source /usr/local/rvm/scripts/rvm' >> /root/.bashrc
 
 # Go
-RUN curl -sL -o /tmp/golang.tar.gz https://go.dev/dl/go$GO_VERSION.linux-arm64.tar.gz \
-  && echo "$GO_SHA256_LINUX_ARM64  /tmp/golang.tar.gz" | sha256sum --check \
-  && tar -C /usr/local -xzf /tmp/golang.tar.gz \
-  && rm -f /tmp/golang.tar.gz
-ENV PATH="/usr/local/go/bin:${PATH}"
+COPY setup_go.sh /
+RUN ./setup_go.sh
 ENV PATH="${GOPATH}/bin:${PATH}"
 
 # Rust is needed to compile the SDS library

setup_go.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+set -eo pipefail
+
+case $DD_TARGET_ARCH in
+"x64")
+    GO_SHA256="${GO_SHA256_LINUX_AMD64}"
+    MSGO_SHA256="${MSGO_SHA256_LINUX_AMD64}"
+    GOARCH="amd64"
+    ;;
+"aarch64")
+    GO_SHA256="${GO_SHA256_LINUX_ARM64}"
+    MSGO_SHA256="${MSGO_SHA256_LINUX_ARM64}"
+    GOARCH="arm64"
+    ;;
+"armhf")
+    GO_SHA256="${GO_SHA256_LINUX_ARMV6L}"
+    MSGO_SHA256="${MSGO_SHA256_LINUX_ARMV6L}"
+    GOARCH="armv6l"
+    ;;
+*)
+    echo "Unknown or unsupported architecture ${DD_TARGET_ARCH}"
+    exit -1
+esac
+
+echo "Installing upstream Go"
+curl -sL -o /tmp/golang.tar.gz https://go.dev/dl/go${GO_VERSION}.linux-${GOARCH}.tar.gz
+echo "$GO_SHA256  /tmp/golang.tar.gz" | sha256sum --check
+tar -C /usr/local -xzf /tmp/golang.tar.gz && rm -f /tmp/golang.tar.gz
+
+echo "Installing Microsoft Go"
+curl -SL -o /tmp/golang.tar.gz https://aka.ms/golang/release/latest/go${GO_VERSION}-1.linux-${GOARCH}.tar.gz
+echo "$MSGO_SHA256  /tmp/golang.tar.gz" | sha256sum --check
+mkdir /usr/local/msgo && tar --strip-components=1 -C /usr/local/msgo/ -xzf /tmp/golang.tar.gz && rm -f /tmp/golang.tar.gz;
+
+cat << EOF >> /root/.bashrc
+if [ "\$DD_GO_TOOLCHAIN" = "msgo" ]; then
+    export PATH="/usr/local/msgo/bin:\$PATH"
+    export GOROOT="/usr/local/msgo"
+else
+   export PATH="/usr/local/go/bin:\$PATH"
+   export GOROOT="/usr/local/go"
+fi
+EOF

tasks/update_go.py

@@ -29,30 +29,42 @@ def _get_archive_extension(os: str) -> str:
     return "tar.gz"
 
 
-def _get_expected_sha256(version: str) -> List[Tuple[Platform, str]]:
+def _get_expected_sha256(version: str, base_url: str) -> List[Tuple[Platform, str]]:
     """returns a map from platform to sha of the archive"""
 
     shas: List[Tuple[Platform, str]] = []
     for os, arch in PLATFORMS:
         ext = _get_archive_extension(os)
-        url = f"https://storage.googleapis.com/golang/go{version}.{os}-{arch}.{ext}.sha256"
+        url = f"{base_url}/go{version}.{os}-{arch}.{ext}.sha256"
         res = requests.get(url)
         res.raise_for_status()
 
-        sha = res.text.strip()
+        # Handle both format "<sha256>" and "<sha256>..<filename>"
+        sha_elts = res.text.strip().split("  ")
+        if sha_elts == [] or len(sha_elts) > 2:
+            raise exceptions.Exit(
+                f"The SHA256 of Go on {os}/{arch} has an unexpected format: '{res.text}'"
+            )
+
+        sha = sha_elts[0]
         if len(sha) != 64:
             raise exceptions.Exit(
                 f"The SHA256 of Go on {os}/{arch} has an unexpected format: '{sha}'"
             )
         shas.append(((os, arch), sha))
     return shas
 
+def _get_go_upstream_sha256(version):
+    return _get_expected_sha256(version, "https://storage.googleapis.com/golang")
 
-def _check_archive(version: str, shas: List[Tuple[Platform, str]]):
+def _get_msgo_sha256(version):
+    return _get_expected_sha256(f"{version}-1", "https://aka.ms/golang/release/latest")
+
+def _check_archive(version: str, shas: List[Tuple[Platform, str]], base_url: str):
     """checks that the archive sha is the same as the given one"""
     for (os, arch), expected_sha in shas:
         ext = _get_archive_extension(os)
-        url = f"https://go.dev/dl/go{version}.{os}-{arch}.{ext}"
+        url = f"{base_url}/go{version}.{os}-{arch}.{ext}"
         print(f"[check-archive] Fetching archive at {url}", file=sys.stderr)
         # Using `curl` through `ctx.run` takes way too much time due to the archive being huge
         # use `requests` as a workaround
@@ -63,6 +75,11 @@ def _check_archive(version: str, shas: List[Tuple[Platform, str]]):
                 f"The SHA256 of Go on {os}/{arch} should be {expected_sha}, but got {sha}"
             )
 
+def _display_shas(shas: List[Tuple[Platform, str]], toolchain: str):
+    print(f"--- {toolchain} ---")
+    for (os, arch), sha in shas:
+        platform = f"[{os}/{arch}]"
+        print(f"{platform : <15} {sha}")
 
 @task(
     help={
@@ -80,18 +97,23 @@ def update_go(_: Context, version: str, check_archive: Optional[bool] = False):
             f"The version {version} doesn't have an expected format, it should be 3 numbers separated with a dot."
         )
 
-    shas = _get_expected_sha256(version)
+    shas = _get_go_upstream_sha256(version)
+    msgo_shas = _get_msgo_sha256(version)
     if check_archive:
-        _check_archive(version, shas)
+        _check_archive(version, shas, "https://go.dev/dl")
+        _check_archive(version, msgo_shas, "https://aka.ms/golang/release/latest")
 
     print(
         f"Please check that you see the same SHAs on https://go.dev/dl for go{version}:"
     )
-    for (os, arch), sha in shas:
-        platform = f"[{os}/{arch}]"
-        print(f"{platform : <15} {sha}")
+
+    _display_shas(shas, "Upstream Go")
+    _display_shas(msgo_shas, "Microsoft Go")
 
     with open("go.env", "w") as writer:
         print(f"GO_VERSION={version}", file=writer)
         for (os, arch), sha in shas:
             print(f"GO_SHA256_{os.upper()}_{arch.upper()}={sha}", file=writer)
+        for (os, arch), sha in msgo_shas:
+            print(f"MSGO_SHA256_{os.upper()}_{arch.upper()}={sha}", file=writer)
+