update builder-base to pull upstream tar, unpack, and link matching e…

…ksgo installs
aws · Jul 1, 2024 · 1513776 · 1513776
1 parent 447de69
commit 1513776
Show file tree

Hide file tree

Showing 3 changed files with 137 additions and 84 deletions.
diff --git a/builder-base/Dockerfile b/builder-base/Dockerfile
@@ -359,8 +359,8 @@ ENV PATH="/go/bin/:$PATH"
 COPY --link --from=upx  /upx /
 COPY ./scripts/install_base_yum_packages.sh ./scripts/remove_yum_packages.sh ./scripts/common_vars.sh \
     ./scripts/install_golang.sh /
-COPY $GOLANG_RPM_SOURCE_DIR/x86_64/golang*1.21*.rpm /tmp/x86_64/
-COPY $GOLANG_RPM_SOURCE_DIR/aarch64/golang*1.21*.rpm /tmp/aarch64/
+COPY $GOLANG_RPM_SOURCE_DIR/linux/arm64/go1.21*.tar.gz /tmp/linux/arm64/
+COPY $GOLANG_RPM_SOURCE_DIR/linux/amd64/go1.21*.tar.gz /tmp/linux/amd64/
 RUN --mount=type=cache,target=/var/cache/yum,sharing=locked \
     /install_base_yum_packages.sh && \
     /install_golang.sh $GOLANG_VERSION_121 && \
@@ -377,8 +377,8 @@ ENV PATH="/go/bin/:$PATH"
 COPY --link --from=upx  /upx /
 COPY ./scripts/install_base_yum_packages.sh ./scripts/remove_yum_packages.sh ./scripts/common_vars.sh \
     ./scripts/install_golang.sh /
-COPY $GOLANG_RPM_SOURCE_DIR/x86_64/golang*1.22*.rpm /tmp/x86_64/
-COPY $GOLANG_RPM_SOURCE_DIR/aarch64/golang*1.22*.rpm /tmp/aarch64/
+COPY $GOLANG_RPM_SOURCE_DIR/linux/arm64/go1.22*.tar.gz /tmp/linux/arm64/
+COPY $GOLANG_RPM_SOURCE_DIR/linux/amd64/go1.22*.tar.gz /tmp/linux/amd64/
 RUN --mount=type=cache,target=/var/cache/yum,sharing=locked \
     /install_base_yum_packages.sh && \
     /install_golang.sh $GOLANG_VERSION_122 && \

diff --git a/builder-base/scripts/download_golang.sh b/builder-base/scripts/download_golang.sh
@@ -27,44 +27,63 @@ RELEASE_NUMBER="$(echo $VERSION | cut -d'-' -f 2)"
 
 source $SCRIPT_ROOT/common_vars.sh
 
-function build::go::download(){
-    # Set up specific go version by using go get, additional versions apart from default can be installed by calling
-    # the function again with the specific parameter.
-    local version=${1%-*}
-    local outputDir=${2}
-    local arch=${3}
+function build::eksgo::download() {
+  # Set up specific go version by using go get, additional versions apart from default can be installed by calling
+  # the function again with the specific parameter.
+  local version=${1%-*}
+  local outputDir=${2}
+  local arch=${3}
 
-    for artifact in golang golang-bin; do
-        local filename="$outputDir/$arch/$artifact-$version-$RELEASE_NUMBER.amzn2.eks.$arch.rpm"
-        if [ ! -f $filename ]; then
-            curl -sSLf --retry 5 https://distro.eks.amazonaws.com/golang-go$version/releases/$RELEASE_NUMBER/$arch/RPMS/$arch/$artifact-$version-$RELEASE_NUMBER.amzn2.eks.$arch.rpm -o $filename --create-dirs
-            curl -sSLf --retry 5 https://distro.eks.amazonaws.com/golang-go$version/releases/$RELEASE_NUMBER/$arch/RPMS/$arch/$artifact-$version-$RELEASE_NUMBER.amzn2.eks.$arch.rpm.sha256 -o $filename.sha256
+  for artifact in golang golang-bin; do
+    local filename="$outputDir/$arch/$artifact-$version-$RELEASE_NUMBER.amzn2.eks.$arch.rpm"
+    if [ ! -f $filename ]; then
+      curl -sSLf --retry 5 https://distro.eks.amazonaws.com/golang-go$version/releases/$RELEASE_NUMBER/$arch/RPMS/$arch/$artifact-$version-$RELEASE_NUMBER.amzn2.eks.$arch.rpm -o $filename --create-dirs
+      curl -sSLf --retry 5 https://distro.eks.amazonaws.com/golang-go$version/releases/$RELEASE_NUMBER/$arch/RPMS/$arch/$artifact-$version-$RELEASE_NUMBER.amzn2.eks.$arch.rpm.sha256 -o $filename.sha256
 
-            if [[ $(sha256sum ${filename} | cut -d' ' -f1) != $(cut -d' ' -f1 "${filename}.sha256") ]] ; then 
-                echo "Checksum doesn't match!"
-                exit 1
-            fi
-        fi
-    done
+      if [[ $(sha256sum ${filename} | cut -d' ' -f1) != $(cut -d' ' -f1 "${filename}.sha256") ]]; then
+        echo "Checksum doesn't match!"
+        exit 1
+      fi
+    fi
+  done
 
-    for artifact in golang-docs golang-misc golang-tests golang-src; do
-        local filename="$outputDir/$arch/$artifact-$version-$RELEASE_NUMBER.amzn2.eks.noarch.rpm"
-        if [ ! -f $filename ]; then
-            curl -sSLf --retry 5 https://distro.eks.amazonaws.com/golang-go$version/releases/$RELEASE_NUMBER/$arch/RPMS/noarch/$artifact-$version-$RELEASE_NUMBER.amzn2.eks.noarch.rpm -o $filename --create-dirs
-            curl -sSLf --retry 5 https://distro.eks.amazonaws.com/golang-go$version/releases/$RELEASE_NUMBER/$arch/RPMS/noarch/$artifact-$version-$RELEASE_NUMBER.amzn2.eks.noarch.rpm.sha256 -o $filename.sha256
+  for artifact in golang-docs golang-misc golang-tests golang-src; do
+    local filename="$outputDir/$arch/$artifact-$version-$RELEASE_NUMBER.amzn2.eks.noarch.rpm"
+    if [ ! -f $filename ]; then
+      curl -sSLf --retry 5 https://distro.eks.amazonaws.com/golang-go$version/releases/$RELEASE_NUMBER/$arch/RPMS/noarch/$artifact-$version-$RELEASE_NUMBER.amzn2.eks.noarch.rpm -o $filename --create-dirs
+      curl -sSLf --retry 5 https://distro.eks.amazonaws.com/golang-go$version/releases/$RELEASE_NUMBER/$arch/RPMS/noarch/$artifact-$version-$RELEASE_NUMBER.amzn2.eks.noarch.rpm.sha256 -o $filename.sha256
 
-            if [[ $(sha256sum ${filename} | cut -d' ' -f1) != $(cut -d' ' -f1 "${filename}.sha256") ]] ; then 
-                echo "Checksum doesn't match!"
-                exit 1
-            fi
-        fi
-    done
+      if [[ $(sha256sum ${filename} | cut -d' ' -f1) != $(cut -d' ' -f1 "${filename}.sha256") ]]; then
+        echo "Checksum doesn't match!"
+        exit 1
+      fi
+    fi
+  done
 }
 
-if [[ $ARCHITECTURE =~ "linux/amd64" ]] ; then 
-    build::go::download "${VERSION}" "$OUTPUT_DIR" "x86_64"
-fi
+function build::go::download {
+  # Set up specific go version by using go get, additional versions apart from default can be installed by calling
+  # the function again with the specific parameter.
+  local version=${1%-*}
+  local outputDir=${2}
+  local archs=${3}
+
+  for arch in ${archs/,/ }; do
+    local filename="$outputDir/${arch}/go$version.${arch/\//-}.tar.gz"
+    if [ ! -f $filename ]; then
+      curl -sSLf --retry 5 "https://go.dev/dl/go$version.${arch/\//-}.tar.gz" -o $filename --create-dirs
+    fi
+  done
+}
 
-if [[ $ARCHITECTURE =~ "linux/arm64" ]] ; then 
-    build::go::download "${VERSION}" "$OUTPUT_DIR" "aarch64"
-fi
+if [[ ${VERSION:2:2} -ge "21" ]]; then
+  build::go::download "${VERSION}" "$OUTPUT_DIR" "$ARCHITECTURE"
+else
+  if [[ $ARCHITECTURE =~ "linux/amd64" ]]; then
+    build::eksgo::download "${VERSION}" "$OUTPUT_DIR" "x86_64"
+  fi
+
+  if [[ $ARCHITECTURE =~ "linux/arm64" ]]; then
+    build::eksgo::download "${VERSION}" "$OUTPUT_DIR" "aarch64"
+  fi
+fi
diff --git a/builder-base/scripts/install_golang.sh b/builder-base/scripts/install_golang.sh
@@ -21,7 +21,7 @@ SCRIPT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
 
 VERSION="$1"
 
-GOLANG_MAJOR_VERSION=$(if [[ $(echo "$VERSION"|awk -F'.' '{print NF}') -ge 3 ]]; then echo ${VERSION%.*}; else echo ${VERSION%-*}; fi)
+GOLANG_MAJOR_VERSION=$(if [[ $(echo "$VERSION" | awk -F'.' '{print NF}') -ge 3 ]]; then echo ${VERSION%.*}; else echo ${VERSION%-*}; fi)
 
 NEWROOT=/golang-${GOLANG_MAJOR_VERSION}
 
@@ -30,68 +30,102 @@ source $SCRIPT_ROOT/common_vars.sh
 mkdir -p /go/src /go/bin /go/pkg
 
 function build::go::symlink() {
-    local -r version=$1
-
-    # Removing the last number as we only care about the major version of golang
-    local -r majorversion=$(if [[ $(echo "$version"|awk -F'.' '{print NF}') -ge 3 ]]; then echo ${version%.*}; else echo ${version%-*}; fi)
-    mkdir -p ${GOPATH}/go${majorversion}/bin
-    
-    for binary in go gofmt; do
-        ln -s /root/sdk/go${version}/bin/${binary} ${GOPATH}/go${majorversion}/bin/${binary}
-    done
-    
-    if [ "$version" != "$majorversion" ]; then
-        ln -s ${GOPATH}/bin/go${version} ${GOPATH}/bin/go${majorversion}
-    fi
+  local -r version=$1
+
+  # Removing the last number as we only care about the major version of golang
+  local -r majorversion=$(if [[ $(echo "$version" | awk -F'.' '{print NF}') -ge 3 ]]; then echo ${version%.*}; else echo ${version%-*}; fi)
+  mkdir -p ${GOPATH}/go${majorversion}/bin
+
+  for binary in go gofmt; do
+    ln -s /root/sdk/go${version}/bin/${binary} ${GOPATH}/go${majorversion}/bin/${binary}
+  done
+
+  if [ "$version" != "$majorversion" ]; then
+    ln -s ${GOPATH}/bin/go${version} ${GOPATH}/bin/go${majorversion}
+  fi
 }
 
-function build::go::install(){
-    # Set up specific go version by using go get, additional versions apart from default can be installed by calling
-    # the function again with the specific parameter.
-    local version=${1%-*}
-    build::go::extract $version
-    build::go::symlink $version
+function build::go::install {
+  # Set up specific go version by using go get, additional versions apart from default can be installed by calling
+  # the function again with the specific parameter.
+  local version=${1%-*}
+  build::go::extract $version
+  build::go::symlink $version
 }
 
-function build::go::extract() {
-    local version=$1
+function build::go::extract {
+  local version=$1
 
-    if [ $TARGETARCH == 'amd64' ]; then
-        local arch='x86_64'
-    else
-        local arch='aarch64'
-    fi
+  mkdir -p /tmp/go-extracted
+  cd /tmp/go-extracted
+  tar -xzf /tmp/linux/$TARGETARCH/go$version.linux-$TARGETARCH.tar.gz
 
-    mkdir -p /tmp/go-extracted
-    for rpm in /tmp/$arch/golang-*.noarch.rpm /tmp/$arch/golang-*.$arch.rpm ; do $(cd /tmp/go-extracted && rpm2cpio $rpm | cpio -idm && rm -f $rpm); done
+  local -r golang_version=$(/tmp/go-extracted/go/bin/go version | grep -o "go[0-9].* " | xargs)
 
-    local -r golang_version=$(/tmp/go-extracted/usr/lib/golang/bin/go version | grep -o "go[0-9].* " | xargs)
+  mkdir -p /root/sdk/$golang_version
+  mv /tmp/go-extracted/go/* /root/sdk/$golang_version
 
-    mkdir -p /root/sdk/$golang_version
-    mv /tmp/go-extracted/usr/lib/golang/* /root/sdk/$golang_version
+  version=$(echo "$golang_version" | grep -o "[0-9].*")
+  ln -s /root/sdk/go${version}/bin/go ${GOPATH}/bin/$golang_version
 
-    for license_dir in "/usr/share/licenses/golang" "/usr/share/doc/golang-$VERSION"; do
-        if [ -d /tmp/go-extracted/$license_dir ]; then
-            mv /tmp/go-extracted/$license_dir/* /root/sdk/$golang_version
-        fi
-    done
+  # This is only for 1.21+ and is causing failures.
+  # newer versions of golang stopped shipping the compiled .a lib files
+  # removing from old versions since we do not need these during our builds
+  # find /root/sdk/go${version}/pkg -type f -name "*.a" -delete
 
-    version=$(echo "$golang_version" | grep -o "[0-9].*")
-    ln -s /root/sdk/go${version}/bin/go ${GOPATH}/bin/$golang_version
-
-    # newer versions of golang stopped shipping the compiled .a lib files
-    # removing from old versions since we do not need these during our builds
-    find /root/sdk/go${version}/pkg -type f -name "*.a" -delete
+  rm -rf /tmp/go-extracted /tmp/linux/$TARGETARCH/go*.tar.gz
+}
 
-    rm -rf /tmp/go-extracted /tmp/golang-*.rpm
+function build::eksgo::install() {
+  # Set up specific go version by using go get, additional versions apart from default can be installed by calling
+  # the function again with the specific parameter.
+  local version=${1%-*}
+  build::eksgo::extract $version
+  build::go::symlink $version
 }
 
+function build::eksgo::extract() {
+  local version=$1
+
+  if [ $TARGETARCH == 'amd64' ]; then
+    local arch='x86_64'
+  else
+    local arch='aarch64'
+  fi
+
+  mkdir -p /tmp/go-extracted
+  for rpm in /tmp/$arch/golang-*.noarch.rpm /tmp/$arch/golang-*.$arch.rpm; do $(cd /tmp/go-extracted && rpm2cpio $rpm | cpio -idm && rm -f $rpm); done
+
+  local -r golang_version=$(/tmp/go-extracted/usr/lib/golang/bin/go version | grep -o "go[0-9].* " | xargs)
+
+  mkdir -p /root/sdk/$golang_version
+  mv /tmp/go-extracted/usr/lib/golang/* /root/sdk/$golang_version
+
+  for license_dir in "/usr/share/licenses/golang" "/usr/share/doc/golang-$VERSION"; do
+    if [ -d /tmp/go-extracted/$license_dir ]; then
+      mv /tmp/go-extracted/$license_dir/* /root/sdk/$golang_version
+    fi
+  done
+
+  version=$(echo "$golang_version" | grep -o "[0-9].*")
+  ln -s /root/sdk/go${version}/bin/go ${GOPATH}/bin/$golang_version
+
+  # newer versions of golang stopped shipping the compiled .a lib files
+  # removing from old versions since we do not need these during our builds
+  find /root/sdk/go${version}/pkg -type f -name "*.a" -delete
+
+  rm -rf /tmp/go-extracted /tmp/golang-*.rpm
+}
 
-build::go::install "${VERSION}"
+if [[ ${VERSION:2:2} -ge "21" ]]; then
+  build::go::install "${VERSION}"
+else
+  build::eksgo::install "${VERSION}"
+fi
 
 # symlink default golang install to newroot bin
 for binary in go gofmt; do
-    ln -s ${GOPATH}/go${GOLANG_MAJOR_VERSION}/bin/${binary} ${USR_BIN}/${binary}
+  ln -s ${GOPATH}/go${GOLANG_MAJOR_VERSION}/bin/${binary} ${USR_BIN}/${binary}
 done
 
 mkdir -p ${NEWROOT}/root