Skip to content

Commit

Permalink
address reviews
Browse files Browse the repository at this point in the history
  • Loading branch information
ofek committed Nov 15, 2024
1 parent 4c97c5e commit 6b10e8a
Show file tree
Hide file tree
Showing 10 changed files with 322 additions and 118 deletions.
10 changes: 5 additions & 5 deletions .gitlab/dev_envs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ lint_dev_envs:
- hadolint dev-envs/linux/Dockerfile

test_dev_env_linux_amd64:
extends: [.build_dev_env, .linux_dev_env_amd64]
extends: [.linux_dev_env_amd64, .build_dev_env]
stage: dev_envs
needs: [lint_dev_envs]
rules:
Expand All @@ -49,7 +49,7 @@ test_dev_env_linux_amd64:
PLATFORM: linux

test_dev_env_linux_arm64:
extends: [.build_dev_env, .linux_dev_env_arm64]
extends: [.linux_dev_env_arm64, .build_dev_env]
stage: dev_envs
needs: [lint_dev_envs]
rules:
Expand All @@ -58,7 +58,7 @@ test_dev_env_linux_arm64:
PLATFORM: linux

build_dev_env_linux_amd64:
extends: [.build_dev_env, .linux_dev_env_amd64, .publish_dev_env]
extends: [.linux_dev_env_amd64, .publish_dev_env]
stage: dev_envs
needs: [lint_dev_envs]
rules:
Expand All @@ -67,7 +67,7 @@ build_dev_env_linux_amd64:
PLATFORM: linux

build_dev_env_linux_arm64:
extends: [.build_dev_env, .linux_dev_env_arm64, .publish_dev_env]
extends: [.linux_dev_env_arm64, .publish_dev_env]
stage: dev_envs
needs: [lint_dev_envs]
rules:
Expand All @@ -86,6 +86,6 @@ release_dev_env_linux:
strategy: depend
variables:
IMG_SOURCES: registry.ddbuild.io/ci/datadog-agent-buildimages/dev-env-linux:amd64,registry.ddbuild.io/ci/datadog-agent-buildimages/dev-env-linux:arm64
IMG_DESTINATIONS: agent-dev-env-linux:latest
IMG_DESTINATIONS: agent-dev-env-linux:latest,agent-dev-env-linux:v$CI_PIPELINE_ID-$CI_COMMIT_SHORT_SHA
IMG_REGISTRIES: dockerhub
IMG_SIGNING: "false"
24 changes: 8 additions & 16 deletions dev-envs/linux/git.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,24 +2,16 @@
IFS=$'\n\t'
set -euxo pipefail

VERSION="2.37.3"
url="https://github.com/git/git/archive/refs/tags/v${VERSION}.tar.gz"

archive_name=$(basename "${url}")
workdir="/tmp/setup-${archive_name}"
mkdir -p "${workdir}"
curl "${url}" -Lo "${workdir}/${archive_name}"
tar -xf "${workdir}/${archive_name}" -C "${workdir}" --strip-components 1

pushd "${workdir}"
git_dir="${HOME}/git"

# https://git-scm.com/book/en/v2/Getting-Started-Installing-Git#_installing_from_source
make configure
./configure --prefix="${git_dir}"
make prefix="${git_dir}" -j "$(nproc)" all
make prefix="${git_dir}" install
popd
rm -rf "${workdir}"
install-from-source \
--version "2.37.3" \
--digest "730ea150a9af30e6301d3ff4169567d5a5c52950e122c1a10d21998f7a7f70d7" \
--url "https://github.com/git/git/archive/refs/tags/v{{version}}.tar.gz" \
--relative-path "git-{{version}}" \
--configure-script "make configure && ./configure --prefix=\"${git_dir}\"" \
--install-script "make prefix=\"${git_dir}\" -j \"$(nproc)\" all && make prefix=\"${git_dir}\" install"

# The Agent build defines an older version but we require a newer one for SSH signing so we
# install at a different path so as to not conflict with the build process
Expand Down
10 changes: 10 additions & 0 deletions dev-envs/linux/nushell.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,23 @@ IFS=$'\n\t'
set -euxo pipefail

VERSION="0.99.1"
DIGEST="bf1224c7866a670022232c2e832a9f63141378d1f1c3552defa4200902c4379a"

arch=$(uname -m)
url="https://github.com/nushell/nushell/releases/download/${VERSION}/nu-${VERSION}-${arch}-unknown-linux-musl.tar.gz"
install_dir="${HOME}/.nushell"

mkdir -p "${install_dir}"
curl "${url}" -Lo archive.tar.gz

digest=$(openssl dgst -sha256 archive.tar.gz | cut -d' ' -f2)
if [[ "${digest}" != "${DIGEST}" ]]; then
echo "Digest mismatch"
echo "Expected: ${DIGEST}"
echo "Got: ${digest}"
exit 1
fi

tar -xzf archive.tar.gz -C "${install_dir}" --strip-components=1
rm archive.tar.gz

Expand Down
9 changes: 9 additions & 0 deletions dev-envs/linux/python.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ set -euxo pipefail

VERSION="3.12.7"
RELEASE="20241016"
DIGEST="c3055f2aaa2ca942cbf652bf8655f69390cdf920ae911a6778a8d92c6169a808"

arch=$(uname -m)
if [[ "$arch" == "aarch64" ]]; then
Expand All @@ -16,6 +17,14 @@ url="https://github.com/indygreg/python-build-standalone/releases/download/${REL
curl "${url}" -Lo cpython.tar.gz
mkdir -p /tools
tar -xf cpython.tar.gz -C /tools

digest=$(openssl dgst -sha256 cpython.tar.gz | cut -d' ' -f2)
if [[ "${digest}" != "${DIGEST}" ]]; then
echo "Digest mismatch"
echo "Expected: ${DIGEST}"
echo "Got: ${digest}"
exit 1
fi
rm cpython.tar.gz

/tools/python/bin/python -m venv "${HOME}/.venv"
Expand Down
95 changes: 95 additions & 0 deletions dev-envs/linux/scripts/install-binary.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
#!/bin/bash -l
IFS=$'\n\t'
set -euxo pipefail

BINARIES=()
# Set default values
ALGORITHM="sha256"
TOP_LEVEL="0"
UNPACK_COMMAND=""

while [[ $# -gt 0 ]]; do
case $1 in
--version)
VERSION="$2"
shift
shift
;;
--url)
URL="$2"
shift
shift
;;
--digest)
DIGEST="$2"
shift
shift
;;
--name)
BINARIES+=("$2")
shift
shift
;;
--algorithm)
ALGORITHM="$2"
shift
shift
;;
--unpack-command)
UNPACK_COMMAND="$2"
shift
shift
;;
--top-level)
TOP_LEVEL="1"
shift
;;
*)
echo "Unknown option $1"
exit 1
;;
esac
done

URL="${URL//'{{version}}'/${VERSION}}"

file_name=$(basename "${URL}")
file_path="/tmp/${file_name}"

curl "${URL}" -Lo "${file_path}"

digest=$(openssl dgst -"${ALGORITHM}" "${file_path}" | cut -d' ' -f2)
if [[ "${digest}" != "${DIGEST}" ]]; then
echo "Digest mismatch"
echo "Expected: ${DIGEST}"
echo "Got: ${digest}"
exit 1
fi

if [[ "${file_name}" =~ \.tar\.gz$ ]]; then
for binary in "${BINARIES[@]}"; do
if [[ "${TOP_LEVEL}" == "1" ]]; then
tar -xf "${file_path}" -C /usr/local/bin "${binary}"
else
tar -xf "${file_path}" -C /usr/local/bin --strip-components=1 --wildcards "*/${binary}"
fi
chmod +x "/usr/local/bin/${binary}"
done
elif [[ "${file_name}" =~ \.zip$ ]]; then
if [[ -n "${UNPACK_COMMAND}" ]]; then
unpack_command="${UNPACK_COMMAND//'{{file_path}}'/${file_path}}"
eval "${unpack_command}"
for binary in "${BINARIES[@]}"; do
chmod +x "/usr/local/bin/${binary}"
done
else
for binary in "${BINARIES[@]}"; do
unzip -j "${file_path}" "${binary}" -d /usr/local/bin
chmod +x "/usr/local/bin/${binary}"
done
fi
else
target_path="/usr/local/bin/${BINARIES[0]}"
mv "${file_path}" "${target_path}"
chmod +x "${target_path}"
fi
78 changes: 78 additions & 0 deletions dev-envs/linux/scripts/install-from-source.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
#!/bin/bash -l
IFS=$'\n\t'
set -euxo pipefail

# Set default values
ALGORITHM="sha256"
CONFIGURE_SCRIPT="./configure"
INSTALL_SCRIPT="make -j \"$(nproc)\" && make install"

while [[ $# -gt 0 ]]; do
case $1 in
--version)
VERSION="$2"
shift
shift
;;
--url)
URL="$2"
shift
shift
;;
--relative-path)
RELATIVE_PATH="$2"
shift
shift
;;
--digest)
DIGEST="$2"
shift
shift
;;
--algorithm)
ALGORITHM="$2"
shift
shift
;;
--configure-script)
CONFIGURE_SCRIPT="$2"
shift
shift
;;
--install-script)
INSTALL_SCRIPT="$2"
shift
shift
;;
*)
echo "Unknown option $1"
exit 1
;;
esac
done

URL="${URL//'{{version}}'/${VERSION}}"
RELATIVE_PATH="${RELATIVE_PATH//'{{version}}'/${VERSION}}"

archive_name=$(basename "${URL}")
work_dir="/tmp/$(openssl rand -base64 16)"
archive_path="${work_dir}/${archive_name}"
mkdir -p "${work_dir}"
curl "${URL}" -Lo "${archive_path}"

digest=$(openssl dgst -"${ALGORITHM}" "${archive_path}" | cut -d' ' -f2)
if [[ "${digest}" != "${DIGEST}" ]]; then
echo "Digest mismatch"
echo "Expected: ${DIGEST}"
echo "Got: ${digest}"
exit 1
fi

tar -xf "${archive_path}" -C "${work_dir}"
ls "${work_dir}"

pushd "${work_dir}/${RELATIVE_PATH}"
eval "${CONFIGURE_SCRIPT}"
eval "${INSTALL_SCRIPT}"
popd
rm -rf "${work_dir}"
22 changes: 6 additions & 16 deletions dev-envs/linux/ssh.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,25 +3,15 @@ IFS=$'\n\t'
set -euxo pipefail

# New versions require OpenSSL >= 1.1.1
VERSION="V_9_3_P2"
url="https://github.com/openssh/openssh-portable/archive/refs/tags/${VERSION}.tar.gz"

archive_name=$(basename "${url}")
workdir="/tmp/setup-${archive_name}"
mkdir -p "${workdir}"
curl "${url}" -Lo "${workdir}/${archive_name}"
tar -xf "${workdir}/${archive_name}" -C "${workdir}" --strip-components 1
install-from-source \
--version "V_9_3_P2" \
--digest "db2463f84e50bd2f3c1a0dd6cc9b652f48b024d65ba43cca29ae451087dd6738" \
--url "https://github.com/openssh/openssh-portable/archive/refs/tags/{{version}}.tar.gz" \
--relative-path "openssh-portable-{{version}}" \
--configure-script "autoreconf && ./configure"

useradd -r -s /sbin/nologin sshd

pushd "${workdir}"
autoreconf
./configure
make -j "$(nproc)"
make install
popd
rm -rf "${workdir}"

# Ameliorate transient network issues for things that can
# retry like Visual Studio Code's Remote - SSH extension
passwd -d root
Expand Down
Loading

0 comments on commit 6b10e8a

Please sign in to comment.