diff --git a/agent-deploy/Dockerfile b/agent-deploy/Dockerfile index 14c19460..c8ffed4e 100644 --- a/agent-deploy/Dockerfile +++ b/agent-deploy/Dockerfile @@ -40,9 +40,6 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ libgmp-dev \ libssl-dev \ procps \ - python-boto \ - python-deltarpm \ - python2.7-dev \ shellcheck \ make \ zlib1g-dev \ @@ -73,15 +70,6 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ # Use gnupg1 instead of gnupg2, our scripts and rpm-s3 are currently not compatible with gnupg2 RUN mv /usr/bin/gpg /usr/bin/gpg2 && ln -s /usr/bin/gpg1 /usr/bin/gpg -# Python 2 deps -RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py -o get-pip.py \ - && python2 get-pip.py - -RUN python2 -m pip install \ - awscli==1.18.140 \ - boto3==1.14.7 \ - pexpect==3.2 - # Python install ENV PYENV_ROOT="/.pyenv" ENV PATH="$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH" @@ -128,7 +116,7 @@ RUN git clone https://github.com/chocolatey/choco.git \ && cp ./docker/choco_wrapper /usr/local/bin/choco \ && rm -rf /choco -ENV ChocolateyInstall=/opt/chocolatey +ENV ChocolateyInstall=/opt/chocolatey # RVM COPY ./rvm/gpg-keys /gpg-keys diff --git a/build-container.ps1 b/build-container.ps1 index b2f394ac..0908d210 100644 --- a/build-container.ps1 +++ b/build-container.ps1 @@ -39,6 +39,13 @@ foreach ($line in $lines) { $arglist += "$line" } +# Read arguments from deva.env file +$lines = Get-Content -Path 'deva.env' +foreach ($line in $lines) { + $arglist += "--build-arg" + $arglist += "$line" +} + foreach ($h in $SoftwareTable.GetEnumerator()){ if( -not ($($h.Key) -like "*SHA256")){ $arglist += "--build-arg" @@ -50,7 +57,7 @@ if( -not $Cache) { $arglist += "--no-cache" } -$arglist += -split "-m 4096M --build-arg BASE_IMAGE=$($BaseTable[$kernelver]) --build-arg DD_TARGET_ARCH=$Arch --build-arg WINDOWS_VERSION=$kernelver -t $Tag --file .\windows\Dockerfile ." +$arglist += -split "-m 4096M --build-arg BASE_IMAGE=$($BaseTable[$kernelver]) --build-arg DD_TARGET_ARCH=$Arch --build-arg WINDOWS_VERSION=$kernelver -t $Tag --file .\windows\Dockerfile ." # Write-Host -ForegroundColor Green "Building with the following command:" # Write-Host -ForegroundColor Green "$buildcommand `n" filter timestamp {"$(Get-Date -Format o): $_"} diff --git a/deb-arm/Dockerfile b/deb-arm/Dockerfile index bb716336..20cd8e5e 100644 --- a/deb-arm/Dockerfile +++ b/deb-arm/Dockerfile @@ -106,7 +106,7 @@ RUN if [ "$DD_TARGET_ARCH" = "armhf" ] ; then \ ENV PATH "~/.cargo/bin:${PATH}" # CONDA -COPY python-packages-versions.txt setup_python.sh requirements.txt requirements-py2.txt / +COPY python-packages-versions.txt setup_python.sh requirements.txt / COPY requirements /requirements ENV DEVA_VERSION=$DEVA_VERSION ENV DEVA_NO_DYNAMIC_DEPS=$DEVA_NO_DYNAMIC_DEPS diff --git a/deb-x64/Dockerfile b/deb-x64/Dockerfile index 65946a52..d5a7b4b2 100644 --- a/deb-x64/Dockerfile +++ b/deb-x64/Dockerfile @@ -142,7 +142,7 @@ RUN curl -fsSL https://github.com/DataDog/datadog-ci/releases/download/v${CI_UPL chmod +x /usr/local/bin/datadog-ci # CONDA -COPY python-packages-versions.txt setup_python.sh requirements.txt requirements-py2.txt / +COPY python-packages-versions.txt setup_python.sh requirements.txt / COPY requirements /requirements ENV DEVA_VERSION=$DEVA_VERSION ENV DEVA_NO_DYNAMIC_DEPS=$DEVA_NO_DYNAMIC_DEPS diff --git a/linux-glibc-2.17-x64/Dockerfile b/linux-glibc-2.17-x64/Dockerfile index 012c2b00..cd12d6ea 100644 --- a/linux-glibc-2.17-x64/Dockerfile +++ b/linux-glibc-2.17-x64/Dockerfile @@ -75,7 +75,7 @@ RUN mkdir -p /opt/mqm \ && rm -rf /tmp/mq_client.tar.gz # CONDA -COPY python-packages-versions.txt setup_python.sh requirements.txt requirements-py2.txt / +COPY python-packages-versions.txt setup_python.sh requirements.txt / COPY requirements /requirements ENV DEVA_VERSION=$DEVA_VERSION ENV DEVA_NO_DYNAMIC_DEPS=$DEVA_NO_DYNAMIC_DEPS diff --git a/linux-glibc-2.23-arm64/Dockerfile b/linux-glibc-2.23-arm64/Dockerfile index 48cc6e44..ebd426da 100644 --- a/linux-glibc-2.23-arm64/Dockerfile +++ b/linux-glibc-2.23-arm64/Dockerfile @@ -65,7 +65,7 @@ RUN cd /build && \ # CONDA -COPY python-packages-versions.txt setup_python.sh requirements.txt requirements-py2.txt / +COPY python-packages-versions.txt setup_python.sh requirements.txt / COPY requirements /requirements ENV DEVA_VERSION=$DEVA_VERSION ENV DEVA_NO_DYNAMIC_DEPS=$DEVA_NO_DYNAMIC_DEPS diff --git a/python-packages-versions.txt b/python-packages-versions.txt index e3f0b9e4..5458173a 100644 --- a/python-packages-versions.txt +++ b/python-packages-versions.txt @@ -1,9 +1,5 @@ # NOTE: this is used in both Linux and Windows image builds; only use comments, newlines # and variable definitions here. Also don't quote variable values. -# These seem to be the latest version that reliably work with Python 2; don't update -DD_PIP_VERSION=20.3.3 -DD_SETUPTOOLS_VERSION=44.1.1 - DD_PIP_VERSION_PY3=24.2 DD_SETUPTOOLS_VERSION_PY3=74.1.3 diff --git a/rpm-arm64/Dockerfile b/rpm-arm64/Dockerfile index be9683ec..21d0bede 100644 --- a/rpm-arm64/Dockerfile +++ b/rpm-arm64/Dockerfile @@ -94,7 +94,7 @@ RUN curl -fsSL https://github.com/DataDog/datadog-ci/releases/download/v${CI_UPL && chmod +x /usr/local/bin/datadog-ci # CONDA -COPY python-packages-versions.txt setup_python.sh requirements.txt requirements-py2.txt / +COPY python-packages-versions.txt setup_python.sh requirements.txt / COPY requirements /requirements ENV DEVA_VERSION=$DEVA_VERSION ENV DEVA_NO_DYNAMIC_DEPS=$DEVA_NO_DYNAMIC_DEPS diff --git a/rpm-armhf/Dockerfile b/rpm-armhf/Dockerfile index 6a184523..fd259baf 100644 --- a/rpm-armhf/Dockerfile +++ b/rpm-armhf/Dockerfile @@ -127,7 +127,7 @@ RUN curl -sSL -o rustup-init https://static.rust-lang.org/rustup/archive/${RUSTU ENV PATH "~/.cargo/bin:${PATH}" # CONDA -COPY python-packages-versions.txt setup_python.sh requirements.txt requirements-py2.txt / +COPY python-packages-versions.txt setup_python.sh requirements.txt / COPY requirements /requirements ENV DEVA_VERSION=$DEVA_VERSION ENV DEVA_NO_DYNAMIC_DEPS=$DEVA_NO_DYNAMIC_DEPS diff --git a/rpm-x64/Dockerfile b/rpm-x64/Dockerfile index 3f943f20..c0f65017 100644 --- a/rpm-x64/Dockerfile +++ b/rpm-x64/Dockerfile @@ -127,7 +127,7 @@ RUN git config --global user.email "package@datadoghq.com" RUN git config --global user.name "Bits" # CONDA -COPY python-packages-versions.txt setup_python.sh requirements.txt requirements-py2.txt / +COPY python-packages-versions.txt setup_python.sh requirements.txt / COPY requirements /requirements ENV DEVA_VERSION=$DEVA_VERSION ENV DEVA_NO_DYNAMIC_DEPS=$DEVA_NO_DYNAMIC_DEPS diff --git a/setup_python.sh b/setup_python.sh index bb2a23dc..8c2cad75 100755 --- a/setup_python.sh +++ b/setup_python.sh @@ -9,7 +9,6 @@ function detect_distro(){ DISTRIBUTION=$(lsb_release -d 2>/dev/null | grep -Eo $KNOWN_DISTRIBUTION || grep -Eo $KNOWN_DISTRIBUTION /etc/issue 2>/dev/null || grep -Eo $KNOWN_DISTRIBUTION /etc/Eos-release 2>/dev/null || grep -m1 -Eo $KNOWN_DISTRIBUTION /etc/os-release 2>/dev/null || uname -s) } -PY2_VERSION=2 PY3_VERSION=3.12.6 DD_CONDA_VERSION=4.9.2-7 @@ -83,17 +82,8 @@ conda init bash source /root/.bashrc # Setup pythons -conda create -n ddpy2 python python=$PY2_VERSION conda create -n ddpy3 python python=$PY3_VERSION -# Update pip, setuptools and misc deps -conda activate ddpy2 -pip install -i https://pypi.python.org/simple pip==${DD_PIP_VERSION} -pip install setuptools==${DD_SETUPTOOLS_VERSION} -pip install --no-build-isolation "cython<3.0.0" PyYAML==5.4.1 -pip install -r requirements-py2.txt -pip uninstall -y cython # remove cython to prevent further issue with nghttp2 - # Update pip, setuptools and misc deps conda activate ddpy3 pip install -i https://pypi.python.org/simple pip==${DD_PIP_VERSION_PY3} @@ -103,15 +93,6 @@ pip install "git+https://github.com/DataDog/datadog-agent-dev.git@${DEVA_VERSION deva -v self dep sync -f legacy-build pip uninstall -y cython # remove cython to prevent further issue with nghttp2 - -if [ "$DD_TARGET_ARCH" = "aarch64" ] ; then - # Conda creates "lib" but on Amazon Linux, the embedded Python2 we use in unit tests will look in "lib64" instead - ln -s "${CONDA_PATH}/envs/ddpy2/lib" "${CONDA_PATH}/envs/ddpy2/lib64" -fi - -# Add python3's invoke to the PATH even when ddpy3 is not active, since we want to use python3 invoke to run python2 tests -ln -s ${CONDA_PATH}/envs/ddpy3/bin/inv /usr/local/bin - conda clean -a echo "conda activate ddpy3" >> /root/.bashrc diff --git a/system-probe_arm64/Dockerfile b/system-probe_arm64/Dockerfile index e7074e01..9dd8ec2a 100644 --- a/system-probe_arm64/Dockerfile +++ b/system-probe_arm64/Dockerfile @@ -55,7 +55,7 @@ ENV CONDA_PATH /root/miniforge3 ENV DD_TARGET_ARCH $DD_TARGET_ARCH # CONDA -COPY python-packages-versions.txt setup_python.sh requirements.txt requirements-py2.txt / +COPY python-packages-versions.txt setup_python.sh requirements.txt / COPY requirements /requirements ENV DEVA_VERSION=$DEVA_VERSION ENV DEVA_NO_DYNAMIC_DEPS=$DEVA_NO_DYNAMIC_DEPS diff --git a/system-probe_x64/Dockerfile b/system-probe_x64/Dockerfile index 30e07086..49aa8e0e 100644 --- a/system-probe_x64/Dockerfile +++ b/system-probe_x64/Dockerfile @@ -60,7 +60,7 @@ ENV CONDA_PATH /root/miniforge3 ENV DD_TARGET_ARCH $DD_TARGET_ARCH # CONDA -COPY python-packages-versions.txt setup_python.sh requirements.txt requirements-py2.txt / +COPY python-packages-versions.txt setup_python.sh requirements.txt / COPY requirements /requirements ENV DEVA_VERSION=$DEVA_VERSION ENV DEVA_NO_DYNAMIC_DEPS=$DEVA_NO_DYNAMIC_DEPS diff --git a/windows/Dockerfile b/windows/Dockerfile index 8a8a6c97..268edb67 100644 --- a/windows/Dockerfile +++ b/windows/Dockerfile @@ -35,7 +35,6 @@ ARG WIX_VERSION ARG CMAKE_VERSION ARG MSYS_VERSION ARG PYTHON_VERSION -ARG EMBEDDED_PYTHON_2_VERSION ARG EMBEDDED_PYTHON_3_VERSION ARG NUGET_VERSION ARG CODEQL_VERSION @@ -53,7 +52,6 @@ LABEL wix_version=${WIX_VERSION} LABEL cmake_version=${CMAKE_VERSION} LABEL msys_version=${MSYS_VERSION} LABEL system_python_version=${PYTHON_VERSION} -LABEL embedded_py2_version=${EMBEDDED_PYTHON_2_VERSION} LABEL embedded_py3_version=${EMBEDDED_PYTHON_3_VERSION} LABEL nuget_version=${NUGET_VERSION} LABEL codeql_version=${CODEQL_VERSION} @@ -67,7 +65,7 @@ COPY ./windows/helpers/*.ps1 c:/scripts/helpers/ COPY go.env c:/ COPY deva.env c:/ -COPY python-packages-versions.txt requirements.txt requirements-py2.txt / +COPY python-packages-versions.txt requirements.txt / COPY requirements /requirements COPY ./windows/set_cpython_compiler.cmd set_cpython_compiler.cmd diff --git a/windows/helpers/phase1/install_vcpython.ps1 b/windows/helpers/phase1/install_vcpython.ps1 deleted file mode 100644 index 71160e2c..00000000 --- a/windows/helpers/phase1/install_vcpython.ps1 +++ /dev/null @@ -1,21 +0,0 @@ - - -$vcpmsi = 'https://s3.amazonaws.com/dd-agent-omnibus/VCForPython27.msi' - -Write-host -ForegroundColor Green Downloading VC++ for Python -$out = "$($PSScriptRoot)\vcp.msi" -$sha256 = "070474db76a2e625513a5835df4595df9324d820f9cc97eab2a596dcbc2f5cbf" - -$isInstalled, $isCurrent = Get-InstallUpgradeStatus -Component "vcpython" -Keyname "DownloadFile" -TargetValue $vcpmsi -if($isInstalled){ - Write-Host -ForegroundColor Green "VC++ for Python 2.7 already installed" - return -} -Get-RemoteFile -RemoteFile $vcpmsi -LocalFile $out -VerifyHash $sha256 - -Write-host -ForegroundColor Green VC++ for Python downloaded, installing... -Start-Process msiexec -ArgumentList '/q /i vcp.msi' -Wait -Remove-Item $out - -Set-InstalledVersionKey -Component "vcpython" -Keyname "DownloadFile" -TargetValue $vcpmsi -Write-Host -ForegroundColor Green Done with Visual C++ for Python diff --git a/windows/helpers/phase2/install_embedded_pythons.ps1 b/windows/helpers/phase2/install_embedded_pythons.ps1 index f899d71c..f398b7e4 100644 --- a/windows/helpers/phase2/install_embedded_pythons.ps1 +++ b/windows/helpers/phase2/install_embedded_pythons.ps1 @@ -1,5 +1,5 @@ # -# Installs embedded python2 and python3 for use in unit testing. +# Installs embedded python for use in unit testing. # Be careful to use unique environment variables indicating location # of the python files. # @@ -8,7 +8,6 @@ # # Uses: -# EMBEDDED_PYTHON_2_VERSION # EMBEDDED_PYTHON_3_VERSION # this downloads the necessary file @@ -22,17 +21,12 @@ if($Env:DD_DEV_TARGET -ne "Container") { return } -$py2 = "https://s3.amazonaws.com/dd-agent-omnibus/python-windows-${Env:EMBEDDED_PYTHON_2_VERSION}-amd64.zip" $py3 = "https://s3.amazonaws.com/dd-agent-omnibus/python-windows-${Env:EMBEDDED_PYTHON_3_VERSION}-amd64.zip" -$py2Target = "c:\embeddedpy\py${Env:EMBEDDED_PYTHON_2_VERSION}" $py3Target = "c:\embeddedpy\py${Env:EMBEDDED_PYTHON_3_VERSION}" -DownloadAndExpandTo -TargetDir $py2Target -SourceURL $py2 -Sha256 "$Env:EMBEDDED_PYTHON_2_SHA256" - DownloadAndExpandTo -TargetDir $py3Target -SourceURL $py3 -Sha256 "$Env:EMBEDDED_PYTHON_3_SHA256" -Add-EnvironmentVariable -Variable "TEST_EMBEDDED_PY2" -Value $py2Target -Global Add-EnvironmentVariable -Variable "TEST_EMBEDDED_PY3" -Value $py3Target -Global # Read DD_PIP_VERSION{,_PY3} and DD_SETUPTOOLS_VERSION{,_PY3} to variables @@ -41,15 +35,6 @@ Get-Content \python-packages-versions.txt | Where-Object { $_.Trim() -ne '' } | Add-EnvironmentVariable -Variable $var[0] -Value $var[1] -Local } -# Python 2 -$py2getpip = "https://raw.githubusercontent.com/pypa/get-pip/38e54e5de07c66e875c11a1ebbdb938854625dd8/public/2.7/get-pip.py" -$py2getpipsha256 = "40ee07eac6674b8d60fce2bbabc148cf0e2f1408c167683f110fd608b8d6f416" -Get-RemoteFile -LocalFile "get-pip.py" -RemoteFile $py2getpip -VerifyHash $py2getpipsha256 -& "$py2Target\python" get-pip.py pip==${Env:DD_PIP_VERSION} -If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } -& "$py2Target\python" -m pip install -r ../requirements-py2.txt -If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - # Python 3 $py3getpip = "https://raw.githubusercontent.com/pypa/get-pip/66d8a0f637083e2c3ddffc0cb1e65ce126afb856/public/get-pip.py" $py3getpipsha256 = "6fb7b781206356f45ad79efbb19322caa6c2a5ad39092d0d44d0fec94117e118" diff --git a/windows/helpers/phase2/install_python.ps1 b/windows/helpers/phase2/install_python.ps1 index a8b3809e..c3570190 100644 --- a/windows/helpers/phase2/install_python.ps1 +++ b/windows/helpers/phase2/install_python.ps1 @@ -4,7 +4,7 @@ param ( ) -# https://www.python.org/ftp/python/3.9.1/python-3.9.1-amd64.exe +# https://www.python.org/ftp/python/X.Y.Z/python-X.Y.Z-amd64.exe $pyexe = "https://www.python.org/ftp/python/$($Version)/python-$($Version)-amd64.exe" $isInstalled, $isCurrent = Get-InstallUpgradeStatus -Component "Python" -Keyname "version" -TargetValue $Version diff --git a/windows/versions.ps1 b/windows/versions.ps1 index a859ff7d..5740d1a9 100644 --- a/windows/versions.ps1 +++ b/windows/versions.ps1 @@ -40,8 +40,6 @@ $SoftwareTable = @{ "NUGET_SHA256"="5c5b9c96165d3283b2cb9e5b65825d343e0e7139b9e70a250b4bb24c2285f3ba"; "WINGET_VERSION"="1.6.5.0"; "WINGET_SHA256"="2CCED75B1830246A78FF0E57F18133807F78BA484CCA2D369CBDDF490DDAC1AF"; - "EMBEDDED_PYTHON_2_VERSION"="2.7.17"; - "EMBEDDED_PYTHON_2_SHA256"="557ea6690c5927360656c003d3114b73adbd755b712a2911975dde813d6d7afb"; "EMBEDDED_PYTHON_3_VERSION"="3.12.6"; "EMBEDDED_PYTHON_3_SHA256"="045d20a659fe80041b6fd508b77f250b03330347d64f128b392b88e68897f5a0"; "EMBEDDED_PIP_VERSION"="20.3.4";