Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

GH-38295: [CI][R] Free up disk space for Azure Pipelines jobs #38302

Merged
merged 2 commits into from
Oct 17, 2023
+130 −132
Merged

GH-38295: [CI][R] Free up disk space for Azure Pipelines jobs #38302

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e8f0fa1
GH-38295: [CI][R] Free up disk space for Azure jobs
kou Oct 17, 2023
f1e067c
Remove GITHUB_ACTIONS check
kou Oct 17, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
154 changes: 76 additions & 78 deletions ci/scripts/util_free_space.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,81 +19,79 @@

set -eux

if [ "${GITHUB_ACTIONS}" = "true" ]; then
df -h
echo "::group::/usr/local/*"
du -hsc /usr/local/*
echo "::endgroup::"
# ~1GB
sudo rm -rf \
/usr/local/aws-cli \
/usr/local/aws-sam-cil \
/usr/local/julia* || :
echo "::group::/usr/local/bin/*"
du -hsc /usr/local/bin/*
echo "::endgroup::"
# ~1GB (From 1.2GB to 214MB)
sudo rm -rf \
/usr/local/bin/aliyun \
/usr/local/bin/aws \
/usr/local/bin/aws_completer \
/usr/local/bin/azcopy \
/usr/local/bin/bicep \
/usr/local/bin/cmake-gui \
/usr/local/bin/cpack \
/usr/local/bin/helm \
/usr/local/bin/hub \
/usr/local/bin/kubectl \
/usr/local/bin/minikube \
/usr/local/bin/node \
/usr/local/bin/packer \
/usr/local/bin/pulumi* \
/usr/local/bin/sam \
/usr/local/bin/stack \
/usr/local/bin/terraform || :
# 142M
sudo rm -rf /usr/local/bin/oc || : \
echo "::group::/usr/local/share/*"
du -hsc /usr/local/share/*
echo "::endgroup::"
# 506MB
sudo rm -rf /usr/local/share/chromium || :
# 1.3GB
sudo rm -rf /usr/local/share/powershell || :
echo "::group::/usr/local/lib/*"
du -hsc /usr/local/lib/*
echo "::endgroup::"
# 15GB
sudo rm -rf /usr/local/lib/android || :
# 341MB
sudo rm -rf /usr/local/lib/heroku || :
# 1.2GB
sudo rm -rf /usr/local/lib/node_modules || :
echo "::group::/opt/*"
du -hsc /opt/*
echo "::endgroup::"
# 679MB
sudo rm -rf /opt/az || :
echo "::group::/opt/microsoft/*"
du -hsc /opt/microsoft/*
echo "::endgroup::"
# 197MB
sudo rm -rf /opt/microsoft/powershell || :
echo "::group::/opt/hostedtoolcache/*"
du -hsc /opt/hostedtoolcache/*
echo "::endgroup::"
# 5.3GB
sudo rm -rf /opt/hostedtoolcache/CodeQL || :
# 1.4GB
sudo rm -rf /opt/hostedtoolcache/go || :
# 489MB
sudo rm -rf /opt/hostedtoolcache/PyPy || :
# 376MB
sudo rm -rf /opt/hostedtoolcache/node || :
# Remove Web browser packages
sudo apt purge -y \
firefox \
google-chrome-stable \
microsoft-edge-stable
df -h
fi
df -h
echo "::group::/usr/local/*"
du -hsc /usr/local/*
echo "::endgroup::"
# ~1GB
sudo rm -rf \
/usr/local/aws-cli \
/usr/local/aws-sam-cil \
/usr/local/julia* || :
echo "::group::/usr/local/bin/*"
du -hsc /usr/local/bin/*
echo "::endgroup::"
# ~1GB (From 1.2GB to 214MB)
sudo rm -rf \
/usr/local/bin/aliyun \
/usr/local/bin/aws \
/usr/local/bin/aws_completer \
/usr/local/bin/azcopy \
/usr/local/bin/bicep \
/usr/local/bin/cmake-gui \
/usr/local/bin/cpack \
/usr/local/bin/helm \
/usr/local/bin/hub \
/usr/local/bin/kubectl \
/usr/local/bin/minikube \
/usr/local/bin/node \
/usr/local/bin/packer \
/usr/local/bin/pulumi* \
/usr/local/bin/sam \
/usr/local/bin/stack \
/usr/local/bin/terraform || :
# 142M
sudo rm -rf /usr/local/bin/oc || : \
echo "::group::/usr/local/share/*"
du -hsc /usr/local/share/*
echo "::endgroup::"
# 506MB
sudo rm -rf /usr/local/share/chromium || :
# 1.3GB
sudo rm -rf /usr/local/share/powershell || :
echo "::group::/usr/local/lib/*"
du -hsc /usr/local/lib/*
echo "::endgroup::"
# 15GB
sudo rm -rf /usr/local/lib/android || :
# 341MB
sudo rm -rf /usr/local/lib/heroku || :
# 1.2GB
sudo rm -rf /usr/local/lib/node_modules || :
echo "::group::/opt/*"
du -hsc /opt/*
echo "::endgroup::"
# 679MB
sudo rm -rf /opt/az || :
echo "::group::/opt/microsoft/*"
du -hsc /opt/microsoft/*
echo "::endgroup::"
# 197MB
sudo rm -rf /opt/microsoft/powershell || :
echo "::group::/opt/hostedtoolcache/*"
du -hsc /opt/hostedtoolcache/*
echo "::endgroup::"
# 5.3GB
sudo rm -rf /opt/hostedtoolcache/CodeQL || :
# 1.4GB
sudo rm -rf /opt/hostedtoolcache/go || :
# 489MB
sudo rm -rf /opt/hostedtoolcache/PyPy || :
# 376MB
sudo rm -rf /opt/hostedtoolcache/node || :
# Remove Web browser packages
sudo apt purge -y \
firefox \
google-chrome-stable \
microsoft-edge-stable
df -h
5 changes: 5 additions & 0 deletions dev/tasks/macros.jinja
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,11 @@ on:
displayName: Clone arrow
{% endmacro %}

{%- macro azure_free_space() -%}
- script: arrow/ci/scripts/util_free_space.sh
displayName: Free up disk space
{% endmacro %}

{%- macro azure_upload_releases(pattern) -%}
- task: UsePythonVersion@0
inputs:
Expand Down
103 changes: 49 additions & 54 deletions dev/tasks/r/azure.linux.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,62 +16,57 @@
{% import 'macros.jinja' as macros with context %}

jobs:
- job: linux
pool:
vmImage: ubuntu-latest
timeoutInMinutes: 360
{% if env is defined %}
variables:
- job: linux
pool:
vmImage: ubuntu-latest
timeoutInMinutes: 360
{% if env is defined %}
variables:
{% for key, value in env.items() %}
{{ key }}: {{ value }}
{{ key }}: {{ value }}
{% endfor %}
{% endif %}
steps:
- script: |
set -ex
git clone --no-checkout {{ arrow.remote }} arrow
git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
git -C arrow checkout FETCH_HEAD
git -C arrow submodule update --init --recursive
displayName: Clone arrow
{% endif %}
steps:
{{ macros.azure_checkout_arrow() }}
{{ macros.azure_free_space() }}

- script: |
set -ex
docker -v
docker-compose -v
cd arrow
export R_ORG={{ r_org }}
export R_IMAGE={{ r_image }}
export R_TAG={{ r_tag }}
export DEVTOOLSET_VERSION={{ devtoolset_version|default("") }}
export R_CUSTOM_CCACHE={{ r_custom_ccache|default("false") }}
docker-compose pull --ignore-pull-failures r
docker-compose build r
displayName: Docker build
env:
{{ macros.azure_set_sccache_envvars()|indent(8) }}
- script: |
set -ex
docker -v
docker-compose -v
cd arrow
export R_ORG={{ r_org }}
export R_IMAGE={{ r_image }}
export R_TAG={{ r_tag }}
export DEVTOOLSET_VERSION={{ devtoolset_version|default("") }}
export R_CUSTOM_CCACHE={{ r_custom_ccache|default("false") }}
docker-compose pull --ignore-pull-failures r
docker-compose build r
displayName: Docker build
env:
{{ macros.azure_set_sccache_envvars()|indent(4) }}

- script: |
set -ex
cd arrow
export R_ORG={{ r_org }}
export R_IMAGE={{ r_image }}
export R_TAG={{ r_tag }}
export ARROW_R_DEV={{ not_cran|default("TRUE") }}
# Note that by default, ci/scripts/r_test.sh sets NOT_CRAN=true
# if ARROW_R_DEV=TRUE. Pass `-e NOT_CRAN=false` to turn that off.
docker-compose run {{ flags|default("") }} r
displayName: Docker run
env:
{{ macros.azure_set_sccache_envvars()|indent(8) }}
- script: |
set -ex
cd arrow
export R_ORG={{ r_org }}
export R_IMAGE={{ r_image }}
export R_TAG={{ r_tag }}
export ARROW_R_DEV={{ not_cran|default("TRUE") }}
# Note that by default, ci/scripts/r_test.sh sets NOT_CRAN=true
# if ARROW_R_DEV=TRUE. Pass `-e NOT_CRAN=false` to turn that off.
docker-compose run {{ flags|default("") }} r
displayName: Docker run
env:
{{ macros.azure_set_sccache_envvars()|indent(4) }}

- script: |
set -ex
cat arrow/r/check/arrow.Rcheck/00install.out
displayName: Dump install logs
condition: succeededOrFailed()
- script: |
set -ex
cat arrow/r/check/arrow.Rcheck/tests/testthat.Rout*
displayName: Dump test logs
condition: succeededOrFailed()
- script: |
set -ex
cat arrow/r/check/arrow.Rcheck/00install.out
displayName: Dump install logs
condition: succeededOrFailed()
- script: |
set -ex
cat arrow/r/check/arrow.Rcheck/tests/testthat.Rout*
displayName: Dump test logs
condition: succeededOrFailed()