From 6a5c91c712c269e674b10c552f7e57ff45fe0599 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sat, 15 May 2021 06:42:54 -0400 Subject: [PATCH 1/2] Remove old release scripts --- dev/release/.env.example | 48 - dev/release/.gitignore | 21 - dev/release/01-prepare-test.rb | 665 ------ dev/release/01-prepare.sh | 291 --- dev/release/02-source-test.rb | 146 -- dev/release/02-source.sh | 162 -- dev/release/03-binary-submit.sh | 46 - dev/release/04-binary-download.sh | 38 - dev/release/05-binary-upload.sh | 137 -- dev/release/README.md | 24 - dev/release/Rakefile | 37 - dev/release/VERIFY.md | 76 - dev/release/binary-common.sh | 86 - dev/release/binary-task.rb | 1909 ----------------- dev/release/binary/.dockerignore | 18 - dev/release/binary/Dockerfile | 68 - dev/release/binary/runner.sh | 36 - dev/release/check-rat-report.py | 59 - dev/release/download_rc_binaries.py | 173 -- dev/release/post-01-upload.sh | 71 - dev/release/post-02-binary.sh | 95 - dev/release/post-03-website.sh | 266 --- dev/release/post-04-ruby.sh | 55 - dev/release/post-05-js.sh | 48 - dev/release/post-06-csharp.sh | 59 - dev/release/post-07-rust.sh | 74 - dev/release/post-08-remove-rc.sh | 50 - dev/release/post-09-docs.sh | 68 - dev/release/post-10-python.sh | 44 - dev/release/post-11-java.sh | 69 - dev/release/rat_exclude_files.txt | 107 - dev/release/run-rat.sh | 43 - dev/release/run-test.rb | 31 - dev/release/setup-gpg-agent.sh | 24 - dev/release/test-helper.rb | 96 - dev/release/verify-apt.sh | 150 -- .../verify-release-candidate-wheels.bat | 107 - dev/release/verify-release-candidate.bat | 130 -- dev/release/verify-release-candidate.sh | 808 ------- dev/release/verify-yum.sh | 154 -- dev/tasks/README.md | 19 - ...ion10.2numpy1.17python3.6.____cpython.yaml | 70 - ...ion10.2numpy1.17python3.7.____cpython.yaml | 70 - ...ion10.2numpy1.17python3.8.____cpython.yaml | 70 - ...ion10.2numpy1.19python3.9.____cpython.yaml | 70 - ...ionNonenumpy1.17python3.6.____cpython.yaml | 70 - ...ionNonenumpy1.17python3.7.____cpython.yaml | 70 - ...ionNonenumpy1.17python3.8.____cpython.yaml | 70 - ...ionNonenumpy1.19python3.9.____cpython.yaml | 70 - .../linux_aarch64_python3.6.____cpython.yaml | 71 - .../linux_aarch64_python3.7.____cpython.yaml | 71 - .../linux_aarch64_python3.8.____cpython.yaml | 71 - .../linux_aarch64_python3.9.____cpython.yaml | 71 - ...osx_64_numpy1.17python3.6.____cpython.yaml | 65 - ...osx_64_numpy1.17python3.7.____cpython.yaml | 65 - ...osx_64_numpy1.17python3.8.____cpython.yaml | 65 - ...osx_64_numpy1.19python3.9.____cpython.yaml | 65 - .../osx_arm64_python3.8.____cpython.yaml | 65 - .../osx_arm64_python3.9.____cpython.yaml | 65 - .../.ci_support/r/linux_64_r_base3.6.yaml | 22 - .../.ci_support/r/linux_64_r_base4.0.yaml | 22 - .../.ci_support/r/osx_64_r_base3.6.yaml | 26 - .../.ci_support/r/osx_64_r_base4.0.yaml | 26 - .../.ci_support/r/win_64_r_base3.6.yaml | 12 - .../.ci_support/r/win_64_r_base4.0.yaml | 12 - ...ionNonenumpy1.17python3.6.____cpython.yaml | 55 - ...ionNonenumpy1.17python3.7.____cpython.yaml | 55 - ...ionNonenumpy1.17python3.8.____cpython.yaml | 55 - ...ionNonenumpy1.19python3.9.____cpython.yaml | 55 - .../conda-recipes/.scripts/logging_utils.sh | 30 - dev/tasks/conda-recipes/README.md | 67 - .../conda-recipes/arrow-cpp/LLVM_LICENSE.txt | 68 - .../conda-recipes/arrow-cpp/bld-arrow.bat | 54 - .../conda-recipes/arrow-cpp/bld-pyarrow.bat | 44 - .../conda-recipes/arrow-cpp/build-arrow.sh | 95 - .../conda-recipes/arrow-cpp/build-pyarrow.sh | 49 - dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 302 --- dev/tasks/conda-recipes/azure.clean.yml | 28 - dev/tasks/conda-recipes/azure.linux.yml | 38 - dev/tasks/conda-recipes/azure.osx.yml | 80 - dev/tasks/conda-recipes/azure.win.yml | 77 - dev/tasks/conda-recipes/build_steps.sh | 55 - dev/tasks/conda-recipes/clean.py | 80 - dev/tasks/conda-recipes/conda-forge.yml | 1 - dev/tasks/conda-recipes/drone-steps.sh | 29 - dev/tasks/conda-recipes/drone.yml | 43 - dev/tasks/conda-recipes/parquet-cpp/meta.yaml | 51 - dev/tasks/conda-recipes/r-arrow/bld.bat | 9 - dev/tasks/conda-recipes/r-arrow/build.sh | 3 - dev/tasks/conda-recipes/r-arrow/build_win.sh | 8 - dev/tasks/conda-recipes/r-arrow/configure.win | 8 - .../conda-recipes/r-arrow/install.libs.R | 5 - dev/tasks/conda-recipes/r-arrow/meta.yaml | 66 - dev/tasks/conda-recipes/run_docker_build.sh | 77 - dev/tasks/cpp-examples/github.linux.yml | 46 - dev/tasks/docker-tests/azure.linux.yml | 52 - dev/tasks/docker-tests/circle.linux.yml | 51 - dev/tasks/docker-tests/github.linux.yml | 42 - dev/tasks/gandiva-jars/README.md | 29 - dev/tasks/gandiva-jars/build-cpp-linux.sh | 73 - dev/tasks/gandiva-jars/build-cpp-osx.sh | 49 - dev/tasks/gandiva-jars/build-java.sh | 64 - dev/tasks/gandiva-jars/github.linux.yml | 47 - dev/tasks/gandiva-jars/github.osx.yml | 46 - dev/tasks/homebrew-formulae/apache-arrow.rb | 69 - .../autobrew/apache-arrow.rb | 88 - dev/tasks/homebrew-formulae/github.macos.yml | 56 - dev/tasks/linux-packages/.gitignore | 28 - dev/tasks/linux-packages/README.md | 40 - dev/tasks/linux-packages/Rakefile | 234 -- .../apache-arrow-apt-source/Rakefile | 64 - .../apt/debian-bullseye/Dockerfile | 40 - .../apt/debian-buster/Dockerfile | 41 - .../apt/ubuntu-bionic/Dockerfile | 41 - .../apt/ubuntu-focal/Dockerfile | 41 - .../apt/ubuntu-groovy/Dockerfile | 41 - .../apt/ubuntu-xenial/Dockerfile | 41 - .../debian/apache-arrow-apt-source.install | 2 - .../apache-arrow-apt-source/debian/changelog | 0 .../apache-arrow-apt-source/debian/compat | 1 - .../apache-arrow-apt-source/debian/control | 23 - .../apache-arrow-apt-source/debian/copyright | 26 - .../apache-arrow-apt-source/debian/rules | 37 - .../debian/source/format | 1 - .../apache-arrow-release/Rakefile | 66 - .../yum/Apache-Arrow.repo | 37 - .../yum/apache-arrow-release.spec.in | 110 - .../yum/centos-7/Dockerfile | 28 - .../yum/centos-8/Dockerfile | 28 - .../linux-packages/apache-arrow/Rakefile | 120 -- .../apt/debian-bullseye-arm64/from | 18 - .../apt/debian-bullseye/Dockerfile | 81 - .../apt/debian-bullseye/qemu-dummy-static | 33 - .../apache-arrow/apt/debian-buster-arm64/from | 18 - .../apache-arrow/apt/debian-buster/Dockerfile | 85 - .../apt/debian-buster/qemu-dummy-static | 33 - .../apache-arrow/apt/ubuntu-bionic-arm64/from | 18 - .../apache-arrow/apt/ubuntu-bionic/Dockerfile | 88 - .../apt/ubuntu-bionic/qemu-dummy-static | 33 - .../apache-arrow/apt/ubuntu-focal-arm64/from | 18 - .../apache-arrow/apt/ubuntu-focal/Dockerfile | 78 - .../apt/ubuntu-focal/qemu-dummy-static | 33 - .../apache-arrow/apt/ubuntu-groovy-arm64/from | 18 - .../apache-arrow/apt/ubuntu-groovy/Dockerfile | 79 - .../apt/ubuntu-groovy/qemu-dummy-static | 33 - .../apache-arrow/debian/changelog | 111 - .../linux-packages/apache-arrow/debian/compat | 1 - .../apache-arrow/debian/control.in | 583 ----- .../apache-arrow/debian/copyright | 193 -- .../debian/gir1.2-arrow-1.0.install | 1 - .../debian/gir1.2-arrow-cuda-1.0.install | 1 - .../debian/gir1.2-arrow-dataset-1.0.install | 1 - .../debian/gir1.2-gandiva-1.0.install | 1 - .../debian/gir1.2-parquet-1.0.install | 1 - .../debian/gir1.2-plasma-1.0.install | 1 - .../debian/libarrow-cuda-dev.install | 6 - .../debian/libarrow-cuda-glib-dev.install | 4 - .../debian/libarrow-cuda-glib400.install | 1 - .../debian/libarrow-cuda400.install | 1 - .../debian/libarrow-dataset-dev.install | 6 - .../debian/libarrow-dataset-glib-dev.install | 4 - .../debian/libarrow-dataset-glib-doc.doc-base | 9 - .../debian/libarrow-dataset-glib-doc.install | 1 - .../debian/libarrow-dataset-glib-doc.links | 3 - .../debian/libarrow-dataset-glib400.install | 1 - .../debian/libarrow-dataset400.install | 1 - .../apache-arrow/debian/libarrow-dev.install | 21 - .../debian/libarrow-flight-dev.install | 6 - .../debian/libarrow-flight400.install | 1 - .../debian/libarrow-glib-dev.install | 6 - .../debian/libarrow-glib-doc.doc-base | 9 - .../debian/libarrow-glib-doc.install | 2 - .../debian/libarrow-glib-doc.links | 3 - .../debian/libarrow-glib400.install | 1 - .../debian/libarrow-python-dev.install | 6 - .../debian/libarrow-python-flight-dev.install | 6 - .../debian/libarrow-python-flight400.install | 1 - .../debian/libarrow-python400.install | 1 - .../apache-arrow/debian/libarrow400.install | 1 - .../debian/libgandiva-dev.install | 7 - .../debian/libgandiva-glib-dev.install | 4 - .../debian/libgandiva-glib-doc.doc-base | 9 - .../debian/libgandiva-glib-doc.install | 1 - .../debian/libgandiva-glib-doc.links | 3 - .../debian/libgandiva-glib400.install | 1 - .../apache-arrow/debian/libgandiva400.install | 1 - .../debian/libparquet-dev.install | 7 - .../debian/libparquet-glib-dev.install | 4 - .../debian/libparquet-glib-doc.doc-base | 9 - .../debian/libparquet-glib-doc.install | 1 - .../debian/libparquet-glib-doc.links | 3 - .../debian/libparquet-glib400.install | 1 - .../apache-arrow/debian/libparquet400.install | 1 - .../apache-arrow/debian/libplasma-dev.install | 7 - .../debian/libplasma-glib-dev.install | 4 - .../debian/libplasma-glib-doc.doc-base | 9 - .../debian/libplasma-glib-doc.install | 1 - .../debian/libplasma-glib-doc.links | 3 - .../debian/libplasma-glib400.install | 1 - .../apache-arrow/debian/libplasma400.install | 1 - .../apache-arrow/debian/patches/series | 0 .../debian/plasma-store-server.install | 1 - .../linux-packages/apache-arrow/debian/rules | 103 - .../apache-arrow/debian/source/format | 1 - .../linux-packages/apache-arrow/debian/watch | 2 - .../apache-arrow/yum/arrow.spec.in | 802 ------- .../apache-arrow/yum/centos-7/Dockerfile | 62 - .../yum/centos-7/qemu-dummy-static | 33 - .../apache-arrow/yum/centos-8-aarch64/from | 18 - .../apache-arrow/yum/centos-8/Dockerfile | 65 - .../yum/centos-8/qemu-dummy-static | 33 - dev/tasks/linux-packages/apt/build.sh | 115 - .../linux-packages/github.linux.amd64.yml | 100 - dev/tasks/linux-packages/helper.rb | 70 - dev/tasks/linux-packages/package-task.rb | 622 ------ .../linux-packages/travis.linux.arm64.yml | 149 -- dev/tasks/linux-packages/yum/build.sh | 157 -- dev/tasks/macros.jinja | 198 -- dev/tasks/nightlies.sample.yml | 68 - dev/tasks/nuget-packages/github.linux.yml | 43 - dev/tasks/python-sdist/github.yml | 45 - .../python-wheels/github.linux.amd64.yml | 48 - dev/tasks/python-wheels/github.osx.yml | 133 -- dev/tasks/python-wheels/github.windows.yml | 53 - .../python-wheels/travis.linux.arm64.yml | 73 - dev/tasks/r/azure.linux.yml | 74 - dev/tasks/r/github.devdocs.yml | 92 - dev/tasks/r/github.linux.cran.yml | 79 - .../r/github.linux.version.compatibility.yml | 109 - dev/tasks/r/github.linux.versions.yml | 80 - dev/tasks/r/github.macos-linux.local.yml | 88 - dev/tasks/r/github.macos.autobrew.yml | 78 - dev/tasks/tasks.yml | 1703 --------------- dev/tasks/vcpkg-tests/cpp-build-vcpkg.bat | 90 - dev/tasks/vcpkg-tests/github.windows.yml | 63 - dev/tasks/verify-rc/github.linux.yml | 75 - dev/tasks/verify-rc/github.macos.yml | 50 - dev/tasks/verify-rc/github.win.yml | 45 - 238 files changed, 18710 deletions(-) delete mode 100644 dev/release/.env.example delete mode 100644 dev/release/.gitignore delete mode 100644 dev/release/01-prepare-test.rb delete mode 100755 dev/release/01-prepare.sh delete mode 100644 dev/release/02-source-test.rb delete mode 100755 dev/release/02-source.sh delete mode 100755 dev/release/03-binary-submit.sh delete mode 100755 dev/release/04-binary-download.sh delete mode 100755 dev/release/05-binary-upload.sh delete mode 100644 dev/release/README.md delete mode 100644 dev/release/Rakefile delete mode 100644 dev/release/VERIFY.md delete mode 100644 dev/release/binary-common.sh delete mode 100644 dev/release/binary-task.rb delete mode 100644 dev/release/binary/.dockerignore delete mode 100644 dev/release/binary/Dockerfile delete mode 100755 dev/release/binary/runner.sh delete mode 100644 dev/release/check-rat-report.py delete mode 100755 dev/release/download_rc_binaries.py delete mode 100755 dev/release/post-01-upload.sh delete mode 100755 dev/release/post-02-binary.sh delete mode 100755 dev/release/post-03-website.sh delete mode 100755 dev/release/post-04-ruby.sh delete mode 100755 dev/release/post-05-js.sh delete mode 100755 dev/release/post-06-csharp.sh delete mode 100755 dev/release/post-07-rust.sh delete mode 100755 dev/release/post-08-remove-rc.sh delete mode 100755 dev/release/post-09-docs.sh delete mode 100755 dev/release/post-10-python.sh delete mode 100755 dev/release/post-11-java.sh delete mode 100644 dev/release/rat_exclude_files.txt delete mode 100755 dev/release/run-rat.sh delete mode 100755 dev/release/run-test.rb delete mode 100644 dev/release/setup-gpg-agent.sh delete mode 100644 dev/release/test-helper.rb delete mode 100755 dev/release/verify-apt.sh delete mode 100644 dev/release/verify-release-candidate-wheels.bat delete mode 100644 dev/release/verify-release-candidate.bat delete mode 100755 dev/release/verify-release-candidate.sh delete mode 100755 dev/release/verify-yum.sh delete mode 100644 dev/tasks/README.md delete mode 100644 dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.6.____cpython.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.7.____cpython.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.8.____cpython.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.19python3.9.____cpython.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.6.____cpython.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.7.____cpython.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.8.____cpython.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.6.____cpython.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.7.____cpython.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.8.____cpython.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.9.____cpython.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.6.____cpython.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.7.____cpython.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.8.____cpython.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.19python3.9.____cpython.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/osx_arm64_python3.8.____cpython.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/osx_arm64_python3.9.____cpython.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base3.6.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.0.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base3.6.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.0.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/r/win_64_r_base3.6.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/r/win_64_r_base4.0.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.6.____cpython.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.7.____cpython.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.8.____cpython.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython.yaml delete mode 100644 dev/tasks/conda-recipes/.scripts/logging_utils.sh delete mode 100644 dev/tasks/conda-recipes/README.md delete mode 100644 dev/tasks/conda-recipes/arrow-cpp/LLVM_LICENSE.txt delete mode 100644 dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat delete mode 100644 dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat delete mode 100644 dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh delete mode 100644 dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh delete mode 100644 dev/tasks/conda-recipes/arrow-cpp/meta.yaml delete mode 100644 dev/tasks/conda-recipes/azure.clean.yml delete mode 100755 dev/tasks/conda-recipes/azure.linux.yml delete mode 100755 dev/tasks/conda-recipes/azure.osx.yml delete mode 100755 dev/tasks/conda-recipes/azure.win.yml delete mode 100755 dev/tasks/conda-recipes/build_steps.sh delete mode 100644 dev/tasks/conda-recipes/clean.py delete mode 100644 dev/tasks/conda-recipes/conda-forge.yml delete mode 100755 dev/tasks/conda-recipes/drone-steps.sh delete mode 100644 dev/tasks/conda-recipes/drone.yml delete mode 100644 dev/tasks/conda-recipes/parquet-cpp/meta.yaml delete mode 100644 dev/tasks/conda-recipes/r-arrow/bld.bat delete mode 100644 dev/tasks/conda-recipes/r-arrow/build.sh delete mode 100755 dev/tasks/conda-recipes/r-arrow/build_win.sh delete mode 100755 dev/tasks/conda-recipes/r-arrow/configure.win delete mode 100644 dev/tasks/conda-recipes/r-arrow/install.libs.R delete mode 100644 dev/tasks/conda-recipes/r-arrow/meta.yaml delete mode 100755 dev/tasks/conda-recipes/run_docker_build.sh delete mode 100644 dev/tasks/cpp-examples/github.linux.yml delete mode 100644 dev/tasks/docker-tests/azure.linux.yml delete mode 100644 dev/tasks/docker-tests/circle.linux.yml delete mode 100644 dev/tasks/docker-tests/github.linux.yml delete mode 100644 dev/tasks/gandiva-jars/README.md delete mode 100755 dev/tasks/gandiva-jars/build-cpp-linux.sh delete mode 100755 dev/tasks/gandiva-jars/build-cpp-osx.sh delete mode 100755 dev/tasks/gandiva-jars/build-java.sh delete mode 100644 dev/tasks/gandiva-jars/github.linux.yml delete mode 100644 dev/tasks/gandiva-jars/github.osx.yml delete mode 100644 dev/tasks/homebrew-formulae/apache-arrow.rb delete mode 100644 dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb delete mode 100644 dev/tasks/homebrew-formulae/github.macos.yml delete mode 100644 dev/tasks/linux-packages/.gitignore delete mode 100644 dev/tasks/linux-packages/README.md delete mode 100644 dev/tasks/linux-packages/Rakefile delete mode 100644 dev/tasks/linux-packages/apache-arrow-apt-source/Rakefile delete mode 100644 dev/tasks/linux-packages/apache-arrow-apt-source/apt/debian-bullseye/Dockerfile delete mode 100644 dev/tasks/linux-packages/apache-arrow-apt-source/apt/debian-buster/Dockerfile delete mode 100644 dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-bionic/Dockerfile delete mode 100644 dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-focal/Dockerfile delete mode 100644 dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-groovy/Dockerfile delete mode 100644 dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-xenial/Dockerfile delete mode 100644 dev/tasks/linux-packages/apache-arrow-apt-source/debian/apache-arrow-apt-source.install delete mode 100644 dev/tasks/linux-packages/apache-arrow-apt-source/debian/changelog delete mode 100644 dev/tasks/linux-packages/apache-arrow-apt-source/debian/compat delete mode 100644 dev/tasks/linux-packages/apache-arrow-apt-source/debian/control delete mode 100644 dev/tasks/linux-packages/apache-arrow-apt-source/debian/copyright delete mode 100755 dev/tasks/linux-packages/apache-arrow-apt-source/debian/rules delete mode 100644 dev/tasks/linux-packages/apache-arrow-apt-source/debian/source/format delete mode 100644 dev/tasks/linux-packages/apache-arrow-release/Rakefile delete mode 100644 dev/tasks/linux-packages/apache-arrow-release/yum/Apache-Arrow.repo delete mode 100644 dev/tasks/linux-packages/apache-arrow-release/yum/apache-arrow-release.spec.in delete mode 100644 dev/tasks/linux-packages/apache-arrow-release/yum/centos-7/Dockerfile delete mode 100644 dev/tasks/linux-packages/apache-arrow-release/yum/centos-8/Dockerfile delete mode 100644 dev/tasks/linux-packages/apache-arrow/Rakefile delete mode 100644 dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye-arm64/from delete mode 100644 dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye/Dockerfile delete mode 100755 dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye/qemu-dummy-static delete mode 100644 dev/tasks/linux-packages/apache-arrow/apt/debian-buster-arm64/from delete mode 100644 dev/tasks/linux-packages/apache-arrow/apt/debian-buster/Dockerfile delete mode 100755 dev/tasks/linux-packages/apache-arrow/apt/debian-buster/qemu-dummy-static delete mode 100644 dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic-arm64/from delete mode 100644 dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile delete mode 100755 dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/qemu-dummy-static delete mode 100644 dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal-arm64/from delete mode 100644 dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile delete mode 100755 dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/qemu-dummy-static delete mode 100644 dev/tasks/linux-packages/apache-arrow/apt/ubuntu-groovy-arm64/from delete mode 100644 dev/tasks/linux-packages/apache-arrow/apt/ubuntu-groovy/Dockerfile delete mode 100755 dev/tasks/linux-packages/apache-arrow/apt/ubuntu-groovy/qemu-dummy-static delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/changelog delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/compat delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/control.in delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/copyright delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-1.0.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-cuda-1.0.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-dataset-1.0.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/gir1.2-gandiva-1.0.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/gir1.2-parquet-1.0.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/gir1.2-plasma-1.0.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-dev.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib-dev.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib400.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda400.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-dev.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-dev.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.doc-base delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.links delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib400.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset400.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-dev.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight400.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-dev.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.doc-base delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.links delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib400.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-dev.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight-dev.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight400.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libarrow-python400.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libarrow400.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libgandiva-dev.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-dev.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.doc-base delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.links delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib400.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libgandiva400.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libparquet-dev.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-dev.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.doc-base delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.links delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib400.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libparquet400.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libplasma-dev.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-dev.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.doc-base delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.links delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib400.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libplasma400.install delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/patches/series delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/plasma-store-server.install delete mode 100755 dev/tasks/linux-packages/apache-arrow/debian/rules delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/source/format delete mode 100644 dev/tasks/linux-packages/apache-arrow/debian/watch delete mode 100644 dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in delete mode 100644 dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile delete mode 100755 dev/tasks/linux-packages/apache-arrow/yum/centos-7/qemu-dummy-static delete mode 100644 dev/tasks/linux-packages/apache-arrow/yum/centos-8-aarch64/from delete mode 100644 dev/tasks/linux-packages/apache-arrow/yum/centos-8/Dockerfile delete mode 100755 dev/tasks/linux-packages/apache-arrow/yum/centos-8/qemu-dummy-static delete mode 100755 dev/tasks/linux-packages/apt/build.sh delete mode 100644 dev/tasks/linux-packages/github.linux.amd64.yml delete mode 100644 dev/tasks/linux-packages/helper.rb delete mode 100644 dev/tasks/linux-packages/package-task.rb delete mode 100644 dev/tasks/linux-packages/travis.linux.arm64.yml delete mode 100755 dev/tasks/linux-packages/yum/build.sh delete mode 100644 dev/tasks/macros.jinja delete mode 100644 dev/tasks/nightlies.sample.yml delete mode 100644 dev/tasks/nuget-packages/github.linux.yml delete mode 100644 dev/tasks/python-sdist/github.yml delete mode 100644 dev/tasks/python-wheels/github.linux.amd64.yml delete mode 100644 dev/tasks/python-wheels/github.osx.yml delete mode 100644 dev/tasks/python-wheels/github.windows.yml delete mode 100644 dev/tasks/python-wheels/travis.linux.arm64.yml delete mode 100644 dev/tasks/r/azure.linux.yml delete mode 100644 dev/tasks/r/github.devdocs.yml delete mode 100644 dev/tasks/r/github.linux.cran.yml delete mode 100644 dev/tasks/r/github.linux.version.compatibility.yml delete mode 100644 dev/tasks/r/github.linux.versions.yml delete mode 100644 dev/tasks/r/github.macos-linux.local.yml delete mode 100644 dev/tasks/r/github.macos.autobrew.yml delete mode 100644 dev/tasks/tasks.yml delete mode 100644 dev/tasks/vcpkg-tests/cpp-build-vcpkg.bat delete mode 100644 dev/tasks/vcpkg-tests/github.windows.yml delete mode 100644 dev/tasks/verify-rc/github.linux.yml delete mode 100644 dev/tasks/verify-rc/github.macos.yml delete mode 100644 dev/tasks/verify-rc/github.win.yml diff --git a/dev/release/.env.example b/dev/release/.env.example deleted file mode 100644 index 0126cdd3f298..000000000000 --- a/dev/release/.env.example +++ /dev/null @@ -1,48 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# The GPG key ID to sign artifacts. The GPG key ID must be registered -# to both of the followings: -# -# * https://dist.apache.org/repos/dist/dev/arrow/KEYS -# * https://dist.apache.org/repos/dist/release/arrow/KEYS -# -# See these files how to import your GPG key ID to these files. -# -# You must set this. -#GPG_KEY_ID=08D3564B7C6A9CAFBFF6A66791D18FCF079F8007 - -# The Bintray repository where artifacts are uploaded. -# You can use your Bintray repository such as kou/arrow for test. -BINTRAY_REPOSITORY=apache/arrow - -# The Bintray repository where released artifacts exist. -# The released artifacts are used to build APT/Yum repository. -# The Bintray repository isn't changed. (Download only. No upload.) -# -# Normally, you don't need to change this. -SOURCE_BINTRAY_REPOSITORY=apache/arrow - -# The Bintray user name to upload artifacts to Bintray. -# -# You must set this. -#BINTRAY_USER=kou - -# The Bintray API key to upload artifacts to Bintray. -# -# You must set this. -#BINTRAY_API_KEY=secret diff --git a/dev/release/.gitignore b/dev/release/.gitignore deleted file mode 100644 index f3d708a6a2cc..000000000000 --- a/dev/release/.gitignore +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -/.env -/binary/id_rsa -/binary/id_rsa.pub -/binary/tmp/ diff --git a/dev/release/01-prepare-test.rb b/dev/release/01-prepare-test.rb deleted file mode 100644 index b316ad20a9d8..000000000000 --- a/dev/release/01-prepare-test.rb +++ /dev/null @@ -1,665 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -class PrepareTest < Test::Unit::TestCase - include GitRunnable - include VersionDetectable - - def setup - @current_commit = git_current_commit - detect_versions - - top_dir = Pathname(__dir__).parent.parent - @original_git_repository = top_dir + ".git" - Dir.mktmpdir do |dir| - @test_git_repository = Pathname(dir) + "arrow" - git("clone", @original_git_repository.to_s, @test_git_repository.to_s) - Dir.chdir(@test_git_repository) do - @tag_name = "apache-arrow-#{@release_version}" - @release_branch = "release-#{@release_version}-rc0" - @script = "dev/release/01-prepare.sh" - git("checkout", "-b", @release_branch, @current_commit) - yield - end - FileUtils.rm_rf(@test_git_repository) - end - end - - def omit_on_release_branch - omit("Not for release branch") if on_release_branch? - end - - def prepare(*targets) - if targets.last.is_a?(Hash) - additional_env = targets.pop - else - additional_env = {} - end - env = {"PREPARE_DEFAULT" => "0"} - targets.each do |target| - env["PREPARE_#{target}"] = "1" - end - env = env.merge(additional_env) - sh(env, @script, @release_version, @next_version, "0") - end - - def parse_patch(patch) - diffs = [] - in_hunk = false - patch.each_line do |line| - case line - when /\A--- a\// - path = $POSTMATCH.chomp - diffs << {path: path, hunks: []} - in_hunk = false - when /\A@@/ - in_hunk = true - diffs.last[:hunks] << [] - when /\A[-+]/ - next unless in_hunk - diffs.last[:hunks].last << line.chomp - end - end - diffs.sort_by do |diff| - diff[:path] - end - end - - def test_linux_packages - user = "Arrow Developers" - email = "dev@arrow.apache.org" - prepare("LINUX_PACKAGES", - "DEBFULLNAME" => user, - "DEBEMAIL" => email) - changes = parse_patch(git("log", "-n", "1", "-p")) - sampled_changes = changes.collect do |change| - { - path: change[:path], - sampled_hunks: change[:hunks].collect(&:first), - # sampled_hunks: change[:hunks], - } - end - base_dir = "dev/tasks/linux-packages" - today = Time.now.utc.strftime("%a %b %d %Y") - expected_changes = [ - { - path: "#{base_dir}/apache-arrow-apt-source/debian/changelog", - sampled_hunks: [ - "+apache-arrow-apt-source (#{@release_version}-1) " + - "unstable; urgency=low", - ], - }, - { - path: - "#{base_dir}/apache-arrow-release/yum/apache-arrow-release.spec.in", - sampled_hunks: [ - "+* #{today} #{user} <#{email}> - #{@release_version}-1", - ], - }, - { - path: "#{base_dir}/apache-arrow/debian/changelog", - sampled_hunks: [ - "+apache-arrow (#{@release_version}-1) unstable; urgency=low", - ], - }, - { - path: "#{base_dir}/apache-arrow/yum/arrow.spec.in", - sampled_hunks: [ - "+* #{today} #{user} <#{email}> - #{@release_version}-1", - ], - }, - ] - assert_equal(expected_changes, sampled_changes) - end - - def test_version_pre_tag - omit_on_release_branch - prepare("VERSION_PRE_TAG") - assert_equal([ - { - path: "c_glib/meson.build", - hunks: [ - ["-version = '#{@snapshot_version}'", - "+version = '#{@release_version}'"], - ], - }, - { - path: "ci/scripts/PKGBUILD", - hunks: [ - ["-pkgver=#{@previous_version}.9000", - "+pkgver=#{@release_version}"], - ], - }, - { - path: "cpp/CMakeLists.txt", - hunks: [ - ["-set(ARROW_VERSION \"#{@snapshot_version}\")", - "+set(ARROW_VERSION \"#{@release_version}\")"], - ], - }, - { - path: "cpp/vcpkg.json", - hunks: [ - ["- \"version-string\": \"#{@snapshot_version}\",", - "+ \"version-string\": \"#{@release_version}\","], - ], - }, - { - path: "csharp/Directory.Build.props", - hunks: [ - ["- #{@snapshot_version}", - "+ #{@release_version}"], - ], - }, - { - path: "dev/tasks/homebrew-formulae/apache-arrow.rb", - hunks: [ - ["- url \"https://www.apache.org/dyn/closer.lua?path=arrow/arrow-#{@snapshot_version}/apache-arrow-#{@snapshot_version}.tar.gz\"", - "+ url \"https://www.apache.org/dyn/closer.lua?path=arrow/arrow-#{@release_version}/apache-arrow-#{@release_version}.tar.gz\""], - ], - }, - { - path: "dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb", - hunks: [ - ["- url \"https://www.apache.org/dyn/closer.lua?path=arrow/arrow-#{@previous_version}.9000/apache-arrow-#{@previous_version}.9000.tar.gz\"", - "+ url \"https://www.apache.org/dyn/closer.lua?path=arrow/arrow-#{@release_version}/apache-arrow-#{@release_version}.tar.gz\""], - ], - }, - { - path: "js/package.json", - hunks: [ - ["- \"version\": \"#{@snapshot_version}\"", - "+ \"version\": \"#{@release_version}\""] - ], - }, - { - path: "matlab/CMakeLists.txt", - hunks: [ - ["-set(MLARROW_VERSION \"#{@snapshot_version}\")", - "+set(MLARROW_VERSION \"#{@release_version}\")"], - ], - }, - { - path: "python/setup.py", - hunks: [ - ["-default_version = '#{@snapshot_version}'", - "+default_version = '#{@release_version}'"], - ], - }, - { - path: "r/DESCRIPTION", - hunks: [ - ["-Version: #{@previous_version}.9000", - "+Version: #{@release_version}"], - ], - }, - { - path: "r/NEWS.md", - hunks: [ - ["-\# arrow #{@previous_version}.9000", - "+\# arrow #{@release_version}"], - ], - }, - { - path: "ruby/red-arrow-cuda/lib/arrow-cuda/version.rb", - hunks: [ - ["- VERSION = \"#{@snapshot_version}\"", - "+ VERSION = \"#{@release_version}\""], - ], - }, - { - path: "ruby/red-arrow-dataset/lib/arrow-dataset/version.rb", - hunks: [ - ["- VERSION = \"#{@snapshot_version}\"", - "+ VERSION = \"#{@release_version}\""], - ], - }, - { - path: "ruby/red-arrow/lib/arrow/version.rb", - hunks: [ - ["- VERSION = \"#{@snapshot_version}\"", - "+ VERSION = \"#{@release_version}\""], - ], - }, - { - path: "ruby/red-gandiva/lib/gandiva/version.rb", - hunks: [ - ["- VERSION = \"#{@snapshot_version}\"", - "+ VERSION = \"#{@release_version}\""], - ], - }, - { - path: "ruby/red-parquet/lib/parquet/version.rb", - hunks: [ - ["- VERSION = \"#{@snapshot_version}\"", - "+ VERSION = \"#{@release_version}\""], - ], - }, - { - path: "ruby/red-plasma/lib/plasma/version.rb", - hunks: [ - ["- VERSION = \"#{@snapshot_version}\"", - "+ VERSION = \"#{@release_version}\""], - ], - }, - { - path: "rust/arrow-flight/Cargo.toml", - hunks: [ - ["-version = \"#{@snapshot_version}\"", - "+version = \"#{@release_version}\""], - ["-arrow = { path = \"../arrow\", version = \"#{@snapshot_version}\" }", - "+arrow = { path = \"../arrow\", version = \"#{@release_version}\" }"], - ], - }, - { - path: "rust/arrow-pyarrow-integration-testing/Cargo.toml", - hunks: [ - ["-version = \"#{@snapshot_version}\"", - "+version = \"#{@release_version}\""], - ["-arrow = { path = \"../arrow\", version = \"#{@snapshot_version}\" }", - "+arrow = { path = \"../arrow\", version = \"#{@release_version}\" }"], - ], - }, - { - path: "rust/arrow/Cargo.toml", - hunks: [ - ["-version = \"#{@snapshot_version}\"", - "+version = \"#{@release_version}\""], - ], - }, - { - path: "rust/benchmarks/Cargo.toml", - hunks: [ - ["-version = \"#{@snapshot_version}\"", - "+version = \"#{@release_version}\""], - ], - }, - { - path: "rust/datafusion-examples/Cargo.toml", - hunks: [ - ["-version = \"#{@snapshot_version}\"", - "+version = \"#{@release_version}\""], - ], - }, - { - path: "rust/datafusion/Cargo.toml", - hunks: [ - ["-version = \"#{@snapshot_version}\"", - "+version = \"#{@release_version}\""], - ["-arrow = { path = \"../arrow\", version = \"#{@snapshot_version}\", features = [\"prettyprint\"] }", - "-parquet = { path = \"../parquet\", version = \"#{@snapshot_version}\", features = [\"arrow\"] }", - "+arrow = { path = \"../arrow\", version = \"#{@release_version}\", features = [\"prettyprint\"] }", - "+parquet = { path = \"../parquet\", version = \"#{@release_version}\", features = [\"arrow\"] }"], - ], - }, - { - path: "rust/datafusion/README.md", - hunks: [ - ["-datafusion = \"#{@snapshot_version}\"", - "+datafusion = \"#{@release_version}\""], - ], - }, - { - path: "rust/integration-testing/Cargo.toml", - hunks: [ - ["-version = \"#{@snapshot_version}\"", - "+version = \"#{@release_version}\""], - ], - }, - { - path: "rust/parquet/Cargo.toml", - hunks: [ - ["-version = \"#{@snapshot_version}\"", - "+version = \"#{@release_version}\""], - ["-arrow = { path = \"../arrow\", version = \"#{@snapshot_version}\", optional = true }", - "+arrow = { path = \"../arrow\", version = \"#{@release_version}\", optional = true }"], - ["-arrow = { path = \"../arrow\", version = \"#{@snapshot_version}\" }", - "+arrow = { path = \"../arrow\", version = \"#{@release_version}\" }"] - ], - }, - { - path: "rust/parquet/README.md", - hunks: [ - ["-parquet = \"#{@snapshot_version}\"", - "+parquet = \"#{@release_version}\""], - ["-See [crate documentation](https://docs.rs/crate/parquet/#{@snapshot_version}) on available API.", - "+See [crate documentation](https://docs.rs/crate/parquet/#{@release_version}) on available API."], - ], - }, - { - path: "rust/parquet_derive/Cargo.toml", - hunks: [ - ["-version = \"#{@snapshot_version}\"", - "+version = \"#{@release_version}\""], - ["-parquet = { path = \"../parquet\", version = \"#{@snapshot_version}\" }", - "+parquet = { path = \"../parquet\", version = \"#{@release_version}\" }"], - ], - }, - { - path: "rust/parquet_derive/README.md", - hunks: [ - ["-parquet = \"#{@snapshot_version}\"", - "-parquet_derive = \"#{@snapshot_version}\"", - "+parquet = \"#{@release_version}\"", - "+parquet_derive = \"#{@release_version}\""], - ], - }, - { - path: "rust/parquet_derive_test/Cargo.toml", - hunks: [ - ["-version = \"#{@snapshot_version}\"", - "+version = \"#{@release_version}\""], - ["-parquet = { path = \"../parquet\", version = \"#{@snapshot_version}\" }", - "-parquet_derive = { path = \"../parquet_derive\", version = \"#{@snapshot_version}\" }", - "+parquet = { path = \"../parquet\", version = \"#{@release_version}\" }", - "+parquet_derive = { path = \"../parquet_derive\", version = \"#{@release_version}\" }"], - ], - }, - ], - parse_patch(git("log", "-n", "1", "-p"))) - end - - def test_version_post_tag - if on_release_branch? - prepare("VERSION_POST_TAG") - else - prepare("VERSION_PRE_TAG", - "VERSION_POST_TAG") - end - assert_equal([ - { - path: "c_glib/meson.build", - hunks: [ - ["-version = '#{@release_version}'", - "+version = '#{@next_snapshot_version}'"], - ], - }, - { - path: "ci/scripts/PKGBUILD", - hunks: [ - ["-pkgver=#{@release_version}", - "+pkgver=#{@release_version}.9000"], - ], - }, - { - path: "cpp/CMakeLists.txt", - hunks: [ - ["-set(ARROW_VERSION \"#{@release_version}\")", - "+set(ARROW_VERSION \"#{@next_snapshot_version}\")"], - ], - }, - { - path: "cpp/vcpkg.json", - hunks: [ - ["- \"version-string\": \"#{@release_version}\",", - "+ \"version-string\": \"#{@next_snapshot_version}\","], - ], - }, - { - path: "csharp/Directory.Build.props", - hunks: [ - ["- #{@release_version}", - "+ #{@next_snapshot_version}"], - ], - }, - { - path: "dev/tasks/homebrew-formulae/apache-arrow.rb", - hunks: [ - ["- url \"https://www.apache.org/dyn/closer.lua?path=arrow/arrow-#{@release_version}/apache-arrow-#{@release_version}.tar.gz\"", - "+ url \"https://www.apache.org/dyn/closer.lua?path=arrow/arrow-#{@next_snapshot_version}/apache-arrow-#{@next_snapshot_version}.tar.gz\""], - ], - }, - { - path: "dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb", - hunks: [ - ["- url \"https://www.apache.org/dyn/closer.lua?path=arrow/arrow-#{@release_version}/apache-arrow-#{@release_version}.tar.gz\"", - "+ url \"https://www.apache.org/dyn/closer.lua?path=arrow/arrow-#{@release_version}.9000/apache-arrow-#{@release_version}.9000.tar.gz\""], - ], - }, - { - path: "js/package.json", - hunks: [ - ["- \"version\": \"#{@release_version}\"", - "+ \"version\": \"#{@next_snapshot_version}\""], - ], - }, - { - path: "matlab/CMakeLists.txt", - hunks: [ - ["-set(MLARROW_VERSION \"#{@release_version}\")", - "+set(MLARROW_VERSION \"#{@next_snapshot_version}\")"], - ], - }, - { - path: "python/setup.py", - hunks: [ - ["-default_version = '#{@release_version}'", - "+default_version = '#{@next_snapshot_version}'"], - ], - }, - { - path: "r/DESCRIPTION", - hunks: [ - ["-Version: #{@release_version}", - "+Version: #{@release_version}.9000"], - ], - }, - { - path: "r/NEWS.md", - # Note that these are additions only, no replacement - hunks: [ - ["+# arrow #{@release_version}.9000", - "+"], - ], - }, - { - path: "ruby/red-arrow-cuda/lib/arrow-cuda/version.rb", - hunks: [ - ["- VERSION = \"#{@release_version}\"", - "+ VERSION = \"#{@next_snapshot_version}\""], - ], - }, - { - path: "ruby/red-arrow-dataset/lib/arrow-dataset/version.rb", - hunks: [ - ["- VERSION = \"#{@release_version}\"", - "+ VERSION = \"#{@next_snapshot_version}\""], - ], - }, - { - path: "ruby/red-arrow/lib/arrow/version.rb", - hunks: [ - ["- VERSION = \"#{@release_version}\"", - "+ VERSION = \"#{@next_snapshot_version}\""], - ], - }, - { - path: "ruby/red-gandiva/lib/gandiva/version.rb", - hunks: [ - ["- VERSION = \"#{@release_version}\"", - "+ VERSION = \"#{@next_snapshot_version}\""], - ], - }, - { - path: "ruby/red-parquet/lib/parquet/version.rb", - hunks: [ - ["- VERSION = \"#{@release_version}\"", - "+ VERSION = \"#{@next_snapshot_version}\""], - ], - }, - { - path: "ruby/red-plasma/lib/plasma/version.rb", - hunks: [ - ["- VERSION = \"#{@release_version}\"", - "+ VERSION = \"#{@next_snapshot_version}\""], - ], - }, - { - path: "rust/arrow-flight/Cargo.toml", - hunks: [ - ["-version = \"#{@release_version}\"", - "+version = \"#{@next_snapshot_version}\""], - ["-arrow = { path = \"../arrow\", version = \"#{@release_version}\" }", - "+arrow = { path = \"../arrow\", version = \"#{@next_snapshot_version}\" }"], - ], - }, - { - path: "rust/arrow-pyarrow-integration-testing/Cargo.toml", - hunks: [ - ["-version = \"#{@release_version}\"", - "+version = \"#{@next_snapshot_version}\""], - ["-arrow = { path = \"../arrow\", version = \"#{@release_version}\" }", - "+arrow = { path = \"../arrow\", version = \"#{@next_snapshot_version}\" }"], - ], - }, - { - path: "rust/arrow/Cargo.toml", - hunks: [ - ["-version = \"#{@release_version}\"", - "+version = \"#{@next_snapshot_version}\""], - ], - }, - { - path: "rust/benchmarks/Cargo.toml", - hunks: [ - ["-version = \"#{@release_version}\"", - "+version = \"#{@next_snapshot_version}\""], - ], - }, - { - path: "rust/datafusion-examples/Cargo.toml", - hunks: [ - ["-version = \"#{@release_version}\"", - "+version = \"#{@next_snapshot_version}\""], - ], - }, - { - path: "rust/datafusion/Cargo.toml", - hunks: [ - ["-version = \"#{@release_version}\"", - "+version = \"#{@next_snapshot_version}\""], - ["-arrow = { path = \"../arrow\", version = \"#{@release_version}\", features = [\"prettyprint\"] }", - "-parquet = { path = \"../parquet\", version = \"#{@release_version}\", features = [\"arrow\"] }", - "+arrow = { path = \"../arrow\", version = \"#{@next_snapshot_version}\", features = [\"prettyprint\"] }", - "+parquet = { path = \"../parquet\", version = \"#{@next_snapshot_version}\", features = [\"arrow\"] }"], - ], - }, - { - path: "rust/datafusion/README.md", - hunks: [ - ["-datafusion = \"#{@release_version}\"", - "+datafusion = \"#{@next_snapshot_version}\""], - ], - }, - { - path: "rust/integration-testing/Cargo.toml", - hunks: [ - ["-version = \"#{@release_version}\"", - "+version = \"#{@next_snapshot_version}\""], - ], - }, - { - path: "rust/parquet/Cargo.toml", - hunks: [ - ["-version = \"#{@release_version}\"", - "+version = \"#{@next_snapshot_version}\""], - ["-arrow = { path = \"../arrow\", version = \"#{@release_version}\", optional = true }", - "+arrow = { path = \"../arrow\", version = \"#{@next_snapshot_version}\", optional = true }"], - ["-arrow = { path = \"../arrow\", version = \"#{@release_version}\" }", - "+arrow = { path = \"../arrow\", version = \"#{@next_snapshot_version}\" }"] - ], - }, - { - path: "rust/parquet/README.md", - hunks: [ - ["-parquet = \"#{@release_version}\"", - "+parquet = \"#{@next_snapshot_version}\""], - ["-See [crate documentation](https://docs.rs/crate/parquet/#{@release_version}) on available API.", - "+See [crate documentation](https://docs.rs/crate/parquet/#{@next_snapshot_version}) on available API."], - ], - }, - { - path: "rust/parquet_derive/Cargo.toml", - hunks: [ - ["-version = \"#{@release_version}\"", - "+version = \"#{@next_snapshot_version}\""], - ["-parquet = { path = \"../parquet\", version = \"#{@release_version}\" }", - "+parquet = { path = \"../parquet\", version = \"#{@next_snapshot_version}\" }"], - ], - }, - { - path: "rust/parquet_derive/README.md", - hunks: [ - ["-parquet = \"#{@release_version}\"", - "-parquet_derive = \"#{@release_version}\"", - "+parquet = \"#{@next_snapshot_version}\"", - "+parquet_derive = \"#{@next_snapshot_version}\""], - ], - }, - { - path: "rust/parquet_derive_test/Cargo.toml", - hunks: [ - ["-version = \"#{@release_version}\"", - "+version = \"#{@next_snapshot_version}\""], - ["-parquet = { path = \"../parquet\", version = \"#{@release_version}\" }", - "-parquet_derive = { path = \"../parquet_derive\", version = \"#{@release_version}\" }", - "+parquet = { path = \"../parquet\", version = \"#{@next_snapshot_version}\" }", - "+parquet_derive = { path = \"../parquet_derive\", version = \"#{@next_snapshot_version}\" }"], - ], - }, - ], - parse_patch(git("log", "-n", "1", "-p"))) - end - - def test_deb_package_names - prepare("DEB_PACKAGE_NAMES") - changes = parse_patch(git("log", "-n", "1", "-p")) - sampled_changes = changes.collect do |change| - first_hunk = change[:hunks][0] - first_removed_line = first_hunk.find {|line| line.start_with?("-")} - first_added_line = first_hunk.find {|line| line.start_with?("+")} - { - sampled_diff: [first_removed_line, first_added_line], - path: change[:path], - } - end - expected_changes = [ - { - sampled_diff: [ - "-dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib#{@so_version}.install", - "+dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib#{@next_so_version}.install", - ], - path: "dev/release/rat_exclude_files.txt" - }, - { - sampled_diff: [ - "-Package: libarrow#{@so_version}", - "+Package: libarrow#{@next_so_version}", - ], - path: "dev/tasks/linux-packages/apache-arrow/debian/control.in" - }, - { - sampled_diff: [ - "- - libarrow-glib#{@so_version}-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb", - "+ - libarrow-glib#{@next_so_version}-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb", - ], - path: "dev/tasks/tasks.yml", - }, - ] - assert_equal(expected_changes, sampled_changes) - end -end diff --git a/dev/release/01-prepare.sh b/dev/release/01-prepare.sh deleted file mode 100755 index 80703c2d87fc..000000000000 --- a/dev/release/01-prepare.sh +++ /dev/null @@ -1,291 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -set -ue - -SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -if [ "$#" -ne 3 ]; then - echo "Usage: $0 " - exit 1 -fi - -update_versions() { - local base_version=$1 - local next_version=$2 - local type=$3 - - case ${type} in - release) - local version=${base_version} - local r_version=${base_version} - ;; - snapshot) - local version=${next_version}-SNAPSHOT - local r_version=${base_version}.9000 - ;; - esac - - cd "${SOURCE_DIR}/../../c_glib" - sed -i.bak -E -e \ - "s/^version = '.+'/version = '${version}'/" \ - meson.build - rm -f meson.build.bak - git add meson.build - cd - - - cd "${SOURCE_DIR}/../../ci/scripts" - sed -i.bak -E -e \ - "s/^pkgver=.+/pkgver=${r_version}/" \ - PKGBUILD - rm -f PKGBUILD.bak - git add PKGBUILD - cd - - - cd "${SOURCE_DIR}/../../cpp" - sed -i.bak -E -e \ - "s/^set\(ARROW_VERSION \".+\"\)/set(ARROW_VERSION \"${version}\")/" \ - CMakeLists.txt - rm -f CMakeLists.txt.bak - git add CMakeLists.txt - - sed -i.bak -E -e \ - "s/\"version-string\": \".+\"/\"version-string\": \"${version}\"/" \ - vcpkg.json - rm -f vcpkg.json.bak - git add vcpkg.json - cd - - - cd "${SOURCE_DIR}/../../csharp" - sed -i.bak -E -e \ - "s/^ .+<\/Version>/ ${version}<\/Version>/" \ - Directory.Build.props - rm -f Directory.Build.props.bak - git add Directory.Build.props - cd - - - cd "${SOURCE_DIR}/../../dev/tasks/homebrew-formulae" - sed -i.bak -E -e \ - "s/arrow-[0-9.]+[0-9]+/arrow-${r_version}/g" \ - autobrew/apache-arrow.rb - rm -f autobrew/apache-arrow.rb.bak - git add autobrew/apache-arrow.rb - sed -i.bak -E -e \ - "s/arrow-[0-9.\-]+[0-9SNAPHOT]+/arrow-${version}/g" \ - apache-arrow.rb - rm -f apache-arrow.rb.bak - git add apache-arrow.rb - cd - - - cd "${SOURCE_DIR}/../../js" - sed -i.bak -E -e \ - "s/^ \"version\": \".+\"/ \"version\": \"${version}\"/" \ - package.json - rm -f package.json.bak - git add package.json - cd - - - cd "${SOURCE_DIR}/../../matlab" - sed -i.bak -E -e \ - "s/^set\(MLARROW_VERSION \".+\"\)/set(MLARROW_VERSION \"${version}\")/" \ - CMakeLists.txt - rm -f CMakeLists.txt.bak - git add CMakeLists.txt - cd - - - cd "${SOURCE_DIR}/../../python" - sed -i.bak -E -e \ - "s/^default_version = '.+'/default_version = '${version}'/" \ - setup.py - rm -f setup.py.bak - git add setup.py - cd - - - cd "${SOURCE_DIR}/../../r" - sed -i.bak -E -e \ - "s/^Version: .+/Version: ${r_version}/" \ - DESCRIPTION - rm -f DESCRIPTION.bak - git add DESCRIPTION - if [ ${type} = "snapshot" ]; then - # Add a news entry for the new dev version - echo "dev" - sed -i.bak -E -e \ - "0,/^# arrow /s/^(# arrow .+)/# arrow ${r_version}\n\n\1/" \ - NEWS.md - else - # Replace dev version with release version - echo "release" - sed -i.bak -E -e \ - "0,/^# arrow /s/^# arrow .+/# arrow ${r_version}/" \ - NEWS.md - fi - rm -f NEWS.md.bak - git add NEWS.md - cd - - - cd "${SOURCE_DIR}/../../ruby" - sed -i.bak -E -e \ - "s/^ VERSION = \".+\"/ VERSION = \"${version}\"/g" \ - */*/*/version.rb - rm -f */*/*/version.rb.bak - git add */*/*/version.rb - cd - - - cd "${SOURCE_DIR}/../../rust" - sed -i.bak -E \ - -e "s/^version = \".+\"/version = \"${version}\"/g" \ - -e "s/^(arrow = .* version = )\".*\"(( .*)|(, features = .*)|(, optional = .*))$/\\1\"${version}\"\\2/g" \ - -e "s/^(arrow-flight = .* version = )\".+\"( .*)/\\1\"${version}\"\\2/g" \ - -e "s/^(parquet = .* version = )\".*\"(( .*)|(, features = .*))$/\\1\"${version}\"\\2/g" \ - -e "s/^(parquet_derive = .* version = )\".*\"(( .*)|(, features = .*))$/\\1\"${version}\"\\2/g" \ - */Cargo.toml - rm -f */Cargo.toml.bak - git add */Cargo.toml - - sed -i.bak -E \ - -e "s/^([^ ]+) = \".+\"/\\1 = \"${version}\"/g" \ - -e "s,docs\.rs/crate/([^/]+)/[^)]+,docs.rs/crate/\\1/${version},g" \ - */README.md - rm -f */README.md.bak - git add */README.md - cd - -} - -############################## Pre-Tag Commits ############################## - -version=$1 -next_version=$2 -next_version_snapshot="${next_version}-SNAPSHOT" -rc_number=$3 - -release_tag="apache-arrow-${version}" -release_branch="release-${version}" -release_candidate_branch="release-${version}-rc${rc_number}" - -: ${PREPARE_DEFAULT:=1} -: ${PREPARE_CHANGELOG:=${PREPARE_DEFAULT}} -: ${PREPARE_LINUX_PACKAGES:=${PREPARE_DEFAULT}} -: ${PREPARE_VERSION_PRE_TAG:=${PREPARE_DEFAULT}} -: ${PREPARE_BRANCH:=${PREPARE_DEFAULT}} -: ${PREPARE_TAG:=${PREPARE_DEFAULT}} -: ${PREPARE_VERSION_POST_TAG:=${PREPARE_DEFAULT}} -: ${PREPARE_DEB_PACKAGE_NAMES:=${PREPARE_DEFAULT}} - -if [ ${PREPARE_TAG} -gt 0 ]; then - if [ $(git tag -l "${release_tag}") ]; then - echo "Delete existing git tag $release_tag" - git tag -d "${release_tag}" - fi -fi - -if [ ${PREPARE_BRANCH} -gt 0 ]; then - if [[ $(git branch -l "${release_candidate_branch}") ]]; then - next_rc_number=$(($rc_number+1)) - echo "Branch ${release_candidate_branch} already exists, so create a new release candidate:" - echo "1. Checkout the master branch for major releases and maint- for patch releases." - echo "2. Execute the script again with bumped RC number." - echo "Commands:" - echo " git checkout master" - echo " dev/release/01-prepare.sh ${version} ${next_version} ${next_rc_number}" - exit 1 - fi - - echo "Create local branch ${release_candidate_branch} for release candidate ${rc_number}" - git checkout -b ${release_candidate_branch} -fi - -############################## Pre-Tag Commits ############################## - -if [ ${PREPARE_CHANGELOG} -gt 0 ]; then - echo "Updating changelog for $version" - # Update changelog - archery release changelog add $version - git add ${SOURCE_DIR}/../../CHANGELOG.md - git commit -m "[Release] Update CHANGELOG.md for $version" -fi - -if [ ${PREPARE_LINUX_PACKAGES} -gt 0 ]; then - echo "Updating .deb/.rpm changelogs for $version" - cd $SOURCE_DIR/../tasks/linux-packages - rake \ - version:update \ - ARROW_RELEASE_TIME="$(date +%Y-%m-%dT%H:%M:%S%z)" \ - ARROW_VERSION=${version} - git add */debian*/changelog */yum/*.spec.in - git commit -m "[Release] Update .deb/.rpm changelogs for $version" - cd - -fi - -if [ ${PREPARE_VERSION_PRE_TAG} -gt 0 ]; then - echo "Prepare release ${version} on tag ${release_tag} then reset to version ${next_version_snapshot}" - - update_versions "${version}" "${next_version}" "release" - git commit -m "[Release] Update versions for ${version}" -fi - -############################## Tag the Release ############################## - -if [ ${PREPARE_TAG} -gt 0 ]; then - git tag -a "${release_tag}" -m "[Release] Apache Arrow Release ${version}" -fi - -############################## Post-Tag Commits ############################# - -if [ ${PREPARE_VERSION_POST_TAG} -gt 0 ]; then - echo "Updating versions for ${next_version_snapshot}" - update_versions "${version}" "${next_version}" "snapshot" - git commit -m "[Release] Update versions for ${next_version_snapshot}" -fi - -if [ ${PREPARE_DEB_PACKAGE_NAMES} -gt 0 ]; then - echo "Updating .deb package names for ${next_version}" - so_version() { - local version=$1 - local major_version=$(echo $version | sed -E -e 's/^([0-9]+)\.[0-9]+\.[0-9]+$/\1/') - local minor_version=$(echo $version | sed -E -e 's/^[0-9]+\.([0-9]+)\.[0-9]+$/\1/') - expr ${major_version} \* 100 + ${minor_version} - } - deb_lib_suffix=$(so_version $version) - next_deb_lib_suffix=$(so_version $next_version) - if [ "${deb_lib_suffix}" != "${next_deb_lib_suffix}" ]; then - cd $SOURCE_DIR/../tasks/linux-packages/apache-arrow - for target in debian*/lib*${deb_lib_suffix}.install; do - git mv \ - ${target} \ - $(echo $target | sed -e "s/${deb_lib_suffix}/${next_deb_lib_suffix}/") - done - deb_lib_suffix_substitute_pattern="s/(lib(arrow|gandiva|parquet|plasma)[-a-z]*)${deb_lib_suffix}/\\1${next_deb_lib_suffix}/g" - sed -i.bak -E -e "${deb_lib_suffix_substitute_pattern}" debian*/control* - rm -f debian*/control*.bak - git add debian*/control* - cd - - cd $SOURCE_DIR/../tasks/ - sed -i.bak -E -e "${deb_lib_suffix_substitute_pattern}" tasks.yml - rm -f tasks.yml.bak - git add tasks.yml - cd - - cd $SOURCE_DIR - sed -i.bak -E -e "${deb_lib_suffix_substitute_pattern}" rat_exclude_files.txt - rm -f rat_exclude_files.txt.bak - git add rat_exclude_files.txt - git commit -m "[Release] Update .deb package names for $next_version" - cd - - fi -fi diff --git a/dev/release/02-source-test.rb b/dev/release/02-source-test.rb deleted file mode 100644 index 7d92881f282a..000000000000 --- a/dev/release/02-source-test.rb +++ /dev/null @@ -1,146 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -class SourceTest < Test::Unit::TestCase - include GitRunnable - include VersionDetectable - - def setup - @current_commit = git_current_commit - detect_versions - @tag_name = "apache-arrow-#{@release_version}" - @script = File.expand_path("dev/release/02-source.sh") - - Dir.mktmpdir do |dir| - Dir.chdir(dir) do - yield - end - end - end - - def source(*targets) - env = { - "SOURCE_DEFAULT" => "0", - "release_hash" => @current_commit, - } - targets.each do |target| - env["SOURCE_#{target}"] = "1" - end - output = sh(env, @script, @release_version, "0") - sh("tar", "xf", "#{@tag_name}.tar.gz") - output - end - - def test_symbolic_links - source - Dir.chdir(@tag_name) do - assert_equal([], - Find.find(".").find_all {|path| File.symlink?(path)}) - end - end - - def test_csharp_git_commit_information - source - Dir.chdir("#{@tag_name}/csharp") do - FileUtils.mv("dummy.git", "../.git") - sh("dotnet", "pack", "-c", "Release") - FileUtils.mv("../.git", "dummy.git") - Dir.chdir("artifacts/Apache.Arrow/Release") do - sh("unzip", "Apache.Arrow.#{@snapshot_version}.nupkg") - FileUtils.chmod(0400, "Apache.Arrow.nuspec") - nuspec = REXML::Document.new(File.read("Apache.Arrow.nuspec")) - nuspec_repository = nuspec.elements["package/metadata/repository"] - attributes = {} - nuspec_repository.attributes.each do |key, value| - attributes[key] = value - end - assert_equal({ - "type" => "git", - "url" => "https://github.com/apache/arrow", - "commit" => @current_commit, - }, - attributes) - end - end - end - - def test_python_version - source - Dir.chdir("#{@tag_name}/python") do - sh("python3", "setup.py", "sdist") - if on_release_branch? - pyarrow_source_archive = "dist/pyarrow-#{@release_version}.tar.gz" - else - pyarrow_source_archive = "dist/pyarrow-#{@release_version}a0.tar.gz" - end - assert_equal([pyarrow_source_archive], - Dir.glob("dist/pyarrow-*.tar.gz")) - end - end - - def test_vote - jira_url = "https://issues.apache.org/jira" - jql_conditions = [ - "project = ARROW", - "status in (Resolved, Closed)", - "fixVersion = #{@release_version}", - ] - jql = jql_conditions.join(" AND ") - n_resolved_issues = nil - search_url = URI("#{jira_url}/rest/api/2/search?jql=#{CGI.escape(jql)}") - search_url.open do |response| - n_resolved_issues = JSON.parse(response.read)["total"] - end - output = source("VOTE") - assert_equal(<<-VOTE.strip, output[/^-+$(.+?)^-+$/m, 1].strip) -To: dev@arrow.apache.org -Subject: [VOTE] Release Apache Arrow #{@release_version} - RC0 - -Hi, - -I would like to propose the following release candidate (RC0) of Apache -Arrow version #{@release_version}. This is a release consisting of #{n_resolved_issues} -resolved JIRA issues[1]. - -This release candidate is based on commit: -#{@current_commit} [2] - -The source release rc0 is hosted at [3]. -The binary artifacts are hosted at [4][5][6][7]. -The changelog is located at [8]. - -Please download, verify checksums and signatures, run the unit tests, -and vote on the release. See [9] for how to validate a release candidate. - -The vote will be open for at least 72 hours. - -[ ] +1 Release this as Apache Arrow #{@release_version} -[ ] +0 -[ ] -1 Do not release this as Apache Arrow #{@release_version} because... - -[1]: https://issues.apache.org/jira/issues/?jql=project%20%3D%20ARROW%20AND%20status%20in%20%28Resolved%2C%20Closed%29%20AND%20fixVersion%20%3D%20#{@release_version} -[2]: https://github.com/apache/arrow/tree/#{@current_commit} -[3]: https://dist.apache.org/repos/dist/dev/arrow/apache-arrow-#{@release_version}-rc0 -[4]: https://bintray.com/apache/arrow/centos-rc/#{@release_version}-rc0 -[5]: https://bintray.com/apache/arrow/debian-rc/#{@release_version}-rc0 -[6]: https://bintray.com/apache/arrow/python-rc/#{@release_version}-rc0 -[7]: https://bintray.com/apache/arrow/ubuntu-rc/#{@release_version}-rc0 -[8]: https://github.com/apache/arrow/blob/#{@current_commit}/CHANGELOG.md -[9]: https://cwiki.apache.org/confluence/display/ARROW/How+to+Verify+Release+Candidates - VOTE - end -end diff --git a/dev/release/02-source.sh b/dev/release/02-source.sh deleted file mode 100755 index 89850e7543dd..000000000000 --- a/dev/release/02-source.sh +++ /dev/null @@ -1,162 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -set -e - -: ${SOURCE_DEFAULT:=1} -: ${SOURCE_RAT:=${SOURCE_DEFAULT}} -: ${SOURCE_UPLOAD:=${SOURCE_DEFAULT}} -: ${SOURCE_VOTE:=${SOURCE_DEFAULT}} - -SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)" - -if [ "$#" -ne 2 ]; then - echo "Usage: $0 " - exit -fi - -version=$1 -rc=$2 - -tag=apache-arrow-${version} -tagrc=${tag}-rc${rc} -rc_url="https://dist.apache.org/repos/dist/dev/arrow/${tagrc}" - -echo "Preparing source for tag ${tag}" - -: ${release_hash:=$(cd "${SOURCE_TOP_DIR}" && git rev-list --max-count=1 ${tag})} - -if [ ${SOURCE_UPLOAD} -gt 0 ]; then - if [ -z "$release_hash" ]; then - echo "Cannot continue: unknown git tag: $tag" - exit - fi -fi - -echo "Using commit $release_hash" - -tarball=${tag}.tar.gz - -rm -rf ${tag} -# be conservative and use the release hash, even though git produces the same -# archive (identical hashes) using the scm tag -(cd "${SOURCE_TOP_DIR}" && \ - git archive ${release_hash} --prefix ${tag}/) | \ - tar xf - - -# Resolve all hard and symbolic links -rm -rf ${tag}.tmp -mv ${tag} ${tag}.tmp -cp -R -L ${tag}.tmp ${tag} -rm -rf ${tag}.tmp - -# Create a dummy .git/ directory to download the source files from GitHub with Source Link in C#. -dummy_git=${tag}/csharp/dummy.git -mkdir ${dummy_git} -pushd ${dummy_git} -echo ${release_hash} > HEAD -echo '[remote "origin"] url = https://github.com/apache/arrow.git' >> config -mkdir objects refs -popd - -# Create new tarball from modified source directory -tar czf ${tarball} ${tag} -rm -rf ${tag} - -if [ ${SOURCE_RAT} -gt 0 ]; then - "${SOURCE_DIR}/run-rat.sh" ${tarball} -fi - -if [ ${SOURCE_UPLOAD} -gt 0 ]; then - # sign the archive - gpg --armor --output ${tarball}.asc --detach-sig ${tarball} - shasum -a 256 $tarball > ${tarball}.sha256 - shasum -a 512 $tarball > ${tarball}.sha512 - - # check out the arrow RC folder - svn co --depth=empty https://dist.apache.org/repos/dist/dev/arrow tmp - - # add the release candidate for the tag - mkdir -p tmp/${tagrc} - - # copy the rc tarball into the tmp dir - cp ${tarball}* tmp/${tagrc} - - # commit to svn - svn add tmp/${tagrc} - svn ci -m "Apache Arrow ${version} RC${rc}" tmp/${tagrc} - - # clean up - rm -rf tmp - - echo "Success! The release candidate is available here:" - echo " ${rc_url}" - echo "" - echo "Commit SHA1: ${release_hash}" - echo "" -fi - -if [ ${SOURCE_VOTE} -gt 0 ]; then - echo "The following draft email has been created to send to the" - echo "dev@arrow.apache.org mailing list" - echo "" - echo "---------------------------------------------------------" - jira_url="https://issues.apache.org/jira" - jql="project%20%3D%20ARROW%20AND%20status%20in%20%28Resolved%2C%20Closed%29%20AND%20fixVersion%20%3D%20${version}" - n_resolved_issues=$(curl "${jira_url}/rest/api/2/search/?jql=${jql}" | jq ".total") - cat < " - exit -fi - -version=$1 -rc_number=$2 -version_with_rc="${version}-rc${rc_number}" -crossbow_job_prefix="release-${version_with_rc}" - -release_tag="apache-arrow-${version}" -release_candidate_branch="release-${version}-rc${rc_number}" - -: ${GIT_REMOTE:="origin"} - -git checkout ${release_candidate_branch} -git push -u ${GIT_REMOTE} ${release_candidate_branch} - -# archery will submit a job with id: "${crossbow_job_prefix}-0" unless there -# are jobs submitted with the same prefix (the integer at the end is auto -# incremented) -archery crossbow submit \ - --job-prefix ${crossbow_job_prefix} \ - --arrow-version ${version_with_rc} \ - --group packaging diff --git a/dev/release/04-binary-download.sh b/dev/release/04-binary-download.sh deleted file mode 100755 index d0b61b05884a..000000000000 --- a/dev/release/04-binary-download.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -set -e - -if [ "$#" -ne 2 ]; then - echo "Usage: $0 " - exit -fi - -version=$1 -rc_number=$2 -version_with_rc="${version}-rc${rc_number}" -crossbow_job_prefix="release-${version_with_rc}" - -# archery will submit a job with id: "${crossbow_job_prefix}-0" unless there -# are jobs submitted with the same prefix (the integer at the end is auto -# incremented) -: ${CROSSBOW_JOB_ID:="${crossbow_job_prefix}-0"} - -archery crossbow download-artifacts ${CROSSBOW_JOB_ID} diff --git a/dev/release/05-binary-upload.sh b/dev/release/05-binary-upload.sh deleted file mode 100755 index 4a360c28b047..000000000000 --- a/dev/release/05-binary-upload.sh +++ /dev/null @@ -1,137 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e -set -u -set -o pipefail - -SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -if [ "$#" -ne 2 ]; then - echo "Usage: $0 " - exit -fi - -version=$1 -rc=$2 - -version_with_rc="${version}-rc${rc}" -crossbow_job_prefix="release-${version_with_rc}" -crossbow_package_dir="${SOURCE_DIR}/../../packages" - -: ${CROSSBOW_JOB_ID:="${crossbow_job_prefix}-0"} -artifact_dir="${crossbow_package_dir}/${CROSSBOW_JOB_ID}" - -if [ ! -e "$artifact_dir" ]; then - echo "$artifact_dir does not exist" - exit 1 -fi - -if [ ! -d "$artifact_dir" ]; then - echo "$artifact_dir is not a directory" - exit 1 -fi - -cd "${SOURCE_DIR}" - -: ${BINTRAY_REPOSITORY_CUSTOM:=${BINTRAY_REPOSITORY:-}} -: ${SOURCE_BINTRAY_REPOSITORY_CUSTOM:=${SOURCE_BINTRAY_REPOSITORY:-}}} - -if [ ! -f .env ]; then - echo "You must create $(pwd)/.env" - echo "You can use $(pwd)/.env.example as template" - exit 1 -fi -. .env - -if [ -n "${BINTRAY_REPOSITORY_CUSTOM}" ]; then - BINTRAY_REPOSITORY=${BINTRAY_REPOSITORY_CUSTOM} -fi - -if [ -n "${SOURCE_BINTRAY_REPOSITORY_CUSTOM}" ]; then - SOURCE_BINTRAY_REPOSITORY=${SOURCE_BINTRAY_REPOSITORY_CUSTOM} -fi - -. binary-common.sh - -# By default upload all artifacts. -# To deactivate one category, deactivate the category and all of its dependents. -# To explicitly select one category, set UPLOAD_DEFAULT=0 UPLOAD_X=1. -: ${UPLOAD_DEFAULT:=1} -: ${UPLOAD_CENTOS_RPM:=${UPLOAD_DEFAULT}} -: ${UPLOAD_CENTOS_YUM:=${UPLOAD_DEFAULT}} -: ${UPLOAD_DEBIAN_APT:=${UPLOAD_DEFAULT}} -: ${UPLOAD_DEBIAN_DEB:=${UPLOAD_DEFAULT}} -: ${UPLOAD_NUGET:=${UPLOAD_DEFAULT}} -: ${UPLOAD_PYTHON:=${UPLOAD_DEFAULT}} -: ${UPLOAD_UBUNTU_APT:=${UPLOAD_DEFAULT}} -: ${UPLOAD_UBUNTU_DEB:=${UPLOAD_DEFAULT}} - -rake_tasks=() -apt_targets=() -yum_targets=() -if [ ${UPLOAD_DEBIAN_DEB} -gt 0 ]; then - rake_tasks+=(deb) - apt_targets+=(debian) -fi -if [ ${UPLOAD_DEBIAN_APT} -gt 0 ]; then - rake_tasks+=(apt:rc) - apt_targets+=(debian) -fi -if [ ${UPLOAD_UBUNTU_DEB} -gt 0 ]; then - rake_tasks+=(deb) - apt_targets+=(ubuntu) -fi -if [ ${UPLOAD_UBUNTU_APT} -gt 0 ]; then - rake_tasks+=(apt:rc) - apt_targets+=(ubuntu) -fi -if [ ${UPLOAD_CENTOS_RPM} -gt 0 ]; then - rake_tasks+=(rpm) - yum_targets+=(centos) -fi -if [ ${UPLOAD_CENTOS_YUM} -gt 0 ]; then - rake_tasks+=(yum:rc) - yum_targets+=(centos) -fi -if [ ${UPLOAD_NUGET} -gt 0 ]; then - rake_tasks+=(nuget:rc) -fi -if [ ${UPLOAD_PYTHON} -gt 0 ]; then - rake_tasks+=(python:rc) -fi -rake_tasks+=(summary:rc) - -tmp_dir=binary/tmp -mkdir -p "${tmp_dir}" -source_artifacts_dir="${tmp_dir}/artifacts" -rm -rf "${source_artifacts_dir}" -cp -a "${artifact_dir}" "${source_artifacts_dir}" - -docker_run \ - ./runner.sh \ - rake \ - "${rake_tasks[@]}" \ - APT_TARGETS=$(IFS=,; echo "${apt_targets[*]}") \ - ARTIFACTS_DIR="${tmp_dir}/artifacts" \ - BINTRAY_REPOSITORY=${BINTRAY_REPOSITORY} \ - RC=${rc} \ - SOURCE_BINTRAY_REPOSITORY=${SOURCE_BINTRAY_REPOSITORY} \ - VERSION=${version} \ - YUM_TARGETS=$(IFS=,; echo "${yum_targets[*]}") diff --git a/dev/release/README.md b/dev/release/README.md deleted file mode 100644 index 0a9cc3e04b3a..000000000000 --- a/dev/release/README.md +++ /dev/null @@ -1,24 +0,0 @@ - - -## Release management scripts - -To learn more, see the project wiki: - -https://cwiki.apache.org/confluence/display/ARROW/Release+Management+Guide diff --git a/dev/release/Rakefile b/dev/release/Rakefile deleted file mode 100644 index ff57bad5e8fa..000000000000 --- a/dev/release/Rakefile +++ /dev/null @@ -1,37 +0,0 @@ -# -*- ruby -*- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -require_relative "binary-task" - -if File.exist?(".env") - File.open(".env") do |env| - env.each_line do |line| - case line.strip - when /\A#/ - when /\A([^=]+)=(.*)\z/ - key = $1 - value = $2 - ENV[key] ||= value - end - end - end -end - -binary_task = BinaryTask.new -binary_task.define diff --git a/dev/release/VERIFY.md b/dev/release/VERIFY.md deleted file mode 100644 index 5b441ac13f1c..000000000000 --- a/dev/release/VERIFY.md +++ /dev/null @@ -1,76 +0,0 @@ - - -# Verifying Arrow releases - -## Windows - -We've provided a convenience script for verifying the C++ and Python builds on -Windows. Read the comments in `verify-release-candidate.bat` for instructions. - -## Linux and macOS - -We've provided a convenience script for verifying the C++, Python, C -GLib, Java and JavaScript builds on Linux and macOS. Read the comments in -`verify-release-candidate.sh` for instructions. - -### C GLib - -You need the followings to verify C GLib build: - - * GLib - * GObject Introspection - * Ruby (not EOL-ed version is required) - * gobject-introspection gem - * test-unit gem - -You can install them by the followings on Debian GNU/Linux and Ubuntu: - -```console -% sudo apt install -y -V libgirepository1.0-dev ruby-dev -% sudo gem install gobject-introspection test-unit -``` - -You can install them by the followings on CentOS: - -```console -% sudo yum install -y gobject-introspection-devel -% git clone https://github.com/sstephenson/rbenv.git ~/.rbenv -% git clone https://github.com/sstephenson/ruby-build.git ~/.rbenv/plugins/ruby-build -% echo 'export PATH="$HOME/.rbenv/bin:$PATH"' >> ~/.bash_profile -% echo 'eval "$(rbenv init -)"' >> ~/.bash_profile -% exec ${SHELL} --login -% sudo yum install -y gcc make patch openssl-devel readline-devel zlib-devel -% rbenv install 2.4.2 -% rbenv global 2.4.2 -% gem install gobject-introspection test-unit -``` - -You can install them by the followings on macOS: - -```console -% brew install -y gobject-introspection -% gem install gobject-introspection test-unit -``` - -You need to set `PKG_CONFIG_PATH` to find libffi on macOS: - -```console -% export PKG_CONFIG_PATH=$(brew --prefix libffi)/lib/pkgconfig:$PKG_CONFIG_PATH -``` diff --git a/dev/release/binary-common.sh b/dev/release/binary-common.sh deleted file mode 100644 index 7c66e375f8d8..000000000000 --- a/dev/release/binary-common.sh +++ /dev/null @@ -1,86 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -docker_image_name=apache-arrow/release-binary -gpg_agent_extra_socket="$(gpgconf --list-dirs agent-extra-socket)" -if [ $(uname) = "Darwin" ]; then - docker_uid=10000 - docker_gid=10000 -else - docker_uid=$(id -u) - docker_gid=$(id -g) -fi -docker_ssh_key="${SOURCE_DIR}/binary/id_rsa" - -if [ ! -f "${docker_ssh_key}" ]; then - ssh-keygen -N "" -f "${docker_ssh_key}" -fi - -docker_gpg_ssh() { - local ssh_port=$1 - shift - local known_hosts_file=$(mktemp -t "arrow-binary-gpg-ssh-known-hosts.XXXXX") - local exit_code= - if ssh \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=${known_hosts_file} \ - -i "${docker_ssh_key}" \ - -p ${ssh_port} \ - -R "/home/arrow/.gnupg/S.gpg-agent:${gpg_agent_extra_socket}" \ - arrow@127.0.0.1 \ - "$@"; then - exit_code=$?; - else - exit_code=$?; - fi - rm -f ${known_hosts_file} - return ${exit_code} -} - -docker_run() { - local container_id_dir=$(mktemp -d -t "arrow-binary-gpg-container.XXXXX") - local container_id_file=${container_id_dir}/id - docker \ - run \ - --cidfile ${container_id_file} \ - --detach \ - --publish-all \ - --rm \ - --volume "$PWD":/host \ - ${docker_image_name} \ - bash -c " -if [ \$(id -u) -ne ${docker_uid} ]; then - usermod --uid ${docker_uid} arrow - chown -R arrow: ~arrow -fi -/usr/sbin/sshd -D -" - local container_id=$(cat ${container_id_file}) - local ssh_port=$(docker port ${container_id} | grep -E -o '[0-9]+$') - # Wait for sshd available - while ! docker_gpg_ssh ${ssh_port} : > /dev/null 2>&1; do - sleep 0.1 - done - gpg --export ${GPG_KEY_ID} | docker_gpg_ssh ${ssh_port} gpg --import - docker_gpg_ssh ${ssh_port} "$@" - docker kill ${container_id} - rm -rf ${container_id_dir} -} - -docker build -t ${docker_image_name} "${SOURCE_DIR}/binary" - -chmod go-rwx "${docker_ssh_key}" diff --git a/dev/release/binary-task.rb b/dev/release/binary-task.rb deleted file mode 100644 index 42bc1fe47665..000000000000 --- a/dev/release/binary-task.rb +++ /dev/null @@ -1,1909 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -require "digest/sha2" -require "io/console" -require "json" -require "net/http" -require "pathname" -require "tempfile" -require "thread" -require "time" - -class BinaryTask - include Rake::DSL - - class ThreadPool - def initialize(use_case, &worker) - @n_workers = choose_n_workers(use_case) - @worker = worker - @jobs = Thread::Queue.new - @workers = @n_workers.times.collect do - Thread.new do - loop do - job = @jobs.pop - break if job.nil? - @worker.call(job) - end - end - end - end - - def <<(job) - @jobs << job - end - - def join - @n_workers.times do - @jobs << nil - end - @workers.each(&:join) - end - - private - def choose_n_workers(use_case) - case use_case - when :bintray - # Too many workers cause Bintray error. - 6 - when :gpg - # Too many workers cause gpg-agent error. - 2 - else - raise "Unknown use case: #{use_case}" - end - end - end - - class ProgressReporter - def initialize(label, count_max=0) - @label = label - @count_max = count_max - - @mutex = Thread::Mutex.new - - @time_start = Time.now - @time_previous = Time.now - @count_current = 0 - @count_previous = 0 - end - - def advance - @mutex.synchronize do - @count_current += 1 - - return if @count_max.zero? - - time_current = Time.now - if time_current - @time_previous <= 1 - return - end - - show_progress(time_current) - end - end - - def increment_max - @mutex.synchronize do - @count_max += 1 - end - end - - def finish - @mutex.synchronize do - return if @count_max.zero? - show_progress(Time.now) - $stderr.puts - end - end - - private - def show_progress(time_current) - n_finishes = @count_current - @count_previous - throughput = n_finishes.to_f / (time_current - @time_previous) - @time_previous = time_current - @count_previous = @count_current - - message = build_message(time_current, throughput) - $stderr.print("\r#{message}") if message - end - - def build_message(time_current, throughput) - percent = (@count_current / @count_max.to_f) * 100 - formatted_count = "[%s/%s]" % [ - format_count(@count_current), - format_count(@count_max), - ] - elapsed_second = time_current - @time_start - if throughput.zero? - rest_second = 0 - else - rest_second = (@count_max - @count_current) / throughput - end - separator = " - " - progress = "%5.1f%% %s %s %s %s" % [ - percent, - formatted_count, - format_time_interval(elapsed_second), - format_time_interval(rest_second), - format_throughput(throughput), - ] - label = @label - - width = guess_terminal_width - return "#{label}#{separator}#{progress}" if width.nil? - - return nil if progress.size > width - - label_width = width - progress.size - separator.size - if label.size > label_width - ellipsis = "..." - shorten_label_width = label_width - ellipsis.size - if shorten_label_width < 1 - return progress - else - label = label[0, shorten_label_width] + ellipsis - end - end - "#{label}#{separator}#{progress}" - end - - def format_count(count) - "%d" % count - end - - def format_time_interval(interval) - if interval < 60 - "00:00:%02d" % interval - elsif interval < (60 * 60) - minute, second = interval.divmod(60) - "00:%02d:%02d" % [minute, second] - elsif interval < (60 * 60 * 24) - minute, second = interval.divmod(60) - hour, minute = minute.divmod(60) - "%02d:%02d:%02d" % [hour, minute, second] - else - minute, second = interval.divmod(60) - hour, minute = minute.divmod(60) - day, hour = hour.divmod(24) - "%dd %02d:%02d:%02d" % [day, hour, minute, second] - end - end - - def format_throughput(throughput) - "%2d/s" % throughput - end - - def guess_terminal_width - guess_terminal_width_from_io || - guess_terminal_width_from_command || - guess_terminal_width_from_env || - 80 - end - - def guess_terminal_width_from_io - if IO.respond_to?(:console) and IO.console - IO.console.winsize[1] - elsif $stderr.respond_to?(:winsize) - begin - $stderr.winsize[1] - rescue SystemCallError - nil - end - else - nil - end - end - - def guess_terminal_width_from_command - IO.pipe do |input, output| - begin - pid = spawn("tput", "cols", {:out => output, :err => output}) - rescue SystemCallError - return nil - end - - output.close - _, status = Process.waitpid2(pid) - return nil unless status.success? - - result = input.read.chomp - begin - Integer(result, 10) - rescue ArgumentError - nil - end - end - end - - def guess_terminal_width_from_env - env = ENV["COLUMNS"] || ENV["TERM_WIDTH"] - return nil if env.nil? - - begin - Integer(env, 10) - rescue ArgumentError - nil - end - end - end - - class BintrayClient - class Error < StandardError - attr_reader :request - attr_reader :response - def initialize(request, response, message) - @request = request - @response = response - super(message) - end - end - - def initialize(options={}) - @options = options - repository = @options[:repository] - @subject, @repository = repository.split("/", 2) if repository - @package = @options[:package] - @version = @options[:version] - @user = @options[:user] - @api_key = @options[:api_key] - end - - def request(method, headers, *components, &block) - url = build_request_url(*components) - http = Net::HTTP.new(url.host, url.port) - http.set_debug_output($stderr) if ENV["DEBUG"] - http.use_ssl = true - http.start do |http| - request = build_request(method, url, headers, &block) - http.request(request) do |response| - case response - when Net::HTTPSuccess - return JSON.parse(response.body) - else - message = "failed to request: " - message << "#{url}: #{request.method}: " - message << "#{response.message} #{response.code}:\n" - message << response.body - raise Error.new(request, response, message) - end - end - end - end - - def repository - request(:get, - {}, - "repos", - @subject, - @repository) - end - - def create_repository - request(:post, - {}, - "repos", - @subject, - @repository) do - request = { - "name" => @repository, - "desc" => "Apache Arrow", - } - JSON.generate(request) - end - end - - def ensure_repository - begin - repository - rescue Error => error - case error.response - when Net::HTTPNotFound - create_repository - else - raise - end - end - end - - def package - request(:get, - {}, - "packages", - @subject, - @repository, - @package) - end - - def package_versions - begin - package["versions"] - rescue Error => error - case error.response - when Net::HTTPNotFound - [] - else - raise - end - end - end - - def create_package(description) - request(:post, - {}, - "packages", - @subject, - @repository) do - request = { - "name" => @package, - "desc" => description, - "licenses" => ["Apache-2.0"], - "vcs_url" => "https://github.com/apache/arrow.git", - "website_url" => "https://arrow.apache.org/", - "issue_tracker_url" => "https://issues.apache.org/jira/browse/ARROW", - "github_repo" => "apache/arrow", - "public_download_numbers" => true, - "public_stats" => true, - } - JSON.generate(request) - end - end - - def ensure_package(description) - begin - package - rescue Error => error - case error.response - when Net::HTTPNotFound - create_package(description) - else - raise - end - end - end - - def create_version(description) - request(:post, - {}, - "packages", - @subject, - @repository, - @package, - "versions") do - request = { - "name" => @version, - "desc" => description, - } - JSON.generate(request) - end - end - - def ensure_version(version, description) - return if package["versions"].include?(version) - create_version(description) - end - - def files - request(:get, - {}, - "packages", - @subject, - @repository, - @package, - "versions", - @version, - "files") - end - - def upload(path, destination_path) - sha256 = Digest::SHA256.file(path).hexdigest - headers = { - "X-Bintray-Override" => "1", - "X-Bintray-Package" => @package, - "X-Bintray-Publish" => "1", - "X-Bintray-Version" => @version, - "X-Checksum-Sha2" => sha256, - "Content-Length" => File.size(path).to_s, - } - File.open(path, "rb") do |input| - request(:put, - headers, - "content", - @subject, - @repository, - destination_path) do - input - end - end - end - - def delete(path) - request(:delete, - {}, - "content", - @subject, - @repository, - path) - end - - private - def build_request_url(*components) - if components.last.is_a?(Hash) - parameters = components.pop - else - parameters = nil - end - path = components.join("/") - url = "https://bintray.com/api/v1/#{path}" - if parameters - separator = "?" - parameters.each do |key, value| - url << "#{separator}#{CGI.escape(key)}=#{CGI.escape(value)}" - separator = "&" - end - end - URI(url) - end - - def build_request(method, url, headers, &block) - case method - when :get - request = Net::HTTP::Get.new(url, headers) - when :post - request = Net::HTTP::Post.new(url, headers) - when :put - request = Net::HTTP::Put.new(url, headers) - when :delete - request = Net::HTTP::Delete.new(url, headers) - else - raise "unsupported HTTP method: #{method.inspect}" - end - request.basic_auth(@user, @api_key) if @user and @api_key - if block_given? - request["Content-Type"] = "application/json" - body = yield - if body.is_a?(String) - request.body = body - else - request.body_stream = body - end - end - request - end - end - - module HashChekable - def same_hash?(path, sha256) - return false unless File.exist?(path) - Digest::SHA256.file(path).hexdigest == sha256 - end - end - - class BintrayDownloader - include HashChekable - - def initialize(repository:, - distribution:, - version:, - rc: nil, - destination:, - user:, - api_key:) - @repository = repository - @distribution = distribution - @version = version - @rc = rc - @destination = destination - @user = user - @api_key = api_key - end - - def download - client.ensure_repository - - progress_label = "Downloading: #{package} #{full_version}" - progress_reporter = ProgressReporter.new(progress_label) - pool = ThreadPool.new(:bintray) do |path, output_path| - download_file(path, output_path) - progress_reporter.advance - end - target_files.each do |file| - path = file["path"] - path_without_package = path.split("/", 2)[1..-1].join("/") - output_path = "#{@destination}/#{path_without_package}" - yield(output_path) - sha256 = file["sha256"] - next if same_hash?(output_path, sha256) - output_dir = File.dirname(output_path) - FileUtils.mkdir_p(output_dir) - progress_reporter.increment_max - pool << [path, output_path] - end - pool.join - progress_reporter.finish - end - - private - def package - if @rc - "#{@distribution}-rc" - else - @distribution - end - end - - def full_version - if @rc - "#{@version}-rc#{@rc}" - else - @version - end - end - - def client(options={}) - default_options = { - repository: @repository, - package: package, - version: full_version, - user: @user, - api_key: @api_key, - } - BintrayClient.new(default_options.merge(options)) - end - - def target_files - begin - client.files - rescue BintrayClient::Error - [] - end - end - - def download_file(path, output_path) - max_n_retries = 5 - n_retries = 0 - url = URI("https://dl.bintray.com/#{@repository}/#{path}") - begin - download_url(url, output_path) - rescue OpenSSL::OpenSSLError, - SocketError, - SystemCallError, - Timeout::Error => error - n_retries += 1 - if n_retries <= max_n_retries - $stderr.puts - $stderr.puts("Retry #{n_retries}: #{url}: " + - "#{error.class}: #{error.message}") - retry - else - raise - end - end - end - - def download_url(url, output_path) - loop do - http = Net::HTTP.new(url.host, url.port) - http.set_debug_output($stderr) if ENV["DEBUG"] - http.use_ssl = true - http.start do |http| - request = Net::HTTP::Get.new(url) - http.request(request) do |response| - case response - when Net::HTTPSuccess - save_response(response, output_path) - return - when Net::HTTPRedirection - url = URI(response["Location"]) - when Net::HTTPNotFound - $stderr.puts(build_download_error_message(url, response)) - return - else - raise build_download_error_message(url, response) - end - end - end - end - end - - def save_response(response, output_path) - File.open(output_path, "wb") do |output| - response.read_body do |chunk| - output.print(chunk) - end - end - last_modified = response["Last-Modified"] - if last_modified - FileUtils.touch(output_path, mtime: Time.rfc2822(last_modified)) - end - end - - def build_download_error_message(url, response) - message = "failed to download: " - message << "#{url}: #{response.message} #{response.code}:\n" - message << response.body - message - end - end - - class BintrayUploader - include HashChekable - - def initialize(repository:, - distribution:, - distribution_label:, - version:, - rc: nil, - source:, - destination_prefix: "", - user:, - api_key:) - @repository = repository - @distribution = distribution - @distribution_label = distribution_label - @version = version - @rc = rc - @source = source - @destination_prefix = destination_prefix - @user = user - @api_key = api_key - end - - def upload - client.ensure_repository - client.ensure_package(package_description) - client.ensure_version(full_version, version_description) - - progress_label = "Uploading: #{package} #{full_version}" - progress_reporter = ProgressReporter.new(progress_label) - pool = ThreadPool.new(:bintray) do |path, relative_path| - upload_file(path, relative_path) - progress_reporter.advance - end - - files = existing_files - source = Pathname(@source) - source.glob("**/*") do |path| - next if path.directory? - destination_path = - "#{package}/#{@destination_prefix}#{path.relative_path_from(source)}" - file = files[destination_path] - next if file and same_hash?(path.to_s, file["sha256"]) - progress_reporter.increment_max - pool << [path, destination_path] - end - pool.join - progress_reporter.finish - end - - private - def package - if @rc - "#{@distribution}-rc" - else - @distribution - end - end - - def full_version - if @rc - "#{@version}-rc#{@rc}" - else - @version - end - end - - def package_description - if @rc - release_type = "RC" - else - release_type = "Release" - end - case @distribution - when "debian", "ubuntu" - "#{release_type} deb packages for #{@distribution_label}" - when "centos" - "#{release_type} RPM packages for #{@distribution_label}" - else - "#{release_type} binaries for #{@distribution_label}" - end - end - - def version_description - if @rc - "Apache Arrow #{@version} RC#{@rc} for #{@distribution_label}" - else - "Apache Arrow #{@version} for #{@distribution_label}" - end - end - - def client - BintrayClient.new(repository: @repository, - package: package, - version: full_version, - user: @user, - api_key: @api_key) - end - - def existing_files - files = {} - client.files.each do |file| - files[file["path"]] = file - end - files - end - - def upload_file(path, destination_path) - max_n_retries = 3 - n_retries = 0 - begin - begin - client.upload(path, destination_path) - rescue BintrayClient::Error => error - case error.response - when Net::HTTPConflict - n_retries += 1 - if n_retries <= max_n_retries - client.delete(destination_path) - retry - else - $stderr.puts(error) - end - else - $stderr.puts(error) - end - end - rescue OpenSSL::OpenSSLError, - SocketError, - SystemCallError, - Timeout::Error => error - n_retries += 1 - if n_retries <= max_n_retries - $stderr.puts - $stderr.puts("Retry #{n_retries}: #{path}: " + - "#{error.class}: #{error.message}") - retry - else - raise - end - end - end - end - - def define - define_apt_tasks - define_yum_tasks - define_python_tasks - define_nuget_tasks - define_summary_tasks - end - - private - def env_value(name) - value = ENV[name] - value = yield(name) if value.nil? and block_given? - raise "Specify #{name} environment variable" if value.nil? - value - end - - def verbose? - ENV["VERBOSE"] == "yes" - end - - def default_output - if verbose? - nil - else - IO::NULL - end - end - - def gpg_key_id - env_value("GPG_KEY_ID") - end - - def shorten_gpg_key_id(id) - id[-8..-1] - end - - def rpm_gpg_key_package_name(id) - "gpg-pubkey-#{shorten_gpg_key_id(id).downcase}" - end - - def bintray_user - env_value("BINTRAY_USER") - end - - def bintray_api_key - env_value("BINTRAY_API_KEY") - end - - def bintray_repository - env_value("BINTRAY_REPOSITORY") - end - - def source_bintray_repository - env_value("SOURCE_BINTRAY_REPOSITORY") do - bintray_repository - end - end - - def artifacts_dir - env_value("ARTIFACTS_DIR") - end - - def version - env_value("VERSION") - end - - def rc - env_value("RC") - end - - def full_version - "#{version}-rc#{rc}" - end - - def valid_sign?(path, sign_path) - IO.pipe do |input, output| - begin - sh({"LANG" => "C"}, - "gpg", - "--verify", - sign_path, - path, - out: default_output, - err: output, - verbose: false) - rescue - return false - end - output.close - /Good signature/ === input.read - end - end - - def sign(source_path, destination_path) - if File.exist?(destination_path) - return if valid_sign?(source_path, destination_path) - rm(destination_path, verbose: false) - end - sh("gpg", - "--detach-sig", - "--local-user", gpg_key_id, - "--output", destination_path, - source_path, - out: default_output, - verbose: verbose?) - end - - def sha512(source_path, destination_path) - if File.exist?(destination_path) - sha512 = File.read(destination_path).split[0] - return if Digest::SHA512.file(source_path).hexdigest == sha512 - end - absolute_destination_path = File.expand_path(destination_path) - Dir.chdir(File.dirname(source_path)) do - sh("shasum", - "--algorithm", "512", - File.basename(source_path), - out: absolute_destination_path, - verbose: verbose?) - end - end - - def sign_dir(label, dir) - progress_label = "Signing: #{label}" - progress_reporter = ProgressReporter.new(progress_label) - - target_paths = [] - Pathname(dir).glob("**/*") do |path| - next if path.directory? - case path.extname - when ".asc", ".sha512" - next - end - progress_reporter.increment_max - target_paths << path.to_s - end - target_paths.each do |path| - sign(path, "#{path}.asc") - sha512(path, "#{path}.sha512") - progress_reporter.advance - end - progress_reporter.finish - end - - def download_distribution(distribution, - destination, - with_source_repository: false) - existing_paths = {} - Pathname(destination).glob("**/*") do |path| - next if path.directory? - existing_paths[path.to_s] = true - end - if with_source_repository - source_client = BintrayClient.new(repository: source_bintray_repository, - package: distribution, - user: bintray_user, - api_key: bintray_api_key) - source_client.package_versions[0, 10].each do |source_version| - downloader = BintrayDownloader.new(repository: source_bintray_repository, - distribution: distribution, - version: source_version, - destination: destination, - user: bintray_user, - api_key: bintray_api_key) - downloader.download do |output_path| - existing_paths.delete(output_path) - end - end - end - downloader = BintrayDownloader.new(repository: bintray_repository, - distribution: distribution, - version: version, - rc: rc, - destination: destination, - user: bintray_user, - api_key: bintray_api_key) - downloader.download do |output_path| - existing_paths.delete(output_path) - end - existing_paths.each_key do |path| - rm_f(path, verbose: verbose?) - end - end - - def same_content?(path1, path2) - File.exist?(path1) and - File.exist?(path2) and - Digest::SHA256.file(path1) == Digest::SHA256.file(path2) - end - - def copy_artifact(source_path, - destination_path, - progress_reporter) - return if same_content?(source_path, destination_path) - progress_reporter.increment_max - destination_dir = File.dirname(destination_path) - unless File.exist?(destination_dir) - mkdir_p(destination_dir, verbose: verbose?) - end - cp(source_path, destination_path, verbose: verbose?) - progress_reporter.advance - end - - def tmp_dir - "binary/tmp" - end - - def rc_dir - "#{tmp_dir}/rc" - end - - def release_dir - "#{tmp_dir}/release" - end - - def deb_dir - "#{rc_dir}/deb/#{full_version}" - end - - def apt_repository_label - "Apache Arrow" - end - - def apt_repository_description - "Apache Arrow packages" - end - - def apt_rc_repositories_dir - "#{rc_dir}/apt/repositories" - end - - def apt_release_repositories_dir - "#{release_dir}/apt/repositories" - end - - def available_apt_targets - [ - ["debian", "buster", "main"], - ["debian", "bullseye", "main"], - ["ubuntu", "bionic", "main"], - ["ubuntu", "focal", "main"], - ["ubuntu", "groovy", "main"], - ] - end - - def apt_distribution_label(distribution) - case distribution - when "debian" - "Debian" - when "ubuntu" - "Ubuntu" - else - distribution - end - end - - def apt_targets - env_apt_targets = (ENV["APT_TARGETS"] || "").split(",") - if env_apt_targets.empty? - available_apt_targets - else - available_apt_targets.select do |distribution, code_name, component| - env_apt_targets.any? do |env_apt_target| - env_apt_target.start_with?("#{distribution}-#{code_name}") - end - end - end - end - - def apt_distributions - apt_targets.collect(&:first).uniq - end - - def apt_architectures - [ - "amd64", - "arm64", - ] - end - - def define_deb_tasks - directory deb_dir - - namespace :deb do - desc "Copy deb packages" - task :copy => deb_dir do - apt_targets.each do |distribution, code_name, component| - progress_label = "Copying: #{distribution} #{code_name}" - progress_reporter = ProgressReporter.new(progress_label) - - source_dir_prefix = "#{artifacts_dir}/#{distribution}-#{code_name}" - Dir.glob("#{source_dir_prefix}*/**/*") do |path| - next if File.directory?(path) - base_name = File.basename(path) - if base_name.start_with?("apache-arrow-archive-keyring") - package_name = "apache-arrow-archive-keyring" - else - package_name = "apache-arrow" - end - distribution_dir = [ - deb_dir, - distribution, - ].join("/") - destination_path = [ - distribution_dir, - "pool", - code_name, - component, - package_name[0], - package_name, - base_name, - ].join("/") - copy_artifact(path, - destination_path, - progress_reporter) - case base_name - when /\A[^_]+-archive-keyring_.*\.deb\z/ - latest_archive_keyring_package_path = [ - distribution_dir, - "#{package_name}-latest-#{code_name}.deb" - ].join("/") - copy_artifact(path, - latest_archive_keyring_package_path, - progress_reporter) - end - end - progress_reporter.finish - end - end - - desc "Sign deb packages" - task :sign => deb_dir do - apt_distributions.each do |distribution| - distribution_dir = "#{deb_dir}/#{distribution}" - Dir.glob("#{distribution_dir}/**/*.dsc") do |path| - begin - sh({"LANG" => "C"}, - "gpg", - "--verify", - path, - out: IO::NULL, - err: IO::NULL, - verbose: false) - rescue - sh("debsign", - "--no-re-sign", - "-k#{gpg_key_id}", - path, - out: default_output, - verbose: verbose?) - end - end - sign_dir(distribution, distribution_dir) - end - end - - desc "Upload deb packages" - task :upload do - apt_distributions.each do |distribution| - distribution_dir = "#{deb_dir}/#{distribution}" - distribution_label = apt_distribution_label(distribution) - uploader = BintrayUploader.new(repository: bintray_repository, - distribution: distribution, - distribution_label: distribution_label, - version: version, - rc: rc, - source: distribution_dir, - user: bintray_user, - api_key: bintray_api_key) - uploader.upload - end - end - end - - desc "Release deb packages" - deb_tasks = [ - "deb:copy", - "deb:sign", - "deb:upload", - ] - task :deb => deb_tasks - end - - def generate_apt_release(dists_dir, code_name, component, architecture) - dir = "#{dists_dir}/#{component}/" - if architecture == "source" - dir << architecture - else - dir << "binary-#{architecture}" - end - - mkdir_p(dir, verbose: verbose?) - File.open("#{dir}/Release", "w") do |release| - release.puts(<<-RELEASE) -Archive: #{code_name} -Component: #{component} -Origin: #{apt_repository_label} -Label: #{apt_repository_label} -Architecture: #{architecture} - RELEASE - end - end - - def generate_apt_ftp_archive_generate_conf(code_name, component) - conf = <<-CONF -Dir::ArchiveDir "."; -Dir::CacheDir "."; -TreeDefault::Directory "pool/#{code_name}/#{component}"; -TreeDefault::SrcDirectory "pool/#{code_name}/#{component}"; -Default::Packages::Extensions ".deb"; -Default::Packages::Compress ". gzip xz"; -Default::Sources::Compress ". gzip xz"; -Default::Contents::Compress "gzip"; - CONF - - apt_architectures.each do |architecture| - conf << <<-CONF - -BinDirectory "dists/#{code_name}/#{component}/binary-#{architecture}" { - Packages "dists/#{code_name}/#{component}/binary-#{architecture}/Packages"; - Contents "dists/#{code_name}/#{component}/Contents-#{architecture}"; - SrcPackages "dists/#{code_name}/#{component}/source/Sources"; -}; - CONF - end - - conf << <<-CONF - -Tree "dists/#{code_name}" { - Sections "#{component}"; - Architectures "#{apt_architectures.join(" ")} source"; -}; - CONF - - conf - end - - def generate_apt_ftp_archive_release_conf(code_name, component) - <<-CONF -APT::FTPArchive::Release::Origin "#{apt_repository_label}"; -APT::FTPArchive::Release::Label "#{apt_repository_label}"; -APT::FTPArchive::Release::Architectures "#{apt_architectures.join(" ")}"; -APT::FTPArchive::Release::Codename "#{code_name}"; -APT::FTPArchive::Release::Suite "#{code_name}"; -APT::FTPArchive::Release::Components "#{component}"; -APT::FTPArchive::Release::Description "#{apt_repository_description}"; - CONF - end - - def apt_update(repositories_dir) - apt_targets.each do |distribution, code_name, component| - base_dir = "#{repositories_dir}/#{distribution}" - pool_dir = "#{base_dir}/pool/#{code_name}" - next unless File.exist?(pool_dir) - dists_dir = "#{base_dir}/dists/#{code_name}" - rm_rf(dists_dir, verbose: verbose?) - generate_apt_release(dists_dir, code_name, component, "source") - apt_architectures.each do |architecture| - generate_apt_release(dists_dir, code_name, component, architecture) - end - - generate_conf_file = Tempfile.new("apt-ftparchive-generate.conf") - File.open(generate_conf_file.path, "w") do |conf| - conf.puts(generate_apt_ftp_archive_generate_conf(code_name, - component)) - end - cd(base_dir, verbose: verbose?) do - sh("apt-ftparchive", - "generate", - generate_conf_file.path, - out: default_output, - verbose: verbose?) - end - - Dir.glob("#{dists_dir}/Release*") do |release| - rm_f(release, verbose: verbose?) - end - Dir.glob("#{base_dir}/*.db") do |db| - rm_f(db, verbose: verbose?) - end - release_conf_file = Tempfile.new("apt-ftparchive-release.conf") - File.open(release_conf_file.path, "w") do |conf| - conf.puts(generate_apt_ftp_archive_release_conf(code_name, - component)) - end - release_file = Tempfile.new("apt-ftparchive-release") - sh("apt-ftparchive", - "-c", release_conf_file.path, - "release", - dists_dir, - out: release_file.path, - verbose: verbose?) - release_path = "#{dists_dir}/Release" - signed_release_path = "#{release_path}.gpg" - in_release_path = "#{dists_dir}/InRelease" - mv(release_file.path, release_path, verbose: verbose?) - chmod(0644, release_path, verbose: verbose?) - sh("gpg", - "--sign", - "--detach-sign", - "--armor", - "--local-user", gpg_key_id, - "--output", signed_release_path, - release_path, - out: default_output, - verbose: verbose?) - sh("gpg", - "--clear-sign", - "--local-user", gpg_key_id, - "--output", in_release_path, - release_path, - out: default_output, - verbose: verbose?) - end - end - - def define_apt_rc_tasks - directory apt_rc_repositories_dir - - namespace :apt do - namespace :rc do - desc "Download deb files for RC APT repositories" - task :download => apt_rc_repositories_dir do - apt_distributions.each do |distribution| - download_distribution(distribution, - "#{apt_rc_repositories_dir}/#{distribution}", - with_source_repository: true) - end - end - - desc "Update RC APT repositories" - task :update do - apt_update(apt_rc_repositories_dir) - apt_targets.each do |distribution, code_name, component| - base_dir = "#{apt_rc_repositories_dir}/#{distribution}" - dists_dir = "#{base_dir}/dists/#{code_name}" - next unless File.exist?(dists_dir) - sign_dir("#{distribution} #{code_name}", - dists_dir) - end - end - - desc "Upload RC APT repositories" - task :upload => apt_rc_repositories_dir do - apt_distributions.each do |distribution| - dists_dir = "#{apt_rc_repositories_dir}/#{distribution}/dists" - distribution_label = apt_distribution_label(distribution) - uploader = BintrayUploader.new(repository: bintray_repository, - distribution: distribution, - distribution_label: distribution_label, - version: version, - rc: rc, - source: dists_dir, - destination_prefix: "dists/", - user: bintray_user, - api_key: bintray_api_key) - uploader.upload - end - end - end - - desc "Release RC APT repositories" - apt_rc_tasks = [ - "apt:rc:download", - "apt:rc:update", - "apt:rc:upload", - ] - task :rc => apt_rc_tasks - end - end - - def define_apt_release_tasks - directory apt_release_repositories_dir - - namespace :apt do - namespace :release do - desc "Download RC APT repositories" - task :download => apt_release_repositories_dir do - apt_distributions.each do |distribution| - distribution_dir = "#{apt_release_repositories_dir}/#{distribution}" - download_distribution(distribution, distribution_dir) - end - end - - desc "Upload release APT repositories" - task :upload => apt_release_repositories_dir do - apt_distributions.each do |distribution| - distribution_dir = "#{apt_release_repositories_dir}/#{distribution}" - distribution_label = apt_distribution_label(distribution) - uploader = BintrayUploader.new(repository: bintray_repository, - distribution: distribution, - distribution_label: distribution_label, - version: version, - source: distribution_dir, - user: bintray_user, - api_key: bintray_api_key) - uploader.upload - end - end - end - - desc "Release APT repositories" - apt_release_tasks = [ - "apt:release:download", - "apt:release:upload", - ] - task :release => apt_release_tasks - end - end - - def define_apt_tasks - define_deb_tasks - define_apt_rc_tasks - define_apt_release_tasks - end - - def rpm_dir - "#{rc_dir}/rpm/#{full_version}" - end - - def yum_rc_repositories_dir - "#{rc_dir}/yum/repositories" - end - - def yum_release_repositories_dir - "#{release_dir}/yum/repositories" - end - - def available_yum_targets - [ - ["centos", "7"], - ["centos", "8"], - ] - end - - def yum_distribution_label(distribution) - case distribution - when "centos" - "CentOS" - else - distribution - end - end - - def yum_targets - env_yum_targets = (ENV["YUM_TARGETS"] || "").split(",") - if env_yum_targets.empty? - available_yum_targets - else - available_yum_targets.select do |distribution, distribution_version| - env_yum_targets.any? do |env_yum_target| - env_yum_target.start_with?("#{distribution}-#{distribution_version}") - end - end - end - end - - def yum_distributions - yum_targets.collect(&:first).uniq - end - - def yum_architectures - [ - "aarch64", - "x86_64", - ] - end - - def signed_rpm?(rpm) - IO.pipe do |input, output| - system("rpm", "--checksig", rpm, out: output) - output.close - signature = input.gets.sub(/\A#{Regexp.escape(rpm)}: /, "") - signature.split.include?("signatures") - end - end - - def sign_rpms(directory) - thread_pool = ThreadPool.new(:gpg) do |rpm| - unless signed_rpm?(rpm) - sh("rpm", - "-D", "_gpg_name #{gpg_key_id}", - "-D", "__gpg_check_password_cmd /bin/true true", - "--resign", - rpm, - out: default_output, - verbose: verbose?) - end - end - Dir.glob("#{directory}/**/*.rpm") do |rpm| - thread_pool << rpm - end - thread_pool.join - end - - def rpm_sign(directory) - unless system("rpm", "-q", - rpm_gpg_key_package_name(gpg_key_id), - out: IO::NULL) - gpg_key = Tempfile.new(["apache-arrow-binary", ".asc"]) - sh("gpg", - "--armor", - "--export", gpg_key_id, - out: gpg_key.path, - verbose: verbose?) - sh("rpm", - "--import", gpg_key.path, - out: default_output, - verbose: verbose?) - gpg_key.close! - end - - yum_targets.each do |distribution, distribution_version| - source_dir = [ - directory, - distribution, - distribution_version, - ].join("/") - sign_rpms(source_dir) - end - end - - def define_rpm_tasks - directory rpm_dir - - namespace :rpm do - desc "Copy RPM packages" - task :copy => rpm_dir do - yum_targets.each do |distribution, distribution_version| - progress_label = "Copying: #{distribution} #{distribution_version}" - progress_reporter = ProgressReporter.new(progress_label) - - destination_prefix = [ - rpm_dir, - distribution, - distribution_version, - ].join("/") - source_dir_prefix = - "#{artifacts_dir}/#{distribution}-#{distribution_version}" - Dir.glob("#{source_dir_prefix}*/**/*") do |path| - next if File.directory?(path) - base_name = File.basename(path) - type = base_name.split(".")[-2] - destination_paths = [] - case type - when "src" - destination_paths << [ - destination_prefix, - "Source", - "SPackages", - base_name, - ].join("/") - when "noarch" - yum_architectures.each do |architecture| - destination_paths << [ - destination_prefix, - architecture, - "Packages", - base_name, - ].join("/") - end - else - destination_paths << [ - destination_prefix, - type, - "Packages", - base_name, - ].join("/") - end - destination_paths.each do |destination_path| - copy_artifact(path, - destination_path, - progress_reporter) - end - case base_name - when /\A(apache-arrow-release)-.*\.noarch\.rpm\z/ - package_name = $1 - latest_release_package_path = [ - destination_prefix, - "#{package_name}-latest.rpm" - ].join("/") - copy_artifact(path, - latest_release_package_path, - progress_reporter) - end - end - - progress_reporter.finish - end - end - - desc "Sign RPM packages" - task :sign do - rpm_sign(rpm_dir) - yum_targets.each do |distribution, distribution_version| - source_dir = [ - rpm_dir, - distribution, - distribution_version, - ].join("/") - sign_dir("#{distribution}-#{distribution_version}", - source_dir) - end - end - - desc "Upload RPM packages" - task :upload do - yum_distributions.each do |distribution| - distribution_dir = "#{rpm_dir}/#{distribution}" - distribution_label = yum_distribution_label(distribution) - uploader = BintrayUploader.new(repository: bintray_repository, - distribution: distribution, - distribution_label: distribution_label, - version: version, - rc: rc, - source: distribution_dir, - user: bintray_user, - api_key: bintray_api_key) - uploader.upload - end - end - end - - desc "Release RPM packages" - rpm_tasks = [ - "rpm:copy", - "rpm:sign", - "rpm:upload", - ] - task :rpm => rpm_tasks - end - - def yum_update(repositories_dir) - yum_distributions.each do |distribution| - distribution_dir = "#{repositories_dir}/#{distribution}" - Dir.glob("#{distribution_dir}/**/repodata") do |repodata| - rm_rf(repodata, verbose: verbose?) - end - end - - yum_targets.each do |distribution, distribution_version| - base_dir = [ - repositories_dir, - distribution, - distribution_version, - ].join("/") - base_dir = Pathname(base_dir) - next unless base_dir.directory? - base_dir.glob("*") do |arch_dir| - next unless arch_dir.directory? - sh(ENV["CREATEREPO"] || "createrepo", - "--update", - arch_dir.to_s, - out: default_output, - verbose: verbose?) - end - end - end - - def define_yum_rc_tasks - directory yum_rc_repositories_dir - - namespace :yum do - namespace :rc do - desc "Download RPM files for RC Yum repositories" - task :download => yum_rc_repositories_dir do - yum_distributions.each do |distribution| - distribution_dir = "#{yum_rc_repositories_dir}/#{distribution}" - download_distribution(distribution, - distribution_dir, - with_source_repository: true) - end - end - - desc "Update RC Yum repositories" - task :update => yum_rc_repositories_dir do - yum_update(yum_rc_repositories_dir) - yum_targets.each do |distribution, distribution_version| - base_dir = [ - yum_rc_repositories_dir, - distribution, - distribution_version, - ].join("/") - base_dir = Pathname(base_dir) - next unless base_dir.directory? - base_dir.glob("*") do |arch_dir| - next unless arch_dir.directory? - sign_label = - "#{distribution}-#{distribution_version} #{arch_dir.basename}" - sign_dir(sign_label, - arch_dir.to_s) - end - end - end - - desc "Upload RC Yum repositories" - task :upload => yum_rc_repositories_dir do - yum_targets.each do |distribution, distribution_version| - distribution_label = yum_distribution_label(distribution) - base_dir = [ - yum_rc_repositories_dir, - distribution, - distribution_version, - ].join("/") - base_dir = Pathname(base_dir) - base_dir.glob("**/repodata") do |repodata_dir| - relative_dir = [ - distribution_version, - repodata_dir.relative_path_from(base_dir).to_s - ].join("/") - uploader = - BintrayUploader.new(repository: bintray_repository, - distribution: distribution, - distribution_label: distribution_label, - version: version, - rc: rc, - source: repodata_dir.to_s, - destination_prefix: "#{relative_dir}/", - user: bintray_user, - api_key: bintray_api_key) - uploader.upload - end - end - end - end - - desc "Release RC Yum packages" - yum_rc_tasks = [ - "yum:rc:download", - "yum:rc:update", - "yum:rc:upload", - ] - task :rc => yum_rc_tasks - end - end - - def define_yum_release_tasks - directory yum_release_repositories_dir - - namespace :yum do - namespace :release do - desc "Download RC Yum repositories" - task :download => yum_release_repositories_dir do - yum_distributions.each do |distribution| - distribution_dir = "#{yum_release_repositories_dir}/#{distribution}" - download_distribution(distribution, distribution_dir) - end - end - - desc "Upload release Yum repositories" - task :upload => yum_release_repositories_dir do - yum_distributions.each do |distribution| - distribution_dir = "#{yum_release_repositories_dir}/#{distribution}" - distribution_label = yum_distribution_label(distribution) - uploader = BintrayUploader.new(repository: bintray_repository, - distribution: distribution, - distribution_label: distribution_label, - version: version, - source: distribution_dir, - user: bintray_user, - api_key: bintray_api_key) - uploader.upload - end - end - end - - desc "Release Yum packages" - yum_release_tasks = [ - "yum:release:download", - "yum:release:upload", - ] - task :release => yum_release_tasks - end - end - - def define_yum_tasks - define_rpm_tasks - define_yum_rc_tasks - define_yum_release_tasks - end - - def define_generic_data_rc_tasks(label, - id, - rc_dir, - target_files_glob) - directory rc_dir - - namespace id do - namespace :rc do - desc "Copy #{label} packages" - task :copy => rc_dir do - progress_label = "Copying: #{label}" - progress_reporter = ProgressReporter.new(progress_label) - - Pathname(artifacts_dir).glob(target_files_glob) do |path| - next if path.directory? - destination_path = [ - rc_dir, - path.basename.to_s, - ].join("/") - copy_artifact(path, destination_path, progress_reporter) - end - - progress_reporter.finish - end - - desc "Sign #{label} packages" - task :sign => rc_dir do - sign_dir(label, rc_dir) - end - - desc "Upload #{label} packages" - task :upload do - uploader = BintrayUploader.new(repository: bintray_repository, - distribution: id.to_s, - distribution_label: label, - version: version, - rc: rc, - source: rc_dir, - destination_prefix: "#{full_version}/", - user: bintray_user, - api_key: bintray_api_key) - uploader.upload - end - end - - desc "Release RC #{label} packages" - rc_tasks = [ - "#{id}:rc:copy", - "#{id}:rc:sign", - "#{id}:rc:upload", - ] - task :rc => rc_tasks - end - end - - def define_generic_data_release_tasks(label, id, release_dir) - directory release_dir - - namespace id do - namespace :release do - desc "Download RC #{label} packages" - task :download => release_dir do - download_distribution(id.to_s, release_dir) - end - - desc "Upload release #{label} packages" - task :upload => release_dir do - packages_dir = "#{release_dir}/#{full_version}" - uploader = BintrayUploader.new(repository: bintray_repository, - distribution: id.to_s, - distribution_label: label, - version: version, - source: packages_dir, - destination_prefix: "#{version}/", - user: bintray_user, - api_key: bintray_api_key) - uploader.upload - end - end - - desc "Release #{label} packages" - release_tasks = [ - "#{id}:release:download", - "#{id}:release:upload", - ] - task :release => release_tasks - end - end - - def define_generic_data_tasks(label, - id, - rc_dir, - release_dir, - target_files_glob) - define_generic_data_rc_tasks(label, id, rc_dir, target_files_glob) - define_generic_data_release_tasks(label, id, release_dir) - end - - def define_python_tasks - define_generic_data_tasks("Python", - :python, - "#{rc_dir}/python/#{full_version}", - "#{release_dir}/python/#{full_version}", - "{conda-*,wheel-*,python-sdist}/**/*") - end - - def define_nuget_tasks - define_generic_data_tasks("NuGet", - :nuget, - "#{rc_dir}/nuget/#{full_version}", - "#{release_dir}/nuget/#{full_version}", - "nuget/**/*") - end - - def define_summary_tasks - namespace :summary do - desc "Show RC summary" - task :rc do - puts(<<-SUMMARY) -Success! The release candidate binaries are available here: - https://bintray.com/#{bintray_repository}/debian-rc/#{full_version} - https://bintray.com/#{bintray_repository}/ubuntu-rc/#{full_version} - https://bintray.com/#{bintray_repository}/centos-rc/#{full_version} - https://bintray.com/#{bintray_repository}/python-rc/#{full_version} - https://bintray.com/#{bintray_repository}/nuget-rc/#{full_version} - SUMMARY - end - - desc "Show release summary" - task :release do - puts(<<-SUMMARY) -Success! The release binaries are available here: - https://bintray.com/#{bintray_repository}/debian/#{version} - https://bintray.com/#{bintray_repository}/ubuntu/#{version} - https://bintray.com/#{bintray_repository}/centos/#{version} - https://bintray.com/#{bintray_repository}/python/#{version} - https://bintray.com/#{bintray_repository}/nuget/#{version} - SUMMARY - end - end - end -end diff --git a/dev/release/binary/.dockerignore b/dev/release/binary/.dockerignore deleted file mode 100644 index f2c46d8ce245..000000000000 --- a/dev/release/binary/.dockerignore +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -/tmp/ diff --git a/dev/release/binary/Dockerfile b/dev/release/binary/Dockerfile deleted file mode 100644 index 26ef3f9e8c05..000000000000 --- a/dev/release/binary/Dockerfile +++ /dev/null @@ -1,68 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -FROM ubuntu:18.04 - -ENV DEBIAN_FRONTEND noninteractive - -ARG DEBUG - -RUN \ - quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ - apt update ${quiet} && \ - apt install -y -V ${quiet} \ - apt-utils \ - createrepo \ - devscripts \ - gpg \ - locales \ - openssh-server \ - rake \ - rpm \ - ruby \ - sudo && \ - apt clean && \ - rm -rf /var/lib/apt/lists/* - -RUN locale-gen en_US.UTF-8 - -RUN mkdir -p /run/sshd -RUN echo "StreamLocalBindUnlink yes" >> /etc/ssh/sshd_config - -ENV ARROW_USER arrow -ENV ARROW_UID 10000 - -RUN \ - groupadd --gid ${ARROW_UID} ${ARROW_USER} && \ - useradd --uid ${ARROW_UID} --gid ${ARROW_UID} --create-home ${ARROW_USER} && \ - mkdir -p /home/arrow/.gnupg /home/arrow/.ssh && \ - chown -R arrow: /home/arrow/.gnupg /home/arrow/.ssh && \ - chmod -R og-rwx /home/arrow/.gnupg /home/arrow/.ssh && \ - echo "${ARROW_USER} ALL=(ALL:ALL) NOPASSWD:ALL" | \ - EDITOR=tee visudo -f /etc/sudoers.d/arrow - -COPY id_rsa.pub /home/arrow/.ssh/authorized_keys -RUN \ - chown -R arrow: /home/arrow/.ssh && \ - chmod -R og-rwx /home/arrow/.ssh - -COPY runner.sh /home/arrow/runner.sh -RUN \ - chown -R arrow: /home/arrow/runner.sh && \ - chmod +x /home/arrow/runner.sh - -EXPOSE 22 diff --git a/dev/release/binary/runner.sh b/dev/release/binary/runner.sh deleted file mode 100755 index 5cf5033d68b5..000000000000 --- a/dev/release/binary/runner.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -u - -export LANG=C - -target_dir=/host/binary/tmp -original_owner=$(stat --format=%u ${target_dir}) -original_group=$(stat --format=%g ${target_dir}) - -sudo -H chown -R ${USER}: ${target_dir} -restore_owner() { - sudo -H chown -R ${original_owner}:${original_group} ${target_dir} -} -trap restore_owner EXIT - -cd /host - -"$@" diff --git a/dev/release/check-rat-report.py b/dev/release/check-rat-report.py deleted file mode 100644 index e30d72bddd7f..000000000000 --- a/dev/release/check-rat-report.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/python -############################################################################## -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -############################################################################## -import fnmatch -import re -import sys -import xml.etree.ElementTree as ET - -if len(sys.argv) != 3: - sys.stderr.write("Usage: %s exclude_globs.lst rat_report.xml\n" % - sys.argv[0]) - sys.exit(1) - -exclude_globs_filename = sys.argv[1] -xml_filename = sys.argv[2] - -globs = [line.strip() for line in open(exclude_globs_filename, "r")] - -tree = ET.parse(xml_filename) -root = tree.getroot() -resources = root.findall('resource') - -all_ok = True -for r in resources: - approvals = r.findall('license-approval') - if not approvals or approvals[0].attrib['name'] == 'true': - continue - clean_name = re.sub('^[^/]+/', '', r.attrib['name']) - excluded = False - for g in globs: - if fnmatch.fnmatch(clean_name, g): - excluded = True - break - if not excluded: - sys.stdout.write("NOT APPROVED: %s (%s): %s\n" % ( - clean_name, r.attrib['name'], approvals[0].attrib['name'])) - all_ok = False - -if not all_ok: - sys.exit(1) - -print('OK') -sys.exit(0) diff --git a/dev/release/download_rc_binaries.py b/dev/release/download_rc_binaries.py deleted file mode 100755 index 5ed8ece77830..000000000000 --- a/dev/release/download_rc_binaries.py +++ /dev/null @@ -1,173 +0,0 @@ -#!/usr/bin/env python - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import re - -import argparse -import concurrent.futures as cf -import functools -import hashlib -import json -import os -import subprocess -import urllib.request - - -BINTRAY_API_ROOT = "https://bintray.com/api/v1" -BINTRAY_DL_ROOT = "https://dl.bintray.com" -BINTRAY_REPO = os.getenv('BINTRAY_REPOSITORY', 'apache/arrow') -DEFAULT_PARALLEL_DOWNLOADS = 8 - - -class Bintray: - - def __init__(self, repo=BINTRAY_REPO): - self.repo = repo - - def get_file_list(self, package, version): - url = os.path.join(BINTRAY_API_ROOT, 'packages', self.repo, package, - 'versions', version, 'files') - request = urllib.request.urlopen(url).read() - return json.loads(request) - - def download_files(self, files, dest=None, num_parallel=None, - re_match=None): - """ - Download files from Bintray in parallel. If file already exists, will - overwrite if the checksum does not match what Bintray says it should be - - Parameters - ---------- - files : List[Dict] - File listing from Bintray - dest : str, default None - Defaults to current working directory - num_parallel : int, default 8 - Number of files to download in parallel. If set to None, uses - default - """ - if dest is None: - dest = os.getcwd() - if num_parallel is None: - num_parallel = DEFAULT_PARALLEL_DOWNLOADS - - if re_match is not None: - regex = re.compile(re_match) - files = [x for x in files if regex.match(x['path'])] - - if num_parallel == 1: - for path in files: - self._download_file(dest, path) - else: - parallel_map_terminate_early( - functools.partial(self._download_file, dest), - files, - num_parallel - ) - - def _download_file(self, dest, info): - relpath = info['path'] - - base, filename = os.path.split(relpath) - - dest_dir = os.path.join(dest, base) - os.makedirs(dest_dir, exist_ok=True) - - dest_path = os.path.join(dest_dir, filename) - - if os.path.exists(dest_path): - with open(dest_path, 'rb') as f: - sha256sum = hashlib.sha256(f.read()).hexdigest() - if sha256sum == info['sha256']: - print('Local file {} sha256 matches, skipping' - .format(dest_path)) - return - else: - print('Local file sha256 does not match, overwriting') - - print("Downloading {} to {}".format(relpath, dest_path)) - - bintray_abspath = os.path.join(BINTRAY_DL_ROOT, self.repo, relpath) - - cmd = [ - 'curl', '--fail', '--location', '--retry', '5', - '--output', dest_path, bintray_abspath - ] - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - stdout, stderr = proc.communicate() - if proc.returncode != 0: - raise Exception("Downloading {} failed\nstdout: {}\nstderr: {}" - .format(relpath, stdout, stderr)) - - -def parallel_map_terminate_early(f, iterable, num_parallel): - tasks = [] - with cf.ProcessPoolExecutor(num_parallel) as pool: - for v in iterable: - tasks.append(pool.submit(functools.partial(f, v))) - - for task in cf.as_completed(tasks): - if task.exception() is not None: - e = task.exception() - for task in tasks: - task.cancel() - raise e - - -ARROW_PACKAGE_TYPES = ['centos', 'debian', 'nuget', 'python', 'ubuntu'] - - -def download_rc_binaries(version, rc_number, re_match=None, dest=None, - num_parallel=None, target_package_type=None): - bintray = Bintray() - - version_string = '{}-rc{}'.format(version, rc_number) - if target_package_type: - package_types = [target_package_type] - else: - package_types = ARROW_PACKAGE_TYPES - for package_type in package_types: - files = bintray.get_file_list('{}-rc'.format(package_type), - version_string) - bintray.download_files(files, re_match=re_match, dest=dest, - num_parallel=num_parallel) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description='Download release candidate binaries' - ) - parser.add_argument('version', type=str, help='The version number') - parser.add_argument('rc_number', type=int, - help='The release candidate number, e.g. 0, 1, etc') - parser.add_argument('-e', '--regexp', type=str, default=None, - help=('Regular expression to match on file names ' - 'to only download certain files')) - parser.add_argument('--dest', type=str, default=os.getcwd(), - help='The output folder for the downloaded files') - parser.add_argument('--num_parallel', type=int, default=8, - help='The number of concurrent downloads to do') - parser.add_argument('--package_type', type=str, default=None, - help='The package type to be downloaded') - args = parser.parse_args() - - download_rc_binaries(args.version, args.rc_number, dest=args.dest, - re_match=args.regexp, num_parallel=args.num_parallel, - target_package_type=args.package_type) diff --git a/dev/release/post-01-upload.sh b/dev/release/post-01-upload.sh deleted file mode 100755 index 4f8053de8c94..000000000000 --- a/dev/release/post-01-upload.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -set -e -set -u - -if [ "$#" -ne 2 ]; then - echo "Usage: $0 " - exit -fi - -version=$1 -rc=$2 - -tmp_dir=tmp-apache-arrow-dist - -echo "Recreate temporary directory: ${tmp_dir}" -rm -rf ${tmp_dir} -mkdir -p ${tmp_dir} - -echo "Clone dev dist repository" -svn \ - co \ - https://dist.apache.org/repos/dist/dev/arrow/apache-arrow-${version}-rc${rc} \ - ${tmp_dir}/dev - -echo "Clone release dist repository" -svn co https://dist.apache.org/repos/dist/release/arrow ${tmp_dir}/release - -echo "Copy ${version}-rc${rc} to release working copy" -release_version=arrow-${version} -mkdir -p ${tmp_dir}/release/${release_version} -cp -r ${tmp_dir}/dev/* ${tmp_dir}/release/${release_version}/ -svn add ${tmp_dir}/release/${release_version} - -echo "Keep only the three most recent versions" -old_releases=$( - svn ls ${tmp_dir}/release/ | \ - grep '^arrow-' | \ - sort --version-sort --reverse | \ - tail -n +4 -) -for old_release_version in $old_releases; do - echo "Remove old release ${old_release_version}" - svn delete ${tmp_dir}/release/${old_release_version} -done - -echo "Commit release" -svn ci -m "Apache Arrow ${version}" ${tmp_dir}/release - -echo "Clean up" -rm -rf ${tmp_dir} - -echo "Success! The release is available here:" -echo " https://dist.apache.org/repos/dist/release/arrow/${release_version}" diff --git a/dev/release/post-02-binary.sh b/dev/release/post-02-binary.sh deleted file mode 100755 index 9f531afad9fd..000000000000 --- a/dev/release/post-02-binary.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e -set -o pipefail - -SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -if [ "$#" -ne 2 ]; then - echo "Usage: $0 " - exit -fi - -version=$1 -rc=$2 - -cd "${SOURCE_DIR}" - -: ${BINTRAY_REPOSITORY_CUSTOM:=${BINTRAY_REPOSITORY:-}} - -if [ ! -f .env ]; then - echo "You must create $(pwd)/.env" - echo "You can use $(pwd)/.env.example as template" - exit 1 -fi -. .env - -if [ -n "${BINTRAY_REPOSITORY_CUSTOM}" ]; then - BINTRAY_REPOSITORY=${BINTRAY_REPOSITORY_CUSTOM} -fi - -. binary-common.sh - -# By default deploy all artifacts. -# To deactivate one category, deactivate the category and all of its dependents. -# To explicitly select one category, set DEPLOY_DEFAULT=0 DEPLOY_X=1. -: ${DEPLOY_DEFAULT:=1} -: ${DEPLOY_CENTOS:=${DEPLOY_DEFAULT}} -: ${DEPLOY_DEBIAN:=${DEPLOY_DEFAULT}} -: ${DEPLOY_NUGET:=${DEPLOY_DEFAULT}} -: ${DEPLOY_PYTHON:=${DEPLOY_DEFAULT}} -: ${DEPLOY_UBUNTU:=${DEPLOY_DEFAULT}} - -rake_tasks=() -apt_targets=() -yum_targets=() -if [ ${DEPLOY_DEBIAN} -gt 0 ]; then - rake_tasks+=(apt:release) - apt_targets+=(debian) -fi -if [ ${DEPLOY_UBUNTU} -gt 0 ]; then - rake_tasks+=(apt:release) - apt_targets+=(ubuntu) -fi -if [ ${DEPLOY_CENTOS} -gt 0 ]; then - rake_tasks+=(yum:release) - yum_targets+=(centos) -fi -if [ ${DEPLOY_NUGET} -gt 0 ]; then - rake_tasks+=(nuget:release) -fi -if [ ${DEPLOY_PYTHON} -gt 0 ]; then - rake_tasks+=(python:release) -fi -rake_tasks+=(summary:release) - -tmp_dir=binary/tmp -mkdir -p "${tmp_dir}" - -docker_run \ - ./runner.sh \ - rake \ - "${rake_tasks[@]}" \ - APT_TARGETS=$(IFS=,; echo "${apt_targets[*]}") \ - ARTIFACTS_DIR="${tmp_dir}/artifacts" \ - BINTRAY_REPOSITORY=${BINTRAY_REPOSITORY} \ - RC=${rc} \ - VERSION=${version} \ - YUM_TARGETS=$(IFS=,; echo "${yum_targets[*]}") diff --git a/dev/release/post-03-website.sh b/dev/release/post-03-website.sh deleted file mode 100755 index b427142ea989..000000000000 --- a/dev/release/post-03-website.sh +++ /dev/null @@ -1,266 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e -set -u - -SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -ARROW_DIR="${SOURCE_DIR}/../.." -ARROW_SITE_DIR="${ARROW_DIR}/../arrow-site" - -if [ "$#" -ne 2 ]; then - echo "Usage: $0 " - exit 1 -fi - -previous_version=$1 -version=$2 - -branch_name=release-note-${version} -release_dir="${ARROW_SITE_DIR}/_release" -announce_file="${release_dir}/${version}.md" -versions_yml="${ARROW_SITE_DIR}/_data/versions.yml" - -pushd "${ARROW_SITE_DIR}" -git checkout master -git checkout -b ${branch_name} -popd - -pushd "${ARROW_DIR}" - -release_date=$(LANG=C date "+%-d %B %Y") -previous_tag_date=$(git log -n 1 --pretty=%aI apache-arrow-${previous_version}) -rough_previous_release_date=$(date --date "${previous_tag_date}" +%s) -rough_release_date=$(date +%s) -rough_n_development_months=$(( - (${rough_release_date} - ${rough_previous_release_date}) / (60 * 60 * 24 * 30) -)) - -git_tag=apache-arrow-${version} -git_range=apache-arrow-${previous_version}..${git_tag} - -committers_command_line="git shortlog -csn ${git_range}" -contributors_command_line="git shortlog -sn ${git_range}" - -committers=$(${committers_command_line}) -contributors=$(${contributors_command_line}) - -n_commits=$(git log --pretty=oneline ${git_range} | wc -l) -n_contributors=$(${contributors_command_line} | wc -l) - -git_tag_hash=$(git log -n 1 --pretty=%H ${git_tag}) - -popd - -pushd "${ARROW_SITE_DIR}" - -# Add announce for the current version -cat < "${announce_file}" ---- -layout: default -title: Apache Arrow ${version} Release -permalink: /release/${version}.html ---- - - -# Apache Arrow ${version} (${release_date}) - -This is a major release covering more than ${rough_n_development_months} months of development. - -## Download - -* [**Source Artifacts**][1] -* **Binary Artifacts** - * [For CentOS][2] - * [For Debian][3] - * [For Python][4] - * [For Ubuntu][5] -* [Git tag][6] - -## Contributors - -This release includes ${n_commits} commits from ${n_contributors} distinct contributors. - -\`\`\`console -$ ${contributors_command_line} -ANNOUNCE - -echo "${contributors}" >> "${announce_file}" - -cat <> "${announce_file}" -\`\`\` - -## Patch Committers - -The following Apache committers merged contributed patches to the repository. - -\`\`\`console -$ ${committers_command_line} -ANNOUNCE - -echo "${committers}" >> "${announce_file}" - -cat <> "${announce_file}" -\`\`\` - -## Changelog - -ANNOUNCE - -archery release changelog generate ${version} | \ - sed -e 's/^#/##/g' >> "${announce_file}" - -cat <> "${announce_file}" -[1]: https://www.apache.org/dyn/closer.lua/arrow/arrow-${version}/ -[2]: https://bintray.com/apache/arrow/centos/${version}/ -[3]: https://bintray.com/apache/arrow/debian/${version}/ -[4]: https://bintray.com/apache/arrow/python/${version}/ -[5]: https://bintray.com/apache/arrow/ubuntu/${version}/ -[6]: https://github.com/apache/arrow/releases/tag/apache-arrow-${version} -ANNOUNCE -git add "${announce_file}" - - -# Update index -pushd "${release_dir}" - -index_file=index.md -rm -f ${index_file} -announce_files="$(ls | sort --version-sort --reverse)" -cat < ${index_file} ---- -layout: default -title: Releases -permalink: /release/index.html ---- - - -# Apache Arrow Releases - -Navigate to the release page for downloads and the changelog. - -INDEX - -i=0 -for md_file in ${announce_files}; do - i=$((i + 1)) - title=$(grep '^# Apache Arrow' ${md_file} | sed -e 's/^# Apache Arrow //') - echo "* [${title}][${i}]" >> ${index_file} -done -echo >> ${index_file} - -i=0 -for md_file in ${announce_files}; do - i=$((i + 1)) - html_file=$(echo ${md_file} | sed -e 's/md$/html/') - echo "[${i}]: {{ site.baseurl }}/release/${html_file}" >> ${index_file} -done - -git add ${index_file} - -popd - - -# Update versions.yml -pinned_version=$(echo ${version} | sed -e 's/\.[^.]*$/.*/') - -apache_download_url=https://downloads.apache.org - -cat < "${versions_yml}" -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to you under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Database of the current version -# -current: - number: '${version}' - pinned_number: '${pinned_version}' - date: '${release_date}' - git-tag: '${git_tag_hash}' - github-tag-link: 'https://github.com/apache/arrow/releases/tag/${git_tag}' - release-notes: 'https://arrow.apache.org/release/${version}.html' - mirrors: 'https://www.apache.org/dyn/closer.lua/arrow/arrow-${version}/' - tarball-name: 'apache-arrow-${version}.tar.gz' - tarball-url: 'https://www.apache.org/dyn/closer.lua?action=download&filename=arrow/arrow-${version}/apache-arrow-${version}.tar.gz' - java-artifacts: 'http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.arrow%22%20AND%20v%3A%22${version}%22' - asc: '${apache_download_url}/arrow/arrow-${version}/apache-arrow-${version}.tar.gz.asc' - sha256: '${apache_download_url}/arrow/arrow-${version}/apache-arrow-${version}.tar.gz.sha256' - sha512: '${apache_download_url}/arrow/arrow-${version}/apache-arrow-${version}.tar.gz.sha512' -YAML -git add "${versions_yml}" - -git commit -m "[Website] Add release note for ${version}" -git push -u origin ${branch_name} - -github_url=$(git remote get-url origin | \ - sed \ - -e 's,^git@github.com:,https://github.com/,' \ - -e 's,\.git$,,') - -echo "Success!" -echo "Create a pull request:" -echo " ${github_url}/pull/new/${branch_name}" - -popd diff --git a/dev/release/post-04-ruby.sh b/dev/release/post-04-ruby.sh deleted file mode 100755 index 7bc42aaeeb1f..000000000000 --- a/dev/release/post-04-ruby.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -# -*- indent-tabs-mode: nil; sh-indentation: 2; sh-basic-offset: 2 -*- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -set -e -set -o pipefail - -SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -if [ "$#" -ne 1 ]; then - echo "Usage: $0 " - exit -fi - -version=$1 -archive_name=apache-arrow-${version} -tar_gz=${archive_name}.tar.gz - -rm -f ${tar_gz} -curl \ - --remote-name \ - --fail \ - https://downloads.apache.org/arrow/arrow-${version}/${tar_gz} -rm -rf ${archive_name} -tar xf ${tar_gz} -modules=() -for module in ${archive_name}/ruby/red-*; do - pushd ${module} - rake release - modules+=($(basename ${module})) - popd -done -rm -rf ${archive_name} -rm -f ${tar_gz} - -echo "Success! The released RubyGems are available here:" -for module in ${modules[@]}; do - echo " https://rubygems.org/gems/${module}/versions/${version}" -done diff --git a/dev/release/post-05-js.sh b/dev/release/post-05-js.sh deleted file mode 100755 index 3df07e43ea1e..000000000000 --- a/dev/release/post-05-js.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash -# -*- indent-tabs-mode: nil; sh-indentation: 2; sh-basic-offset: 2 -*- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -set -e - -SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -if [ "$#" -ne 1 ]; then - echo "Usage: $0 " - exit -fi - -version=$1 -archive_name=apache-arrow-${version} -tar_gz=${archive_name}.tar.gz - -rm -f ${tar_gz} -curl \ - --remote-name \ - --fail \ - https://downloads.apache.org/arrow/arrow-${version}/${tar_gz} -rm -rf ${archive_name} -tar xf ${tar_gz} -pushd ${archive_name}/js -./npm-release.sh -popd -rm -rf ${archive_name} -rm -f ${tar_gz} - -echo "Success! The released npm packages are available here:" -echo " https://www.npmjs.com/package/apache-arrow/v/${version}" diff --git a/dev/release/post-06-csharp.sh b/dev/release/post-06-csharp.sh deleted file mode 100755 index e9572025ab50..000000000000 --- a/dev/release/post-06-csharp.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash -# -*- indent-tabs-mode: nil; sh-indentation: 2; sh-basic-offset: 2 -*- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -set -eux - -SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -if [ "$#" -ne 1 ]; then - echo "Usage: $0 " - exit -fi - -version=$1 - -if [ -z "${NUGET_API_KEY}" ]; then - echo "NUGET_API_KEY is empty" - exit 1 -fi - -base_names=() -base_names+=(Apache.Arrow.${version}) -base_names+=(Apache.Arrow.Flight.${version}) -base_names+=(Apache.Arrow.Flight.AspNetCore.${version}) -for base_name in ${base_names[@]}; do - for extension in nupkg snupkg; do - path=${base_name}.${extension} - rm -f ${path} - curl \ - --fail \ - --location \ - --remote-name \ - https://apache.bintray.com/arrow/nuget/${version}/${path} - done - dotnet nuget push \ - ${base_name}.nupkg \ - -k ${NUGET_API_KEY} \ - -s https://api.nuget.org/v3/index.json -done - -echo "Success! The released NuGet package is available here:" -echo " https://www.nuget.org/packages/Apache.Arrow/${version}" diff --git a/dev/release/post-07-rust.sh b/dev/release/post-07-rust.sh deleted file mode 100755 index 3c94607565f9..000000000000 --- a/dev/release/post-07-rust.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/bash -# -*- indent-tabs-mode: nil; sh-indentation: 2; sh-basic-offset: 2 -*- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -set -e -set -o pipefail - -SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -if [ "$#" -ne 1 ]; then - echo "Usage: $0 " - exit -fi - -version=$1 - -: ${INSTALL_RUST:=no} - -if [ "${INSTALL_RUST}" == "yes" ]; then - export RUSTUP_HOME="$(pwd)/release-rustup" - export CARGO_HOME="${RUSTUP_HOME}" - rm -rf "${RUSTUP_HOME}" - curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path - export PATH="${RUSTUP_HOME}/bin:$PATH" - source "${RUSTUP_HOME}/env" - rustup default stable - cargo login -fi - -archive_name=apache-arrow-${version} -tar_gz=${archive_name}.tar.gz -rm -f ${tar_gz} -curl \ - --remote-name \ - --fail \ - https://downloads.apache.org/arrow/arrow-${version}/${tar_gz} -rm -rf ${archive_name} -tar xf ${tar_gz} -modules=() -for cargo_toml in ${archive_name}/rust/*/Cargo.toml; do - module_dir=$(dirname ${cargo_toml}) - pushd ${module_dir} - cargo publish --allow-dirty - modules+=($(basename ${module_dir})) - popd -done -popd -rm -rf ${archive_name} -rm -f ${tar_gz} - -if [ "${INSTALL_RUST}" == "yes" ]; then - rm -rf "${RUSTUP_HOME}" -fi - -echo "Success! The released packages are available here:" -for module in ${modules[@]}; do - echo " https://crates.io/crates/${module}/${version}" -done diff --git a/dev/release/post-08-remove-rc.sh b/dev/release/post-08-remove-rc.sh deleted file mode 100755 index a02861364645..000000000000 --- a/dev/release/post-08-remove-rc.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -set -e -set -u -set -o pipefail - -SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -if [ "$#" -ne 1 ]; then - echo "Usage: $0 " - exit -fi - -version=$1 - -base_url=https://dist.apache.org/repos/dist/dev/arrow -pattern="^apache-arrow-${version}-rc" -paths=$() -if svn ls ${base_url} | grep "${pattern}" > /dev/null 2>&1; then - rc_paths=$(svn ls ${base_url} | grep "${pattern}") - rc_urls=() - for rc_path in ${rc_paths}; do - rc_urls+=(${base_url}/${rc_path}) - done - svn rm --message "Remove RC for ${version}" ${rc_urls[@]} - echo "Removed RC artifacts:" - for rc_url in ${rc_urls[@]}; do - echo " ${rc_url}" - done -else - echo "No RC artifacts at ${base_url}" -fi diff --git a/dev/release/post-09-docs.sh b/dev/release/post-09-docs.sh deleted file mode 100755 index c9f75b48b2c8..000000000000 --- a/dev/release/post-09-docs.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e -set -u - -SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -ARROW_DIR="${SOURCE_DIR}/../.." -ARROW_SITE_DIR="${ARROW_DIR}/../arrow-site" - -if [ "$#" -ne 1 ]; then - echo "Usage: $0 " - exit 1 -fi - -version=$1 -release_tag="apache-arrow-${version}" -branch_name=release-docs-${version} - -pushd "${ARROW_SITE_DIR}" -git checkout asf-site -git checkout -b ${branch_name} -rm -rf docs/* -git checkout docs/c_glib/index.html -popd - -pushd "${ARROW_DIR}" -git checkout "${release_tag}" - -archery docker run \ - -v "${ARROW_SITE_DIR}/docs:/build/docs" \ - -e ARROW_DOCS_VERSION="${version}" \ - -e UBUNTU=20.10 \ - ubuntu-docs - -: ${PUSH:=1} - -if [ ${PUSH} -gt 0 ]; then - pushd "${ARROW_SITE_DIR}" - git add docs - git commit -m "[Website] Update documentations for ${version}" - git push -u origin ${branch_name} - github_url=$(git remote get-url origin | \ - sed \ - -e 's,^git@github.com:,https://github.com/,' \ - -e 's,\.git$,,') - popd - - echo "Success!" - echo "Create a pull request:" - echo " ${github_url}/pull/new/${branch_name}" -fi diff --git a/dev/release/post-10-python.sh b/dev/release/post-10-python.sh deleted file mode 100755 index 0f7a480cde61..000000000000 --- a/dev/release/post-10-python.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e -set -o pipefail - -SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -if [ "$#" -ne 2 ]; then - echo "Usage: $0 " - exit -fi - -version=$1 -rc=$2 - -tmp=$(mktemp -d -t "arrow-post-python.XXXXX") -${PYTHON:-python} \ - "${SOURCE_DIR}/download_rc_binaries.py" \ - ${version} \ - ${rc} \ - --dest="${tmp}" \ - --package_type=python -twine upload ${tmp}/python-rc/${version}-rc${rc}/*.{whl,tar.gz} -rm -rf "${tmp}" - -echo "Success! The released PyPI packages are available here:" -echo " https://pypi.org/project/pyarrow/${version}" diff --git a/dev/release/post-11-java.sh b/dev/release/post-11-java.sh deleted file mode 100755 index d9dc32a7f557..000000000000 --- a/dev/release/post-11-java.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e -set -o pipefail - -SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -if [ "$#" -ne 1 ]; then - echo "Usage: $0 " - exit -fi - -version=$1 -archive_name=apache-arrow-${version} -tar_gz=${archive_name}.tar.gz - -rm -f ${tar_gz} -curl \ - --remote-name \ - --fail \ - https://downloads.apache.org/arrow/arrow-${version}/${tar_gz} -rm -rf ${archive_name} -tar xf ${tar_gz} - -# build the jni bindings similarly like the 01-perform.sh does -mkdir -p ${archive_name}/cpp/java-build -pushd ${archive_name}/cpp/java-build -cmake \ - -DARROW_GANDIVA=ON \ - -DARROW_GANDIVA_JAVA=ON \ - -DARROW_JNI=ON \ - -DARROW_ORC=ON \ - -DCMAKE_BUILD_TYPE=release \ - -G Ninja \ - .. -ninja -popd - -# go in the java subfolder -pushd ${archive_name}/java -# stage the artifacts using both the apache-release and arrow-jni profiles -mvn -Papache-release,arrow-jni -Darrow.cpp.build.dir=$(realpath ../cpp/java-build) deploy -popd - -echo "Success! The maven artifacts have been stated. Proceed with the following steps:" -echo "1. Login to the apache repository: https://repository.apache.org/#stagingRepositories" -echo "2. Select the arrow staging repository you just just created: orgapachearrow-100x" -echo "3. Click the \"close\" button" -echo "4. Once validation has passed, click the \"release\" button" -echo "" -echo "Note, that you must set up Maven to be able to publish to Apache's repositories." -echo "Read more at https://www.apache.org/dev/publishing-maven-artifacts.html." diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt deleted file mode 100644 index f3eb273adff4..000000000000 --- a/dev/release/rat_exclude_files.txt +++ /dev/null @@ -1,107 +0,0 @@ -*.npmrc -*.gitignore -.gitmodules -*_generated.h -*_generated.js -*_generated.ts -*.csv -*.json -*.snap -.github/ISSUE_TEMPLATE/*.md -.github/pull_request_template.md -ci/etc/rprofile -ci/etc/*.patch -ci/vcpkg/*.patch -CHANGELOG.md -dev/requirements*.txt -dev/archery/MANIFEST.in -dev/archery/requirements*.txt -dev/archery/archery/tests/fixtures/* -dev/archery/archery/crossbow/tests/fixtures/* -dev/release/rat_exclude_files.txt -dev/tasks/homebrew-formulae/apache-arrow.rb -dev/tasks/linux-packages/apache-arrow-apt-source/debian/apache-arrow-apt-source.install -dev/tasks/linux-packages/apache-arrow-apt-source/debian/compat -dev/tasks/linux-packages/apache-arrow-apt-source/debian/control -dev/tasks/linux-packages/apache-arrow-apt-source/debian/rules -dev/tasks/linux-packages/apache-arrow-apt-source/debian/source/format -dev/tasks/linux-packages/apache-arrow/debian/compat -dev/tasks/linux-packages/apache-arrow/debian/control.in -dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-1.0.install -dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-cuda-1.0.install -dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-dataset-1.0.install -dev/tasks/linux-packages/apache-arrow/debian/gir1.2-gandiva-1.0.install -dev/tasks/linux-packages/apache-arrow/debian/gir1.2-parquet-1.0.install -dev/tasks/linux-packages/apache-arrow/debian/gir1.2-plasma-1.0.install -dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install -dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-dev.install -dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.doc-base -dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.install -dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.links -dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib400.install -dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-dev.install -dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib-dev.install -dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib400.install -dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda400.install -dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-dev.install -dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-dev.install -dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.doc-base -dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.install -dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.links -dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib400.install -dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset400.install -dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-dev.install -dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight400.install -dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-dev.install -dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight-dev.install -dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight400.install -dev/tasks/linux-packages/apache-arrow/debian/libarrow-python400.install -dev/tasks/linux-packages/apache-arrow/debian/libarrow400.install -dev/tasks/linux-packages/apache-arrow/debian/libgandiva-dev.install -dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-dev.install -dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.doc-base -dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.install -dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.links -dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib400.install -dev/tasks/linux-packages/apache-arrow/debian/libgandiva400.install -dev/tasks/linux-packages/apache-arrow/debian/libparquet-dev.install -dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-dev.install -dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.doc-base -dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.install -dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.links -dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib400.install -dev/tasks/linux-packages/apache-arrow/debian/libparquet400.install -dev/tasks/linux-packages/apache-arrow/debian/libplasma-dev.install -dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-dev.install -dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.doc-base -dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.install -dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.links -dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib400.install -dev/tasks/linux-packages/apache-arrow/debian/libplasma400.install -dev/tasks/linux-packages/apache-arrow/debian/patches/series -dev/tasks/linux-packages/apache-arrow/debian/plasma-store-server.install -dev/tasks/linux-packages/apache-arrow/debian/rules -dev/tasks/linux-packages/apache-arrow/debian/source/format -dev/tasks/linux-packages/apache-arrow/debian/watch -dev/tasks/requirements*.txt -dev/tasks/conda-recipes/* -pax_global_header -MANIFEST.in -__init__.pxd -__init__.py -requirements.txt -*.html -*.sgml -*.css -*.png -*.ico -*.svg -*.devhelp2 -*.scss -.gitattributes -arrow/test/data/*.csv -rust-toolchain -arrow-flight/src/arrow.flight.protocol.rs -ballista/rust/benchmarks/tpch/queries/q*.sql -ballista/rust/scheduler/testdata/* -ballista/ui/scheduler/yarn.lock diff --git a/dev/release/run-rat.sh b/dev/release/run-rat.sh deleted file mode 100755 index 94fa55fbe097..000000000000 --- a/dev/release/run-rat.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -RAT_VERSION=0.13 - -# download apache rat -if [ ! -f apache-rat-${RAT_VERSION}.jar ]; then - curl -s https://repo1.maven.org/maven2/org/apache/rat/apache-rat/${RAT_VERSION}/apache-rat-${RAT_VERSION}.jar > apache-rat-${RAT_VERSION}.jar -fi - -RAT="java -jar apache-rat-${RAT_VERSION}.jar -x " - -RELEASE_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd) - -# generate the rat report -$RAT $1 > rat.txt -python $RELEASE_DIR/check-rat-report.py $RELEASE_DIR/rat_exclude_files.txt rat.txt > filtered_rat.txt -cat filtered_rat.txt -UNAPPROVED=`cat filtered_rat.txt | grep "NOT APPROVED" | wc -l` - -if [ "0" -eq "${UNAPPROVED}" ]; then - echo "No unapproved licenses" -else - echo "${UNAPPROVED} unapproved licences. Check rat report: rat.txt" - exit 1 -fi diff --git a/dev/release/run-test.rb b/dev/release/run-test.rb deleted file mode 100755 index 90df39b13fa2..000000000000 --- a/dev/release/run-test.rb +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env ruby -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -$VERBOSE = true - -require "pathname" - -test_dir = Pathname.new(__dir__) - -require "test-unit" -require_relative "test-helper" - -ENV["TEST_UNIT_MAX_DIFF_TARGET_STRING_SIZE"] = "10000" - -exit(Test::Unit::AutoRunner.run(true, test_dir.to_s)) diff --git a/dev/release/setup-gpg-agent.sh b/dev/release/setup-gpg-agent.sh deleted file mode 100644 index 3e31d0e4e3c5..000000000000 --- a/dev/release/setup-gpg-agent.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# source me -eval $(gpg-agent --daemon --allow-preset-passphrase) -gpg --use-agent -s LICENSE.txt -rm -rf LICENSE.txt.gpg diff --git a/dev/release/test-helper.rb b/dev/release/test-helper.rb deleted file mode 100644 index 8a272ddfe04a..000000000000 --- a/dev/release/test-helper.rb +++ /dev/null @@ -1,96 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -require "English" -require "cgi/util" -require "fileutils" -require "find" -require "json" -require "open-uri" -require "rexml/document" -require "tempfile" -require "tmpdir" - -module CommandRunnable - class Error < StandardError - end - - def sh(*command_line, check_result: true) - if command_line[0].is_a?(Hash) - env = command_line.shift - else - env = {} - end - stdout = Tempfile.new("command-stdout.log") - stderr = Tempfile.new("command-stderr.log") - success = system(env, *command_line, out: stdout.path, err: stderr.path) - if check_result - unless success - message = "Failed to run: #{command_line.join(" ")}\n" - message << "stdout:\n #{stdout.read}\n" - message << "stderr:\n #{stderr.read}" - raise Error, message - end - end - stdout.read - end -end - -module GitRunnable - include CommandRunnable - - def git(*args) - if args[0].is_a?(Hash) - env = args.shift - else - env = {} - end - sh(env, "git", *args) - end - - def git_current_commit - git("rev-parse", "HEAD").chomp - end - - def git_tags - git("tags").lines(chomp: true) - end -end - -module VersionDetectable - def detect_versions - top_dir = Pathname(__dir__).parent.parent - cpp_cmake_lists = top_dir + "cpp" + "CMakeLists.txt" - @snapshot_version = cpp_cmake_lists.read[/ARROW_VERSION "(.+?)"/, 1] - @release_version = @snapshot_version.gsub(/-SNAPSHOT\z/, "") - @so_version = compute_so_version(@release_version) - @next_version = @release_version.gsub(/\A\d+/) {|major| major.succ} - @next_snapshot_version = "#{@next_version}-SNAPSHOT" - @next_so_version = compute_so_version(@next_version) - r_description = top_dir + "r" + "DESCRIPTION" - @previous_version = r_description.read[/^Version: (.+?)\.9000$/, 1] - end - - def compute_so_version(version) - major, minor, _patch = version.split(".") - Integer(major, 10) * 100 + Integer(minor, 10) - end - - def on_release_branch? - @snapshot_version == @release_version - end -end diff --git a/dev/release/verify-apt.sh b/dev/release/verify-apt.sh deleted file mode 100755 index e7b87a3a4da6..000000000000 --- a/dev/release/verify-apt.sh +++ /dev/null @@ -1,150 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -exu - -if [ $# -lt 2 ]; then - echo "Usage: $0 VERSION rc" - echo " $0 VERSION release" - echo " $0 VERSION local" - echo " e.g.: $0 0.13.0 rc # Verify 0.13.0 RC" - echo " e.g.: $0 0.13.0 release # Verify 0.13.0" - echo " e.g.: $0 0.13.0-dev20210203 local # Verify 0.13.0-dev20210203 on local" - exit 1 -fi - -VERSION="$1" -TYPE="$2" - -local_prefix="/arrow/dev/tasks/linux-packages" - -export DEBIAN_FRONTEND=noninteractive - -apt update -apt install -y -V \ - curl \ - lsb-release - -code_name="$(lsb_release --codename --short)" -distribution="$(lsb_release --id --short | tr 'A-Z' 'a-z')" -artifactory_base_url="https://apache.jfrog.io/artifactory/arrow/${distribution}" -if [ "${TYPE}" = "rc" ]; then - artifactory_base_url+="-rc" -fi - -have_flight=yes -have_plasma=yes -workaround_missing_packages=() -case "${distribution}-${code_name}" in - debian-*) - sed \ - -i"" \ - -e "s/ main$/ main contrib non-free/g" \ - /etc/apt/sources.list - ;; -esac -if [ "$(arch)" = "aarch64" ]; then - have_plasma=no -fi - -if [ "${TYPE}" = "local" ]; then - case "${VERSION}" in - *-dev*) - package_version="$(echo "${VERSION}" | sed -e 's/-dev\(.*\)$/~dev\1/g')" - ;; - *-rc*) - package_version="$(echo "${VERSION}" | sed -e 's/-rc.*$//g')" - ;; - *) - package_version="${VERSION}" - ;; - esac - package_version+="-1" - apt_source_path="${local_prefix}/apt/repositories" - apt_source_path+="/${distribution}/pool/${code_name}/main" - apt_source_path+="/a/apache-arrow-apt-source" - apt_source_path+="/apache-arrow-apt-source_${package_version}_all.deb" - apt install -y -V "${apt_source_path}" -else - package_version="${VERSION}-1" - apt_source_base_name="apache-arrow-apt-source-latest-${code_name}.deb" - curl \ - --output "${apt_source_base_name}" \ - "${artifactory_base_url}/${apt_source_base_name}" - apt install -y -V "./${apt_source_base_name}" -fi - -if [ "${TYPE}" = "local" ]; then - sed \ - -i"" \ - -e "s,^URIs: .*$,URIs: file://${local_prefix}/apt/repositories/${distribution},g" \ - /etc/apt/sources.list.d/apache-arrow.sources - keys="${local_prefix}/KEYS" - if [ -f "${keys}" ]; then - gpg \ - --no-default-keyring \ - --keyring /usr/share/keyrings/apache-arrow-apt-source.gpg \ - --import "${keys}" - fi -else - if [ "${TYPE}" = "rc" ]; then - sed \ - -i"" \ - -e "s,^URIs: \\(.*\\)/,URIs: \\1-rc/,g" \ - /etc/apt/sources.list.d/apache-arrow.sources - fi -fi - -apt update - -apt install -y -V libarrow-glib-dev=${package_version} -required_packages=() -required_packages+=(cmake) -required_packages+=(g++) -required_packages+=(git) -required_packages+=(${workaround_missing_packages[@]}) -apt install -y -V ${required_packages[@]} -mkdir -p build -cp -a /arrow/cpp/examples/minimal_build build -pushd build/minimal_build -cmake . -make -j$(nproc) -./arrow_example -popd - -apt install -y -V libarrow-glib-dev=${package_version} -apt install -y -V libarrow-glib-doc=${package_version} - -if [ "${have_flight}" = "yes" ]; then - apt install -y -V libarrow-flight-dev=${package_version} -fi - -apt install -y -V libarrow-python-dev=${package_version} - -if [ "${have_plasma}" = "yes" ]; then - apt install -y -V libplasma-glib-dev=${package_version} - apt install -y -V libplasma-glib-doc=${package_version} - apt install -y -V plasma-store-server=${package_version} -fi - -apt install -y -V libgandiva-glib-dev=${package_version} -apt install -y -V libgandiva-glib-doc=${package_version} - -apt install -y -V libparquet-glib-dev=${package_version} -apt install -y -V libparquet-glib-doc=${package_version} diff --git a/dev/release/verify-release-candidate-wheels.bat b/dev/release/verify-release-candidate-wheels.bat deleted file mode 100644 index 2b57113a1bf7..000000000000 --- a/dev/release/verify-release-candidate-wheels.bat +++ /dev/null @@ -1,107 +0,0 @@ -@rem Licensed to the Apache Software Foundation (ASF) under one -@rem or more contributor license agreements. See the NOTICE file -@rem distributed with this work for additional information -@rem regarding copyright ownership. The ASF licenses this file -@rem to you under the Apache License, Version 2.0 (the -@rem "License"); you may not use this file except in compliance -@rem with the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, -@rem software distributed under the License is distributed on an -@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -@rem KIND, either express or implied. See the License for the -@rem specific language governing permissions and limitations -@rem under the License. - -@rem This script downloads and installs all Windows wheels for a release -@rem candidate into temporary conda environments and makes sure that imports -@rem work - -@rem To run the script: -@rem verify-release-candidate-wheels.bat VERSION RC_NUM - -@echo on - -set _CURRENT_DIR=%CD% -set _VERIFICATION_DIR=C:\tmp\arrow-verify-release-wheels - -if not exist "C:\tmp\" mkdir C:\tmp -if exist %_VERIFICATION_DIR% rd %_VERIFICATION_DIR% /s /q -if not exist %_VERIFICATION_DIR% mkdir %_VERIFICATION_DIR% - -cd %_VERIFICATION_DIR% - -@rem clone Arrow repository to obtain test requirements -set GIT_ENV_PATH=%_VERIFICATION_DIR%\_git -call conda create -p %GIT_ENV_PATH% ^ - --no-shortcuts -f -q -y git ^ - || EXIT /B 1 -call activate %GIT_ENV_PATH% - -git clone https://github.com/apache/arrow.git || EXIT /B 1 -pushd arrow -git submodule update --init -popd - -call deactivate - -set ARROW_TEST_DATA=%cd%\arrow\testing\data - -CALL :verify_wheel 3.6 %1 %2 m -if errorlevel 1 GOTO error - -CALL :verify_wheel 3.7 %1 %2 m -if errorlevel 1 GOTO error - -CALL :verify_wheel 3.8 %1 %2 -if errorlevel 1 GOTO error - -:done -cd %_CURRENT_DIR% - -EXIT /B %ERRORLEVEL% - -:error -call deactivate -cd %_CURRENT_DIR% - -EXIT /B 1 - -@rem a batch function to verify a single wheel -:verify_wheel - -set PY_VERSION=%1 -set ARROW_VERSION=%2 -set RC_NUMBER=%3 -set ABI_TAG=%4 -set PY_VERSION_NO_PERIOD=%PY_VERSION:.=% - -set CONDA_ENV_PATH=%_VERIFICATION_DIR%\_verify-wheel-%PY_VERSION% -call conda create -p %CONDA_ENV_PATH% ^ - --no-shortcuts -f -q -y python=%PY_VERSION% ^ - || EXIT /B 1 -call activate %CONDA_ENV_PATH% - -set WHEEL_FILENAME=pyarrow-%ARROW_VERSION%-cp%PY_VERSION_NO_PERIOD%-cp%PY_VERSION_NO_PERIOD%%ABI_TAG%-win_amd64.whl - -@rem Requires GNU Wget for Windows -wget --no-check-certificate -O %WHEEL_FILENAME% https://bintray.com/apache/arrow/download_file?file_path=python-rc%%2F%ARROW_VERSION%-rc%RC_NUMBER%%%2F%WHEEL_FILENAME% || EXIT /B 1 - -pip install %WHEEL_FILENAME% || EXIT /B 1 - -pip install -r arrow/python/requirements-test.txt || EXIT /B 1 - -py.test %CONDA_ENV_PATH%\Lib\site-packages\pyarrow --pdb -v || EXIT /B 1 - -python -c "import pyarrow" || EXIT /B 1 -python -c "import pyarrow.parquet" || EXIT /B 1 -python -c "import pyarrow.flight" || EXIT /B 1 -python -c "import pyarrow.dataset" || EXIT /B 1 - -:done - -call deactivate - -EXIT /B 0 diff --git a/dev/release/verify-release-candidate.bat b/dev/release/verify-release-candidate.bat deleted file mode 100644 index bef78fc920cc..000000000000 --- a/dev/release/verify-release-candidate.bat +++ /dev/null @@ -1,130 +0,0 @@ -@rem Licensed to the Apache Software Foundation (ASF) under one -@rem or more contributor license agreements. See the NOTICE file -@rem distributed with this work for additional information -@rem regarding copyright ownership. The ASF licenses this file -@rem to you under the Apache License, Version 2.0 (the -@rem "License"); you may not use this file except in compliance -@rem with the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, -@rem software distributed under the License is distributed on an -@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -@rem KIND, either express or implied. See the License for the -@rem specific language governing permissions and limitations -@rem under the License. - -@rem To run the script: -@rem verify-release-candidate.bat VERSION RC_NUM - -@echo on - -if not exist "C:\tmp\" mkdir C:\tmp -if exist "C:\tmp\arrow-verify-release" rd C:\tmp\arrow-verify-release /s /q -if not exist "C:\tmp\arrow-verify-release" mkdir C:\tmp\arrow-verify-release - -set _VERIFICATION_DIR=C:\tmp\arrow-verify-release -set _VERIFICATION_DIR_UNIX=C:/tmp/arrow-verify-release -set _VERIFICATION_CONDA_ENV=%_VERIFICATION_DIR%\conda-env -set _DIST_URL=https://dist.apache.org/repos/dist/dev/arrow -set _TARBALL=apache-arrow-%1.tar.gz -set ARROW_SOURCE=%_VERIFICATION_DIR%\apache-arrow-%1 -set INSTALL_DIR=%_VERIFICATION_DIR%\install - -@rem Requires GNU Wget for Windows -wget --no-check-certificate -O %_TARBALL% %_DIST_URL%/apache-arrow-%1-rc%2/%_TARBALL% || exit /B 1 - -tar xf %_TARBALL% -C %_VERIFICATION_DIR_UNIX% - -set PYTHON=3.6 - -@rem Using call with conda.bat seems necessary to avoid terminating the batch -@rem script execution -call conda create --no-shortcuts -c conda-forge -f -q -y -p %_VERIFICATION_CONDA_ENV% ^ - --file=ci\conda_env_cpp.yml ^ - --file=ci\conda_env_python.yml ^ - git ^ - python=%PYTHON% ^ - || exit /B 1 - -call activate %_VERIFICATION_CONDA_ENV% || exit /B 1 - -set GENERATOR=Visual Studio 15 2017 Win64 -set CONFIGURATION=release - -pushd %ARROW_SOURCE% - -set ARROW_HOME=%INSTALL_DIR% -set PARQUET_HOME=%INSTALL_DIR% -set PATH=%INSTALL_DIR%\bin;%PATH% - -@rem Build and test Arrow C++ libraries -mkdir %ARROW_SOURCE%\cpp\build -pushd %ARROW_SOURCE%\cpp\build - -@rem This is the path for Visual Studio Community 2017 -call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat" -arch=amd64 - -@rem NOTE(wesm): not using Ninja for now to be able to more easily control the -@rem generator used - -cmake -G "%GENERATOR%" ^ - -DARROW_BOOST_USE_SHARED=ON ^ - -DARROW_BUILD_STATIC=OFF ^ - -DARROW_BUILD_TESTS=ON ^ - -DARROW_CXXFLAGS="/MP" ^ - -DARROW_DATASET=ON ^ - -DARROW_FLIGHT=ON ^ - -DARROW_MIMALLOC=ON ^ - -DARROW_PARQUET=ON ^ - -DARROW_PYTHON=ON ^ - -DARROW_WITH_BROTLI=ON ^ - -DARROW_WITH_BZ2=ON ^ - -DARROW_WITH_LZ4=ON ^ - -DARROW_WITH_SNAPPY=ON ^ - -DARROW_WITH_ZLIB=ON ^ - -DARROW_WITH_ZSTD=ON ^ - -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^ - -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^ - -DCMAKE_UNITY_BUILD=ON ^ - -DGTest_SOURCE=BUNDLED ^ - .. || exit /B - -cmake --build . --target INSTALL --config Release || exit /B 1 - -@rem NOTE(wesm): Building googletest is flaky for me with ninja. Building it -@rem first fixes the problem - -@rem ninja googletest_ep || exit /B 1 -@rem ninja install || exit /B 1 - -@rem Get testing datasets for Parquet unit tests -git clone https://github.com/apache/parquet-testing.git %_VERIFICATION_DIR%\parquet-testing -set PARQUET_TEST_DATA=%_VERIFICATION_DIR%\parquet-testing\data - -git clone https://github.com/apache/arrow-testing.git %_VERIFICATION_DIR%\arrow-testing -set ARROW_TEST_DATA=%_VERIFICATION_DIR%\arrow-testing\data - -@rem Needed so python-test.exe works -set PYTHONPATH_ORIGINAL=%PYTHONPATH% -set PYTHONPATH=%CONDA_PREFIX%\Lib;%CONDA_PREFIX%\Lib\site-packages;%CONDA_PREFIX%\DLLs;%CONDA_PREFIX%;%PYTHONPATH% -ctest -VV || exit /B 1 -set PYTHONPATH=%PYTHONPATH_ORIGINAL% -popd - -@rem Build and import pyarrow -pushd %ARROW_SOURCE%\python - -pip install -r requirements-test.txt || exit /B 1 - -set PYARROW_CMAKE_GENERATOR=%GENERATOR% -set PYARROW_WITH_FLIGHT=1 -set PYARROW_WITH_PARQUET=1 -set PYARROW_WITH_DATASET=1 -python setup.py build_ext --inplace --bundle-arrow-cpp bdist_wheel || exit /B 1 -pytest pyarrow -v -s --enable-parquet || exit /B 1 - -popd - -call deactivate diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh deleted file mode 100755 index 3fc926f4e82e..000000000000 --- a/dev/release/verify-release-candidate.sh +++ /dev/null @@ -1,808 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Requirements -# - Ruby >= 2.3 -# - Maven >= 3.3.9 -# - JDK >=7 -# - gcc >= 4.8 -# - Node.js >= 11.12 (best way is to use nvm) -# - Go >= 1.11 -# -# If using a non-system Boost, set BOOST_ROOT and add Boost libraries to -# LD_LIBRARY_PATH. -# -# To reuse build artifacts between runs set ARROW_TMPDIR environment variable to -# a directory where the temporary files should be placed to, note that this -# directory is not cleaned up automatically. - -case $# in - 3) ARTIFACT="$1" - VERSION="$2" - RC_NUMBER="$3" - case $ARTIFACT in - source|binaries|wheels) ;; - *) echo "Invalid argument: '${ARTIFACT}', valid options are \ -'source', 'binaries', or 'wheels'" - exit 1 - ;; - esac - ;; - *) echo "Usage: $0 source|binaries X.Y.Z RC_NUMBER" - exit 1 - ;; -esac - -set -e -set -x -set -o pipefail - -SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" -ARROW_DIR="$(dirname $(dirname ${SOURCE_DIR}))" - -detect_cuda() { - if ! (which nvcc && which nvidia-smi) > /dev/null; then - return 1 - fi - - local n_gpus=$(nvidia-smi --list-gpus | wc -l) - return $((${n_gpus} < 1)) -} - -# Build options for the C++ library - -if [ -z "${ARROW_CUDA:-}" ] && detect_cuda; then - ARROW_CUDA=ON -fi -: ${ARROW_CUDA:=OFF} -: ${ARROW_FLIGHT:=ON} -: ${ARROW_GANDIVA:=ON} - -ARROW_DIST_URL='https://dist.apache.org/repos/dist/dev/arrow' - -download_dist_file() { - curl \ - --silent \ - --show-error \ - --fail \ - --location \ - --remote-name $ARROW_DIST_URL/$1 -} - -download_rc_file() { - download_dist_file apache-arrow-${VERSION}-rc${RC_NUMBER}/$1 -} - -import_gpg_keys() { - download_dist_file KEYS - gpg --import KEYS -} - -fetch_archive() { - local dist_name=$1 - download_rc_file ${dist_name}.tar.gz - download_rc_file ${dist_name}.tar.gz.asc - download_rc_file ${dist_name}.tar.gz.sha256 - download_rc_file ${dist_name}.tar.gz.sha512 - gpg --verify ${dist_name}.tar.gz.asc ${dist_name}.tar.gz - shasum -a 256 -c ${dist_name}.tar.gz.sha256 - shasum -a 512 -c ${dist_name}.tar.gz.sha512 -} - -verify_dir_artifact_signatures() { - # verify the signature and the checksums of each artifact - find $1 -name '*.asc' | while read sigfile; do - artifact=${sigfile/.asc/} - gpg --verify $sigfile $artifact || exit 1 - - # go into the directory because the checksum files contain only the - # basename of the artifact - pushd $(dirname $artifact) - base_artifact=$(basename $artifact) - if [ -f $base_artifact.sha256 ]; then - shasum -a 256 -c $base_artifact.sha256 || exit 1 - fi - shasum -a 512 -c $base_artifact.sha512 || exit 1 - popd - done -} - -test_binary() { - local download_dir=binaries - mkdir -p ${download_dir} - - python $SOURCE_DIR/download_rc_binaries.py $VERSION $RC_NUMBER \ - --dest=${download_dir} - - verify_dir_artifact_signatures ${download_dir} -} - -test_apt() { - for target in "debian:buster" \ - "arm64v8/debian:buster" \ - "ubuntu:bionic" \ - "arm64v8/ubuntu:bionic" \ - "ubuntu:focal" \ - "arm64v8/ubuntu:focal" \ - "ubuntu:groovy" \ - "arm64v8/ubuntu:groovy"; do \ - case "${target}" in - arm64v8/*) - if [ "$(arch)" = "aarch64" -o -e /usr/bin/qemu-aarch64-static ]; then - : # OK - else - continue - fi - ;; - esac - if ! docker run --rm -v "${SOURCE_DIR}"/../..:/arrow:delegated \ - "${target}" \ - /arrow/dev/release/verify-apt.sh \ - "${VERSION}" \ - "rc" \ - "${BINTRAY_REPOSITORY}"; then - echo "Failed to verify the APT repository for ${target}" - exit 1 - fi - done -} - -test_yum() { - for target in "centos:7" \ - "centos:8" \ - "arm64v8/centos:8"; do - case "${target}" in - arm64v8/*) - if [ "$(arch)" = "aarch64" -o -e /usr/bin/qemu-aarch64-static ]; then - : # OK - else - continue - fi - ;; - esac - if ! docker run --rm -v "${SOURCE_DIR}"/../..:/arrow:delegated \ - "${target}" \ - /arrow/dev/release/verify-yum.sh \ - "${VERSION}" \ - "rc" \ - "${BINTRAY_REPOSITORY}"; then - echo "Failed to verify the Yum repository for ${target}" - exit 1 - fi - done -} - - -setup_tempdir() { - cleanup() { - if [ "${TEST_SUCCESS}" = "yes" ]; then - rm -fr "${ARROW_TMPDIR}" - else - echo "Failed to verify release candidate. See ${ARROW_TMPDIR} for details." - fi - } - - if [ -z "${ARROW_TMPDIR}" ]; then - # clean up automatically if ARROW_TMPDIR is not defined - ARROW_TMPDIR=$(mktemp -d -t "$1.XXXXX") - trap cleanup EXIT - else - # don't clean up automatically - mkdir -p "${ARROW_TMPDIR}" - fi -} - -setup_miniconda() { - # Setup short-lived miniconda for Python and integration tests - if [ "$(uname)" == "Darwin" ]; then - if [ "$(uname -m)" == "arm64" ]; then - MINICONDA_URL=https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-MacOSX-arm64.sh - else - MINICONDA_URL=https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - fi - else - MINICONDA_URL=https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh - fi - - MINICONDA=$PWD/test-miniconda - - if [ ! -d "${MINICONDA}" ]; then - # Setup miniconda only if the directory doesn't exist yet - wget -O miniconda.sh $MINICONDA_URL - bash miniconda.sh -b -p $MINICONDA - rm -f miniconda.sh - fi - echo "Installed miniconda at ${MINICONDA}" - - . $MINICONDA/etc/profile.d/conda.sh - - conda create -n arrow-test -y -q -c conda-forge \ - python=3.8 \ - nomkl \ - numpy \ - pandas \ - cython - conda activate arrow-test - echo "Using conda environment ${CONDA_PREFIX}" -} - -# Build and test Java (Requires newer Maven -- I used 3.3.9) - -test_package_java() { - pushd java - - mvn test - mvn package - - popd -} - -# Build and test C++ - -test_and_install_cpp() { - mkdir -p cpp/build - pushd cpp/build - - ARROW_CMAKE_OPTIONS=" -${ARROW_CMAKE_OPTIONS:-} --DCMAKE_INSTALL_PREFIX=$ARROW_HOME --DCMAKE_INSTALL_LIBDIR=lib --DARROW_FLIGHT=${ARROW_FLIGHT} --DARROW_PLASMA=ON --DARROW_ORC=ON --DARROW_PYTHON=ON --DARROW_GANDIVA=${ARROW_GANDIVA} --DARROW_PARQUET=ON --DARROW_DATASET=ON --DPARQUET_REQUIRE_ENCRYPTION=ON --DARROW_VERBOSE_THIRDPARTY_BUILD=ON --DARROW_WITH_BZ2=ON --DARROW_WITH_ZLIB=ON --DARROW_WITH_ZSTD=ON --DARROW_WITH_LZ4=ON --DARROW_WITH_SNAPPY=ON --DARROW_WITH_BROTLI=ON --DARROW_BOOST_USE_SHARED=ON --DCMAKE_BUILD_TYPE=release --DARROW_BUILD_TESTS=ON --DARROW_BUILD_INTEGRATION=ON --DARROW_CUDA=${ARROW_CUDA} --DARROW_DEPENDENCY_SOURCE=AUTO -" - cmake $ARROW_CMAKE_OPTIONS .. - - make -j$NPROC install - - # TODO: ARROW-5036: plasma-serialization_tests broken - # TODO: ARROW-5054: libgtest.so link failure in flight-server-test - LD_LIBRARY_PATH=$PWD/release:$LD_LIBRARY_PATH ctest \ - --exclude-regex "plasma-serialization_tests" \ - -j$NPROC \ - --output-on-failure \ - -L unittest - popd -} - -test_csharp() { - pushd csharp - - local csharp_bin=${PWD}/bin - mkdir -p ${csharp_bin} - - if which dotnet > /dev/null 2>&1; then - if ! which sourcelink > /dev/null 2>&1; then - local dotnet_tools_dir=$HOME/.dotnet/tools - if [ -d "${dotnet_tools_dir}" ]; then - PATH="${dotnet_tools_dir}:$PATH" - fi - fi - else - local dotnet_version=3.1.405 - local dotnet_platform= - case "$(uname)" in - Linux) - dotnet_platform=linux - ;; - Darwin) - dotnet_platform=macos - ;; - esac - local dotnet_download_thank_you_url=https://dotnet.microsoft.com/download/thank-you/dotnet-sdk-${dotnet_version}-${dotnet_platform}-x64-binaries - local dotnet_download_url=$( \ - curl --location ${dotnet_download_thank_you_url} | \ - grep 'window\.open' | \ - grep -E -o '[^"]+' | \ - sed -n 2p) - curl ${dotnet_download_url} | \ - tar xzf - -C ${csharp_bin} - PATH=${csharp_bin}:${PATH} - fi - - dotnet test - mv dummy.git ../.git - dotnet pack -c Release - mv ../.git dummy.git - - if ! which sourcelink > /dev/null 2>&1; then - dotnet tool install --tool-path ${csharp_bin} sourcelink - PATH=${csharp_bin}:${PATH} - if ! sourcelink --help > /dev/null 2>&1; then - export DOTNET_ROOT=${csharp_bin} - fi - fi - - sourcelink test artifacts/Apache.Arrow/Release/netstandard1.3/Apache.Arrow.pdb - sourcelink test artifacts/Apache.Arrow/Release/netcoreapp2.1/Apache.Arrow.pdb - - popd -} - -# Build and test Python - -test_python() { - pushd python - - pip install -r requirements-build.txt -r requirements-test.txt - - export PYARROW_WITH_DATASET=1 - export PYARROW_WITH_PARQUET=1 - export PYARROW_WITH_PLASMA=1 - if [ "${ARROW_CUDA}" = "ON" ]; then - export PYARROW_WITH_CUDA=1 - fi - if [ "${ARROW_FLIGHT}" = "ON" ]; then - export PYARROW_WITH_FLIGHT=1 - fi - if [ "${ARROW_GANDIVA}" = "ON" ]; then - export PYARROW_WITH_GANDIVA=1 - fi - - python setup.py build_ext --inplace - pytest pyarrow -v --pdb - - popd -} - -test_glib() { - pushd c_glib - - pip install meson - - meson build --prefix=$ARROW_HOME --libdir=lib - ninja -C build - ninja -C build install - - export GI_TYPELIB_PATH=$ARROW_HOME/lib/girepository-1.0:$GI_TYPELIB_PATH - - if ! bundle --version; then - gem install --no-document bundler - fi - - bundle install --path vendor/bundle - bundle exec ruby test/run-test.rb - - popd -} - -test_js() { - pushd js - - if [ "${INSTALL_NODE}" -gt 0 ]; then - export NVM_DIR="`pwd`/.nvm" - mkdir -p $NVM_DIR - curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | \ - PROFILE=/dev/null bash - [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" - - nvm install --lts - npm install -g yarn - fi - - yarn --frozen-lockfile - yarn run-s clean:all lint build - yarn test - popd -} - -test_ruby() { - pushd ruby - - local modules="red-arrow red-plasma red-gandiva red-parquet" - if [ "${ARROW_CUDA}" = "ON" ]; then - modules="${modules} red-arrow-cuda" - fi - - for module in ${modules}; do - pushd ${module} - bundle install --path vendor/bundle - bundle exec ruby test/run-test.rb - popd - done - - popd -} - -test_go() { - local VERSION=1.14.1 - local ARCH=amd64 - - if [ "$(uname)" == "Darwin" ]; then - local OS=darwin - else - local OS=linux - fi - - local GO_ARCHIVE=go$VERSION.$OS-$ARCH.tar.gz - wget https://dl.google.com/go/$GO_ARCHIVE - - mkdir -p local-go - tar -xzf $GO_ARCHIVE -C local-go - rm -f $GO_ARCHIVE - - export GOROOT=`pwd`/local-go/go - export GOPATH=`pwd`/local-go/gopath - export PATH=$GOROOT/bin:$GOPATH/bin:$PATH - - pushd go/arrow - - go get -v ./... - go test ./... - go clean -modcache - - popd -} - -test_rust() { - # install rust toolchain in a similar fashion like test-miniconda - export RUSTUP_HOME=$PWD/test-rustup - export CARGO_HOME=$PWD/test-rustup - - curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path - - export PATH=$RUSTUP_HOME/bin:$PATH - source $RUSTUP_HOME/env - - # build and test rust - pushd rust - - # raises on any formatting errors - rustup component add rustfmt --toolchain stable - cargo +stable fmt --all -- --check - - # we are targeting Rust nightly for releases - rustup default nightly - - # use local modules because we don't publish modules to crates.io yet - sed \ - -i.bak \ - -E \ - -e 's/^arrow = "([^"]*)"/arrow = { version = "\1", path = "..\/arrow" }/g' \ - -e 's/^parquet = "([^"]*)"/parquet = { version = "\1", path = "..\/parquet" }/g' \ - */Cargo.toml - - # raises on any warnings - RUSTFLAGS="-D warnings" cargo build - cargo test - - popd -} - -# Run integration tests -test_integration() { - JAVA_DIR=$PWD/java - CPP_BUILD_DIR=$PWD/cpp/build - - export ARROW_JAVA_INTEGRATION_JAR=$JAVA_DIR/tools/target/arrow-tools-$VERSION-jar-with-dependencies.jar - export ARROW_CPP_EXE_PATH=$CPP_BUILD_DIR/release - - pip install -e dev/archery - - INTEGRATION_TEST_ARGS="" - - if [ "${ARROW_FLIGHT}" = "ON" ]; then - INTEGRATION_TEST_ARGS="${INTEGRATION_TEST_ARGS} --run-flight" - fi - - # Flight integration test executable have runtime dependency on - # release/libgtest.so - LD_LIBRARY_PATH=$ARROW_CPP_EXE_PATH:$LD_LIBRARY_PATH \ - archery integration \ - --with-cpp=${TEST_INTEGRATION_CPP} \ - --with-java=${TEST_INTEGRATION_JAVA} \ - --with-js=${TEST_INTEGRATION_JS} \ - --with-go=${TEST_INTEGRATION_GO} \ - $INTEGRATION_TEST_ARGS -} - -clone_testing_repositories() { - # Clone testing repositories if not cloned already - if [ ! -d "arrow-testing" ]; then - git clone https://github.com/apache/arrow-testing.git - fi - if [ ! -d "parquet-testing" ]; then - git clone https://github.com/apache/parquet-testing.git - fi - export ARROW_TEST_DATA=$PWD/arrow-testing/data - export PARQUET_TEST_DATA=$PWD/parquet-testing/data -} - -test_source_distribution() { - export ARROW_HOME=$ARROW_TMPDIR/install - export PARQUET_HOME=$ARROW_TMPDIR/install - export LD_LIBRARY_PATH=$ARROW_HOME/lib:${LD_LIBRARY_PATH:-} - export PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig:${PKG_CONFIG_PATH:-} - - if [ "$(uname)" == "Darwin" ]; then - NPROC=$(sysctl -n hw.ncpu) - else - NPROC=$(nproc) - fi - - clone_testing_repositories - - if [ ${TEST_JAVA} -gt 0 ]; then - test_package_java - fi - if [ ${TEST_CPP} -gt 0 ]; then - test_and_install_cpp - fi - if [ ${TEST_CSHARP} -gt 0 ]; then - test_csharp - fi - if [ ${TEST_PYTHON} -gt 0 ]; then - test_python - fi - if [ ${TEST_GLIB} -gt 0 ]; then - test_glib - fi - if [ ${TEST_RUBY} -gt 0 ]; then - test_ruby - fi - if [ ${TEST_JS} -gt 0 ]; then - test_js - fi - if [ ${TEST_GO} -gt 0 ]; then - test_go - fi - if [ ${TEST_RUST} -gt 0 ]; then - test_rust - fi - if [ ${TEST_INTEGRATION} -gt 0 ]; then - test_integration - fi -} - -test_binary_distribution() { - : ${BINTRAY_REPOSITORY:=apache/arrow} - - if [ ${TEST_BINARY} -gt 0 ]; then - test_binary - fi - if [ ${TEST_APT} -gt 0 ]; then - test_apt - fi - if [ ${TEST_YUM} -gt 0 ]; then - test_yum - fi -} - -check_python_imports() { - python << IMPORT_TESTS -import platform - -import pyarrow -import pyarrow.parquet -import pyarrow.plasma -import pyarrow.fs -import pyarrow._hdfs -import pyarrow.dataset -import pyarrow.flight - -if platform.system() == "Darwin": - macos_version = tuple(map(int, platform.mac_ver()[0].split('.'))) - check_s3fs = macos_version >= (10, 13) -else: - check_s3fs = True - -if check_s3fs: - import pyarrow._s3fs -IMPORT_TESTS -} - -test_linux_wheels() { - local py_arches="3.6m 3.7m 3.8 3.9" - local manylinuxes="2010 2014" - - for py_arch in ${py_arches}; do - local env=_verify_wheel-${py_arch} - conda create -yq -n ${env} python=${py_arch//[mu]/} - conda activate ${env} - pip install -U pip - - for ml_spec in ${manylinuxes}; do - # check the mandatory and optional imports - pip install python-rc/${VERSION}-rc${RC_NUMBER}/pyarrow-${VERSION}-cp${py_arch//[mu.]/}-cp${py_arch//./}-manylinux${ml_spec}_x86_64.whl - check_python_imports - - # install test requirements and execute the tests - pip install -r ${ARROW_DIR}/python/requirements-test.txt - python -c 'import pyarrow; pyarrow.create_library_symlinks()' - pytest --pyargs pyarrow - done - - conda deactivate - done -} - -test_macos_wheels() { - local py_arches="3.6m 3.7m 3.8 3.9" - - for py_arch in ${py_arches}; do - local env=_verify_wheel-${py_arch} - conda create -yq -n ${env} python=${py_arch//m/} - conda activate ${env} - pip install -U pip - - # check the mandatory and optional imports - pip install --find-links python-rc/${VERSION}-rc${RC_NUMBER} pyarrow==${VERSION} - check_python_imports - - # install test requirements and execute the tests - pip install -r ${ARROW_DIR}/python/requirements-test.txt - python -c 'import pyarrow; pyarrow.create_library_symlinks()' - pytest --pyargs pyarrow - - conda deactivate - done -} - -test_wheels() { - clone_testing_repositories - - local download_dir=binaries - mkdir -p ${download_dir} - - if [ "$(uname)" == "Darwin" ]; then - local filter_regex=.*macosx.* - else - local filter_regex=.*manylinux.* - fi - - python $SOURCE_DIR/download_rc_binaries.py $VERSION $RC_NUMBER \ - --regex=${filter_regex} \ - --dest=${download_dir} - - verify_dir_artifact_signatures ${download_dir} - - pushd ${download_dir} - - if [ "$(uname)" == "Darwin" ]; then - test_macos_wheels - else - test_linux_wheels - fi - - popd -} - -# By default test all functionalities. -# To deactivate one test, deactivate the test and all of its dependents -# To explicitly select one test, set TEST_DEFAULT=0 TEST_X=1 - -# Install NodeJS locally for running the JavaScript tests rather than using the -# system Node installation, which may be too old. -: ${INSTALL_NODE:=1} - -if [ "${ARTIFACT}" == "source" ]; then - : ${TEST_SOURCE:=1} -elif [ "${ARTIFACT}" == "wheels" ]; then - TEST_WHEELS=1 -else - TEST_BINARY_DISTRIBUTIONS=1 -fi -: ${TEST_SOURCE:=0} -: ${TEST_WHEELS:=0} -: ${TEST_BINARY_DISTRIBUTIONS:=0} - -: ${TEST_DEFAULT:=1} -: ${TEST_JAVA:=${TEST_DEFAULT}} -: ${TEST_CPP:=${TEST_DEFAULT}} -: ${TEST_CSHARP:=${TEST_DEFAULT}} -: ${TEST_GLIB:=${TEST_DEFAULT}} -: ${TEST_RUBY:=${TEST_DEFAULT}} -: ${TEST_PYTHON:=${TEST_DEFAULT}} -: ${TEST_JS:=${TEST_DEFAULT}} -: ${TEST_GO:=${TEST_DEFAULT}} -: ${TEST_RUST:=${TEST_DEFAULT}} -: ${TEST_INTEGRATION:=${TEST_DEFAULT}} -if [ ${TEST_BINARY_DISTRIBUTIONS} -gt 0 ]; then - TEST_BINARY_DISTRIBUTIONS_DEFAULT=${TEST_DEFAULT} -else - TEST_BINARY_DISTRIBUTIONS_DEFAULT=0 -fi -: ${TEST_BINARY:=${TEST_BINARY_DISTRIBUTIONS_DEFAULT}} -: ${TEST_APT:=${TEST_BINARY_DISTRIBUTIONS_DEFAULT}} -: ${TEST_YUM:=${TEST_BINARY_DISTRIBUTIONS_DEFAULT}} - -# For selective Integration testing, set TEST_DEFAULT=0 TEST_INTEGRATION_X=1 TEST_INTEGRATION_Y=1 -: ${TEST_INTEGRATION_CPP:=${TEST_INTEGRATION}} -: ${TEST_INTEGRATION_JAVA:=${TEST_INTEGRATION}} -: ${TEST_INTEGRATION_JS:=${TEST_INTEGRATION}} -: ${TEST_INTEGRATION_GO:=${TEST_INTEGRATION}} - -# Automatically test if its activated by a dependent -TEST_GLIB=$((${TEST_GLIB} + ${TEST_RUBY})) -TEST_CPP=$((${TEST_CPP} + ${TEST_GLIB} + ${TEST_PYTHON} + ${TEST_INTEGRATION_CPP})) -TEST_JAVA=$((${TEST_JAVA} + ${TEST_INTEGRATION_JAVA})) -TEST_JS=$((${TEST_JS} + ${TEST_INTEGRATION_JS})) -TEST_GO=$((${TEST_GO} + ${TEST_INTEGRATION_GO})) -TEST_INTEGRATION=$((${TEST_INTEGRATION} + ${TEST_INTEGRATION_CPP} + ${TEST_INTEGRATION_JAVA} + ${TEST_INTEGRATION_JS} + ${TEST_INTEGRATION_GO})) - -NEED_MINICONDA=$((${TEST_CPP} + ${TEST_WHEELS} + ${TEST_BINARY} + ${TEST_INTEGRATION})) - -: ${TEST_ARCHIVE:=apache-arrow-${VERSION}.tar.gz} -case "${TEST_ARCHIVE}" in - /*) - ;; - *) - TEST_ARCHIVE=${PWD}/${TEST_ARCHIVE} - ;; -esac - -TEST_SUCCESS=no - -setup_tempdir "arrow-${VERSION}" -echo "Working in sandbox ${ARROW_TMPDIR}" -cd ${ARROW_TMPDIR} - -if [ ${NEED_MINICONDA} -gt 0 ]; then - setup_miniconda -fi - -if [ "${ARTIFACT}" == "source" ]; then - dist_name="apache-arrow-${VERSION}" - if [ ${TEST_SOURCE} -gt 0 ]; then - import_gpg_keys - if [ ! -d "${dist_name}" ]; then - fetch_archive ${dist_name} - tar xf ${dist_name}.tar.gz - fi - else - mkdir -p ${dist_name} - if [ ! -f ${TEST_ARCHIVE} ]; then - echo "${TEST_ARCHIVE} not found" - exit 1 - fi - tar xf ${TEST_ARCHIVE} -C ${dist_name} --strip-components=1 - fi - pushd ${dist_name} - test_source_distribution - popd -elif [ "${ARTIFACT}" == "wheels" ]; then - import_gpg_keys - test_wheels -else - import_gpg_keys - test_binary_distribution -fi - -TEST_SUCCESS=yes -echo 'Release candidate looks good!' -exit 0 diff --git a/dev/release/verify-yum.sh b/dev/release/verify-yum.sh deleted file mode 100755 index b9c46c43898c..000000000000 --- a/dev/release/verify-yum.sh +++ /dev/null @@ -1,154 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -exu - -if [ $# -lt 2 ]; then - echo "Usage: $0 VERSION rc" - echo " $0 VERSION release" - echo " $0 VERSION local" - echo " e.g.: $0 0.13.0 rc # Verify 0.13.0 RC" - echo " e.g.: $0 0.13.0 release # Verify 0.13.0" - echo " e.g.: $0 0.13.0-dev20210203 local # Verify 0.13.0-dev20210203 on local" - exit 1 -fi - -VERSION="$1" -TYPE="$2" - -local_prefix="/arrow/dev/tasks/linux-packages" - -artifactory_base_url="https://apache.jfrog.io/artifactory/arrow/centos" -if [ "${TYPE}" = "rc" ]; then - artifactory_base_url+="-rc" -fi - -distribution=$(. /etc/os-release && echo "${ID}") -distribution_version=$(. /etc/os-release && echo "${VERSION_ID}") - -cmake_package=cmake -cmake_command=cmake -have_flight=yes -have_gandiva=yes -have_glib=yes -have_parquet=yes -install_command="dnf install -y --enablerepo=powertools" -case "${distribution}-${distribution_version}" in - centos-7) - cmake_package=cmake3 - cmake_command=cmake3 - have_flight=no - have_gandiva=no - install_command="yum install -y" - ;; -esac -if [ "$(arch)" = "aarch64" ]; then - have_gandiva=no -fi - -if [ "${TYPE}" = "local" ]; then - case "${VERSION}" in - *-dev*) - package_version="$(echo "${VERSION}" | sed -e 's/-dev\(.*\)$/-0.dev\1/g')" - ;; - *-rc*) - package_version="$(echo "${VERSION}" | sed -e 's/-rc.*$//g')" - package_version+="-1" - ;; - *) - package_version="${VERSION}-1" - ;; - esac - package_version+=".el${distribution_version}" - release_path="${local_prefix}/yum/repositories" - release_path+="/centos/${distribution_version}/$(arch)/Packages" - release_path+="/apache-arrow-release-${package_version}.noarch.rpm" - ${install_command} "${release_path}" -else - package_version="${VERSION}" - ${install_command} \ - ${artifactory_base_url}/${distribution_version}/apache-arrow-release-latest.rpm -fi - -if [ "${TYPE}" = "local" ]; then - sed \ - -i"" \ - -e "s,baseurl=https://apache\.jfrog\.io/artifactory/arrow/,baseurl=file://${local_prefix}/yum/repositories/,g" \ - /etc/yum.repos.d/Apache-Arrow.repo - keys="${local_prefix}/KEYS" - if [ -f "${keys}" ]; then - cp "${keys}" /etc/pki/rpm-gpg/RPM-GPG-KEY-Apache-Arrow - fi -else - if [ "${TYPE}" = "rc" ]; then - sed \ - -i"" \ - -e "s,/centos/,/centos-rc/,g" \ - /etc/yum.repos.d/Apache-Arrow.repo - fi -fi - -${install_command} --enablerepo=epel arrow-devel-${package_version} -${install_command} \ - ${cmake_package} \ - gcc-c++ \ - git \ - make -mkdir -p build -cp -a /arrow/cpp/examples/minimal_build build -pushd build/minimal_build -${cmake_command} . -make -j$(nproc) -./arrow_example -popd - -if [ "${have_glib}" = "yes" ]; then - ${install_command} --enablerepo=epel arrow-glib-devel-${package_version} - ${install_command} --enablerepo=epel arrow-glib-doc-${package_version} -fi -${install_command} --enablerepo=epel arrow-python-devel-${package_version} - -if [ "${have_glib}" = "yes" ]; then - ${install_command} --enablerepo=epel plasma-glib-devel-${package_version} - ${install_command} --enablerepo=epel plasma-glib-doc-${package_version} -else - ${install_command} --enablerepo=epel plasma-devel-${package_version} -fi - -if [ "${have_flight}" = "yes" ]; then - ${install_command} --enablerepo=epel arrow-flight-devel-${package_version} -fi - -if [ "${have_gandiva}" = "yes" ]; then - if [ "${have_glib}" = "yes" ]; then - ${install_command} --enablerepo=epel gandiva-glib-devel-${package_version} - ${install_command} --enablerepo=epel gandiva-glib-doc-${package_version} - else - ${install_command} --enablerepo=epel gandiva-devel-${package_version} - fi -fi - -if [ "${have_parquet}" = "yes" ]; then - if [ "${have_glib}" = "yes" ]; then - ${install_command} --enablerepo=epel parquet-glib-devel-${package_version} - ${install_command} --enablerepo=epel parquet-glib-doc-${package_version} - else - ${install_command} --enablerepo=epel parquet-devel-${package_version} - fi -fi diff --git a/dev/tasks/README.md b/dev/tasks/README.md deleted file mode 100644 index 1af9739db29e..000000000000 --- a/dev/tasks/README.md +++ /dev/null @@ -1,19 +0,0 @@ - - -See the usage guide under the [documentation page](../../docs/source/developers/crossbow.rst) diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.6.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.6.____cpython.yaml deleted file mode 100644 index dd4c04197c91..000000000000 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.6.____cpython.yaml +++ /dev/null @@ -1,70 +0,0 @@ -aws_sdk_cpp: -- 1.8.151 -bzip2: -- '1' -c_compiler: -- gcc -c_compiler_version: -- '9' -cdt_name: -- cos6 -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- '10.2' -cxx_compiler: -- gxx -cxx_compiler_version: -- '9' -docker_image: -- quay.io/condaforge/linux-anvil-cuda:10.2 -gflags: -- '2.2' -glog: -- 0.4.0 -grpc_cpp: -- '1.36' -libprotobuf: -- '3.15' -lz4_c: -- 1.9.3 -numpy: -- '1.17' -orc: -- 1.6.7 -pin_run_as_build: - bzip2: - max_pin: x - lz4-c: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.6.* *_cpython -re2: -- 2021.04.01 -snappy: -- '1' -target_platform: -- linux-64 -thrift_cpp: -- 0.14.1 -zip_keys: -- - c_compiler_version - - cxx_compiler_version -- - cuda_compiler_version - - cdt_name - - docker_image -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.4' diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.7.____cpython.yaml deleted file mode 100644 index f0c43929b56c..000000000000 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.7.____cpython.yaml +++ /dev/null @@ -1,70 +0,0 @@ -aws_sdk_cpp: -- 1.8.151 -bzip2: -- '1' -c_compiler: -- gcc -c_compiler_version: -- '9' -cdt_name: -- cos6 -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- '10.2' -cxx_compiler: -- gxx -cxx_compiler_version: -- '9' -docker_image: -- quay.io/condaforge/linux-anvil-cuda:10.2 -gflags: -- '2.2' -glog: -- 0.4.0 -grpc_cpp: -- '1.36' -libprotobuf: -- '3.15' -lz4_c: -- 1.9.3 -numpy: -- '1.17' -orc: -- 1.6.7 -pin_run_as_build: - bzip2: - max_pin: x - lz4-c: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.7.* *_cpython -re2: -- 2021.04.01 -snappy: -- '1' -target_platform: -- linux-64 -thrift_cpp: -- 0.14.1 -zip_keys: -- - c_compiler_version - - cxx_compiler_version -- - cuda_compiler_version - - cdt_name - - docker_image -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.4' diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.8.____cpython.yaml deleted file mode 100644 index 149e70f438b1..000000000000 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.8.____cpython.yaml +++ /dev/null @@ -1,70 +0,0 @@ -aws_sdk_cpp: -- 1.8.151 -bzip2: -- '1' -c_compiler: -- gcc -c_compiler_version: -- '9' -cdt_name: -- cos6 -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- '10.2' -cxx_compiler: -- gxx -cxx_compiler_version: -- '9' -docker_image: -- quay.io/condaforge/linux-anvil-cuda:10.2 -gflags: -- '2.2' -glog: -- 0.4.0 -grpc_cpp: -- '1.36' -libprotobuf: -- '3.15' -lz4_c: -- 1.9.3 -numpy: -- '1.17' -orc: -- 1.6.7 -pin_run_as_build: - bzip2: - max_pin: x - lz4-c: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.8.* *_cpython -re2: -- 2021.04.01 -snappy: -- '1' -target_platform: -- linux-64 -thrift_cpp: -- 0.14.1 -zip_keys: -- - c_compiler_version - - cxx_compiler_version -- - cuda_compiler_version - - cdt_name - - docker_image -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.4' diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.19python3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.19python3.9.____cpython.yaml deleted file mode 100644 index fb15d4e71567..000000000000 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.19python3.9.____cpython.yaml +++ /dev/null @@ -1,70 +0,0 @@ -aws_sdk_cpp: -- 1.8.151 -bzip2: -- '1' -c_compiler: -- gcc -c_compiler_version: -- '9' -cdt_name: -- cos6 -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- '10.2' -cxx_compiler: -- gxx -cxx_compiler_version: -- '9' -docker_image: -- quay.io/condaforge/linux-anvil-cuda:10.2 -gflags: -- '2.2' -glog: -- 0.4.0 -grpc_cpp: -- '1.36' -libprotobuf: -- '3.15' -lz4_c: -- 1.9.3 -numpy: -- '1.19' -orc: -- 1.6.7 -pin_run_as_build: - bzip2: - max_pin: x - lz4-c: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.9.* *_cpython -re2: -- 2021.04.01 -snappy: -- '1' -target_platform: -- linux-64 -thrift_cpp: -- 0.14.1 -zip_keys: -- - c_compiler_version - - cxx_compiler_version -- - cuda_compiler_version - - cdt_name - - docker_image -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.4' diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.6.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.6.____cpython.yaml deleted file mode 100644 index d977f9e5779e..000000000000 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.6.____cpython.yaml +++ /dev/null @@ -1,70 +0,0 @@ -aws_sdk_cpp: -- 1.8.151 -bzip2: -- '1' -c_compiler: -- gcc -c_compiler_version: -- '9' -cdt_name: -- cos6 -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- None -cxx_compiler: -- gxx -cxx_compiler_version: -- '9' -docker_image: -- quay.io/condaforge/linux-anvil-comp7 -gflags: -- '2.2' -glog: -- 0.4.0 -grpc_cpp: -- '1.36' -libprotobuf: -- '3.15' -lz4_c: -- 1.9.3 -numpy: -- '1.17' -orc: -- 1.6.7 -pin_run_as_build: - bzip2: - max_pin: x - lz4-c: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.6.* *_cpython -re2: -- 2021.04.01 -snappy: -- '1' -target_platform: -- linux-64 -thrift_cpp: -- 0.14.1 -zip_keys: -- - c_compiler_version - - cxx_compiler_version -- - cuda_compiler_version - - cdt_name - - docker_image -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.4' diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.7.____cpython.yaml deleted file mode 100644 index 6ffa87a5eb9e..000000000000 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.7.____cpython.yaml +++ /dev/null @@ -1,70 +0,0 @@ -aws_sdk_cpp: -- 1.8.151 -bzip2: -- '1' -c_compiler: -- gcc -c_compiler_version: -- '9' -cdt_name: -- cos6 -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- None -cxx_compiler: -- gxx -cxx_compiler_version: -- '9' -docker_image: -- quay.io/condaforge/linux-anvil-comp7 -gflags: -- '2.2' -glog: -- 0.4.0 -grpc_cpp: -- '1.36' -libprotobuf: -- '3.15' -lz4_c: -- 1.9.3 -numpy: -- '1.17' -orc: -- 1.6.7 -pin_run_as_build: - bzip2: - max_pin: x - lz4-c: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.7.* *_cpython -re2: -- 2021.04.01 -snappy: -- '1' -target_platform: -- linux-64 -thrift_cpp: -- 0.14.1 -zip_keys: -- - c_compiler_version - - cxx_compiler_version -- - cuda_compiler_version - - cdt_name - - docker_image -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.4' diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.8.____cpython.yaml deleted file mode 100644 index 7105f634953a..000000000000 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.8.____cpython.yaml +++ /dev/null @@ -1,70 +0,0 @@ -aws_sdk_cpp: -- 1.8.151 -bzip2: -- '1' -c_compiler: -- gcc -c_compiler_version: -- '9' -cdt_name: -- cos6 -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- None -cxx_compiler: -- gxx -cxx_compiler_version: -- '9' -docker_image: -- quay.io/condaforge/linux-anvil-comp7 -gflags: -- '2.2' -glog: -- 0.4.0 -grpc_cpp: -- '1.36' -libprotobuf: -- '3.15' -lz4_c: -- 1.9.3 -numpy: -- '1.17' -orc: -- 1.6.7 -pin_run_as_build: - bzip2: - max_pin: x - lz4-c: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.8.* *_cpython -re2: -- 2021.04.01 -snappy: -- '1' -target_platform: -- linux-64 -thrift_cpp: -- 0.14.1 -zip_keys: -- - c_compiler_version - - cxx_compiler_version -- - cuda_compiler_version - - cdt_name - - docker_image -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.4' diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython.yaml deleted file mode 100644 index efe0148cc810..000000000000 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython.yaml +++ /dev/null @@ -1,70 +0,0 @@ -aws_sdk_cpp: -- 1.8.151 -bzip2: -- '1' -c_compiler: -- gcc -c_compiler_version: -- '9' -cdt_name: -- cos6 -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- None -cxx_compiler: -- gxx -cxx_compiler_version: -- '9' -docker_image: -- quay.io/condaforge/linux-anvil-comp7 -gflags: -- '2.2' -glog: -- 0.4.0 -grpc_cpp: -- '1.36' -libprotobuf: -- '3.15' -lz4_c: -- 1.9.3 -numpy: -- '1.19' -orc: -- 1.6.7 -pin_run_as_build: - bzip2: - max_pin: x - lz4-c: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.9.* *_cpython -re2: -- 2021.04.01 -snappy: -- '1' -target_platform: -- linux-64 -thrift_cpp: -- 0.14.1 -zip_keys: -- - c_compiler_version - - cxx_compiler_version -- - cuda_compiler_version - - cdt_name - - docker_image -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.4' diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.6.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.6.____cpython.yaml deleted file mode 100644 index f2d3ceaac689..000000000000 --- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.6.____cpython.yaml +++ /dev/null @@ -1,71 +0,0 @@ -BUILD: -- aarch64-conda_cos7-linux-gnu -aws_sdk_cpp: -- 1.8.63 -boost_cpp: -- 1.74.0 -bzip2: -- '1' -c_compiler: -- gcc -c_compiler_version: -- '7' -cdt_arch: -- aarch64 -cdt_name: -- cos7 -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cuda_compiler_version: -- None -cxx_compiler: -- gxx -cxx_compiler_version: -- '7' -docker_image: -- condaforge/linux-anvil-aarch64 -gflags: -- '2.2' -glog: -- 0.4.0 -grpc_cpp: -- '1.32' -libprotobuf: -- '3.13' -lz4_c: -- 1.9.2 -numpy: -- '1.16' -orc: -- 1.6.5 -pin_run_as_build: - boost-cpp: - max_pin: x.x.x - bzip2: - max_pin: x - lz4-c: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.6.* *_cpython -re2: -- 2020.10.01 -snappy: -- '1' -target_platform: -- linux-aarch64 -zip_keys: -- - c_compiler_version - - cxx_compiler_version -- - numpy - - python -zlib: -- '1.2' -zstd: -- '1.4' diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.7.____cpython.yaml deleted file mode 100644 index 611c39c907cf..000000000000 --- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.7.____cpython.yaml +++ /dev/null @@ -1,71 +0,0 @@ -BUILD: -- aarch64-conda_cos7-linux-gnu -aws_sdk_cpp: -- 1.8.63 -boost_cpp: -- 1.74.0 -bzip2: -- '1' -c_compiler: -- gcc -c_compiler_version: -- '7' -cdt_arch: -- aarch64 -cdt_name: -- cos7 -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cuda_compiler_version: -- None -cxx_compiler: -- gxx -cxx_compiler_version: -- '7' -docker_image: -- condaforge/linux-anvil-aarch64 -gflags: -- '2.2' -glog: -- 0.4.0 -grpc_cpp: -- '1.32' -libprotobuf: -- '3.13' -lz4_c: -- 1.9.2 -numpy: -- '1.16' -orc: -- 1.6.5 -pin_run_as_build: - boost-cpp: - max_pin: x.x.x - bzip2: - max_pin: x - lz4-c: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.7.* *_cpython -re2: -- 2020.10.01 -snappy: -- '1' -target_platform: -- linux-aarch64 -zip_keys: -- - c_compiler_version - - cxx_compiler_version -- - numpy - - python -zlib: -- '1.2' -zstd: -- '1.4' diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.8.____cpython.yaml deleted file mode 100644 index 2f0fc0e23067..000000000000 --- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.8.____cpython.yaml +++ /dev/null @@ -1,71 +0,0 @@ -BUILD: -- aarch64-conda_cos7-linux-gnu -aws_sdk_cpp: -- 1.8.63 -boost_cpp: -- 1.74.0 -bzip2: -- '1' -c_compiler: -- gcc -c_compiler_version: -- '7' -cdt_arch: -- aarch64 -cdt_name: -- cos7 -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cuda_compiler_version: -- None -cxx_compiler: -- gxx -cxx_compiler_version: -- '7' -docker_image: -- condaforge/linux-anvil-aarch64 -gflags: -- '2.2' -glog: -- 0.4.0 -grpc_cpp: -- '1.32' -libprotobuf: -- '3.13' -lz4_c: -- 1.9.2 -numpy: -- '1.16' -orc: -- 1.6.5 -pin_run_as_build: - boost-cpp: - max_pin: x.x.x - bzip2: - max_pin: x - lz4-c: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.8.* *_cpython -re2: -- 2020.10.01 -snappy: -- '1' -target_platform: -- linux-aarch64 -zip_keys: -- - c_compiler_version - - cxx_compiler_version -- - numpy - - python -zlib: -- '1.2' -zstd: -- '1.4' diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.9.____cpython.yaml deleted file mode 100644 index 2ec87205a0f1..000000000000 --- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.9.____cpython.yaml +++ /dev/null @@ -1,71 +0,0 @@ -BUILD: -- aarch64-conda_cos7-linux-gnu -aws_sdk_cpp: -- 1.8.63 -boost_cpp: -- 1.74.0 -bzip2: -- '1' -c_compiler: -- gcc -c_compiler_version: -- '7' -cdt_arch: -- aarch64 -cdt_name: -- cos7 -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cuda_compiler_version: -- None -cxx_compiler: -- gxx -cxx_compiler_version: -- '7' -docker_image: -- condaforge/linux-anvil-aarch64 -gflags: -- '2.2' -glog: -- 0.4.0 -grpc_cpp: -- '1.32' -libprotobuf: -- '3.13' -lz4_c: -- 1.9.2 -numpy: -- '1.19' -orc: -- 1.6.5 -pin_run_as_build: - boost-cpp: - max_pin: x.x.x - bzip2: - max_pin: x - lz4-c: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.9.* *_cpython -re2: -- 2020.10.01 -snappy: -- '1' -target_platform: -- linux-aarch64 -zip_keys: -- - c_compiler_version - - cxx_compiler_version -- - numpy - - python -zlib: -- '1.2' -zstd: -- '1.4' diff --git a/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.6.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.6.____cpython.yaml deleted file mode 100644 index 7b2dbb34d76d..000000000000 --- a/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.6.____cpython.yaml +++ /dev/null @@ -1,65 +0,0 @@ -MACOSX_DEPLOYMENT_TARGET: -- '10.9' -aws_sdk_cpp: -- 1.8.151 -bzip2: -- '1' -c_compiler: -- clang -c_compiler_version: -- '11' -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cuda_compiler_version: -- None -cxx_compiler: -- clangxx -cxx_compiler_version: -- '11' -gflags: -- '2.2' -glog: -- 0.4.0 -grpc_cpp: -- '1.36' -libprotobuf: -- '3.15' -lz4_c: -- 1.9.3 -macos_machine: -- x86_64-apple-darwin13.4.0 -numpy: -- '1.17' -orc: -- 1.6.7 -pin_run_as_build: - bzip2: - max_pin: x - lz4-c: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.6.* *_cpython -re2: -- 2021.04.01 -snappy: -- '1' -target_platform: -- osx-64 -thrift_cpp: -- 0.14.1 -zip_keys: -- - c_compiler_version - - cxx_compiler_version -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.4' diff --git a/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.7.____cpython.yaml deleted file mode 100644 index 8e3e828ab8a8..000000000000 --- a/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.7.____cpython.yaml +++ /dev/null @@ -1,65 +0,0 @@ -MACOSX_DEPLOYMENT_TARGET: -- '10.9' -aws_sdk_cpp: -- 1.8.151 -bzip2: -- '1' -c_compiler: -- clang -c_compiler_version: -- '11' -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cuda_compiler_version: -- None -cxx_compiler: -- clangxx -cxx_compiler_version: -- '11' -gflags: -- '2.2' -glog: -- 0.4.0 -grpc_cpp: -- '1.36' -libprotobuf: -- '3.15' -lz4_c: -- 1.9.3 -macos_machine: -- x86_64-apple-darwin13.4.0 -numpy: -- '1.17' -orc: -- 1.6.7 -pin_run_as_build: - bzip2: - max_pin: x - lz4-c: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.7.* *_cpython -re2: -- 2021.04.01 -snappy: -- '1' -target_platform: -- osx-64 -thrift_cpp: -- 0.14.1 -zip_keys: -- - c_compiler_version - - cxx_compiler_version -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.4' diff --git a/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.8.____cpython.yaml deleted file mode 100644 index cdd53c6006e6..000000000000 --- a/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.8.____cpython.yaml +++ /dev/null @@ -1,65 +0,0 @@ -MACOSX_DEPLOYMENT_TARGET: -- '10.9' -aws_sdk_cpp: -- 1.8.151 -bzip2: -- '1' -c_compiler: -- clang -c_compiler_version: -- '11' -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cuda_compiler_version: -- None -cxx_compiler: -- clangxx -cxx_compiler_version: -- '11' -gflags: -- '2.2' -glog: -- 0.4.0 -grpc_cpp: -- '1.36' -libprotobuf: -- '3.15' -lz4_c: -- 1.9.3 -macos_machine: -- x86_64-apple-darwin13.4.0 -numpy: -- '1.17' -orc: -- 1.6.7 -pin_run_as_build: - bzip2: - max_pin: x - lz4-c: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.8.* *_cpython -re2: -- 2021.04.01 -snappy: -- '1' -target_platform: -- osx-64 -thrift_cpp: -- 0.14.1 -zip_keys: -- - c_compiler_version - - cxx_compiler_version -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.4' diff --git a/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.19python3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.19python3.9.____cpython.yaml deleted file mode 100644 index 37df6a9ec536..000000000000 --- a/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.19python3.9.____cpython.yaml +++ /dev/null @@ -1,65 +0,0 @@ -MACOSX_DEPLOYMENT_TARGET: -- '10.9' -aws_sdk_cpp: -- 1.8.151 -bzip2: -- '1' -c_compiler: -- clang -c_compiler_version: -- '11' -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cuda_compiler_version: -- None -cxx_compiler: -- clangxx -cxx_compiler_version: -- '11' -gflags: -- '2.2' -glog: -- 0.4.0 -grpc_cpp: -- '1.36' -libprotobuf: -- '3.15' -lz4_c: -- 1.9.3 -macos_machine: -- x86_64-apple-darwin13.4.0 -numpy: -- '1.19' -orc: -- 1.6.7 -pin_run_as_build: - bzip2: - max_pin: x - lz4-c: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.9.* *_cpython -re2: -- 2021.04.01 -snappy: -- '1' -target_platform: -- osx-64 -thrift_cpp: -- 0.14.1 -zip_keys: -- - c_compiler_version - - cxx_compiler_version -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.4' diff --git a/dev/tasks/conda-recipes/.ci_support/osx_arm64_python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_arm64_python3.8.____cpython.yaml deleted file mode 100644 index 5894b8ee70b5..000000000000 --- a/dev/tasks/conda-recipes/.ci_support/osx_arm64_python3.8.____cpython.yaml +++ /dev/null @@ -1,65 +0,0 @@ -MACOSX_DEPLOYMENT_TARGET: -- '11.0' -aws_sdk_cpp: -- 1.8.151 -bzip2: -- '1' -c_compiler: -- clang -c_compiler_version: -- '11' -channel_sources: -- conda-forge/label/rust_dev,conda-forge -channel_targets: -- conda-forge main -cuda_compiler_version: -- None -cxx_compiler: -- clangxx -cxx_compiler_version: -- '11' -gflags: -- '2.2' -glog: -- 0.4.0 -grpc_cpp: -- '1.36' -libprotobuf: -- '3.15' -lz4_c: -- 1.9.3 -macos_machine: -- arm64-apple-darwin20.0.0 -numpy: -- '1.19' -orc: -- 1.6.7 -pin_run_as_build: - bzip2: - max_pin: x - lz4-c: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.8.* *_cpython -re2: -- 2021.04.01 -snappy: -- '1' -target_platform: -- osx-arm64 -thrift_cpp: -- 0.14.1 -zip_keys: -- - c_compiler_version - - cxx_compiler_version -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.4' diff --git a/dev/tasks/conda-recipes/.ci_support/osx_arm64_python3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_arm64_python3.9.____cpython.yaml deleted file mode 100644 index 4e6014c5db8a..000000000000 --- a/dev/tasks/conda-recipes/.ci_support/osx_arm64_python3.9.____cpython.yaml +++ /dev/null @@ -1,65 +0,0 @@ -MACOSX_DEPLOYMENT_TARGET: -- '11.0' -aws_sdk_cpp: -- 1.8.151 -bzip2: -- '1' -c_compiler: -- clang -c_compiler_version: -- '11' -channel_sources: -- conda-forge/label/rust_dev,conda-forge -channel_targets: -- conda-forge main -cuda_compiler_version: -- None -cxx_compiler: -- clangxx -cxx_compiler_version: -- '11' -gflags: -- '2.2' -glog: -- 0.4.0 -grpc_cpp: -- '1.36' -libprotobuf: -- '3.15' -lz4_c: -- 1.9.3 -macos_machine: -- arm64-apple-darwin20.0.0 -numpy: -- '1.19' -orc: -- 1.6.7 -pin_run_as_build: - bzip2: - max_pin: x - lz4-c: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.9.* *_cpython -re2: -- 2021.04.01 -snappy: -- '1' -target_platform: -- osx-arm64 -thrift_cpp: -- 0.14.1 -zip_keys: -- - c_compiler_version - - cxx_compiler_version -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.4' diff --git a/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base3.6.yaml b/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base3.6.yaml deleted file mode 100644 index ac945ce72d38..000000000000 --- a/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base3.6.yaml +++ /dev/null @@ -1,22 +0,0 @@ -c_compiler: -- gcc -c_compiler_version: -- '7' -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cxx_compiler: -- gxx -cxx_compiler_version: -- '7' -docker_image: -- condaforge/linux-anvil-comp7 -pin_run_as_build: - r-base: - min_pin: x.x - max_pin: x.x -r_base: -- '3.6' -target_platform: -- linux-64 diff --git a/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.0.yaml b/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.0.yaml deleted file mode 100644 index 51d26f834cc1..000000000000 --- a/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.0.yaml +++ /dev/null @@ -1,22 +0,0 @@ -c_compiler: -- gcc -c_compiler_version: -- '7' -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cxx_compiler: -- gxx -cxx_compiler_version: -- '7' -docker_image: -- condaforge/linux-anvil-comp7 -pin_run_as_build: - r-base: - min_pin: x.x - max_pin: x.x -r_base: -- '4.0' -target_platform: -- linux-64 diff --git a/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base3.6.yaml b/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base3.6.yaml deleted file mode 100644 index e3c5b898be6f..000000000000 --- a/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base3.6.yaml +++ /dev/null @@ -1,26 +0,0 @@ -MACOSX_DEPLOYMENT_TARGET: -- '10.9' -c_compiler: -- clang -c_compiler_version: -- '10' -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cxx_compiler: -- clangxx -cxx_compiler_version: -- '10' -macos_machine: -- x86_64-apple-darwin13.4.0 -macos_min_version: -- '10.9' -pin_run_as_build: - r-base: - min_pin: x.x - max_pin: x.x -r_base: -- '3.6' -target_platform: -- osx-64 diff --git a/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.0.yaml b/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.0.yaml deleted file mode 100644 index 8343a284b975..000000000000 --- a/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.0.yaml +++ /dev/null @@ -1,26 +0,0 @@ -MACOSX_DEPLOYMENT_TARGET: -- '10.9' -c_compiler: -- clang -c_compiler_version: -- '10' -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cxx_compiler: -- clangxx -cxx_compiler_version: -- '10' -macos_machine: -- x86_64-apple-darwin13.4.0 -macos_min_version: -- '10.9' -pin_run_as_build: - r-base: - min_pin: x.x - max_pin: x.x -r_base: -- '4.0' -target_platform: -- osx-64 diff --git a/dev/tasks/conda-recipes/.ci_support/r/win_64_r_base3.6.yaml b/dev/tasks/conda-recipes/.ci_support/r/win_64_r_base3.6.yaml deleted file mode 100644 index 3fb7f88499af..000000000000 --- a/dev/tasks/conda-recipes/.ci_support/r/win_64_r_base3.6.yaml +++ /dev/null @@ -1,12 +0,0 @@ -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -pin_run_as_build: - r-base: - min_pin: x.x - max_pin: x.x -r_base: -- '3.6' -target_platform: -- win-64 diff --git a/dev/tasks/conda-recipes/.ci_support/r/win_64_r_base4.0.yaml b/dev/tasks/conda-recipes/.ci_support/r/win_64_r_base4.0.yaml deleted file mode 100644 index 02c2a70756d0..000000000000 --- a/dev/tasks/conda-recipes/.ci_support/r/win_64_r_base4.0.yaml +++ /dev/null @@ -1,12 +0,0 @@ -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -pin_run_as_build: - r-base: - min_pin: x.x - max_pin: x.x -r_base: -- '4.0' -target_platform: -- win-64 diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.6.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.6.____cpython.yaml deleted file mode 100644 index 8fbbb64af9c5..000000000000 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.6.____cpython.yaml +++ /dev/null @@ -1,55 +0,0 @@ -aws_sdk_cpp: -- 1.8.151 -bzip2: -- '1' -c_compiler: -- vs2017 -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- None -cxx_compiler: -- vs2017 -gflags: -- '2.2' -glog: -- 0.4.0 -grpc_cpp: -- '1.36' -libprotobuf: -- '3.15' -lz4_c: -- 1.9.3 -numpy: -- '1.17' -pin_run_as_build: - bzip2: - max_pin: x - lz4-c: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.6.* *_cpython -re2: -- 2021.04.01 -snappy: -- '1' -target_platform: -- win-64 -thrift_cpp: -- 0.14.1 -zip_keys: -- - numpy - - python -zlib: -- '1.2' -zstd: -- '1.4' diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.7.____cpython.yaml deleted file mode 100644 index 4b702a389808..000000000000 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.7.____cpython.yaml +++ /dev/null @@ -1,55 +0,0 @@ -aws_sdk_cpp: -- 1.8.151 -bzip2: -- '1' -c_compiler: -- vs2017 -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- None -cxx_compiler: -- vs2017 -gflags: -- '2.2' -glog: -- 0.4.0 -grpc_cpp: -- '1.36' -libprotobuf: -- '3.15' -lz4_c: -- 1.9.3 -numpy: -- '1.17' -pin_run_as_build: - bzip2: - max_pin: x - lz4-c: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.7.* *_cpython -re2: -- 2021.04.01 -snappy: -- '1' -target_platform: -- win-64 -thrift_cpp: -- 0.14.1 -zip_keys: -- - numpy - - python -zlib: -- '1.2' -zstd: -- '1.4' diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.8.____cpython.yaml deleted file mode 100644 index 6ae6c2fde4f6..000000000000 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.8.____cpython.yaml +++ /dev/null @@ -1,55 +0,0 @@ -aws_sdk_cpp: -- 1.8.151 -bzip2: -- '1' -c_compiler: -- vs2017 -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- None -cxx_compiler: -- vs2017 -gflags: -- '2.2' -glog: -- 0.4.0 -grpc_cpp: -- '1.36' -libprotobuf: -- '3.15' -lz4_c: -- 1.9.3 -numpy: -- '1.17' -pin_run_as_build: - bzip2: - max_pin: x - lz4-c: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.8.* *_cpython -re2: -- 2021.04.01 -snappy: -- '1' -target_platform: -- win-64 -thrift_cpp: -- 0.14.1 -zip_keys: -- - numpy - - python -zlib: -- '1.2' -zstd: -- '1.4' diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython.yaml deleted file mode 100644 index 73a8b5099bbf..000000000000 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython.yaml +++ /dev/null @@ -1,55 +0,0 @@ -aws_sdk_cpp: -- 1.8.151 -bzip2: -- '1' -c_compiler: -- vs2017 -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- None -cxx_compiler: -- vs2017 -gflags: -- '2.2' -glog: -- 0.4.0 -grpc_cpp: -- '1.36' -libprotobuf: -- '3.15' -lz4_c: -- 1.9.3 -numpy: -- '1.19' -pin_run_as_build: - bzip2: - max_pin: x - lz4-c: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.9.* *_cpython -re2: -- 2021.04.01 -snappy: -- '1' -target_platform: -- win-64 -thrift_cpp: -- 0.14.1 -zip_keys: -- - numpy - - python -zlib: -- '1.2' -zstd: -- '1.4' diff --git a/dev/tasks/conda-recipes/.scripts/logging_utils.sh b/dev/tasks/conda-recipes/.scripts/logging_utils.sh deleted file mode 100644 index a53ef3f2c7ad..000000000000 --- a/dev/tasks/conda-recipes/.scripts/logging_utils.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -# Provide a unified interface for the different logging -# utilities CI providers offer. If unavailable, provide -# a compatible fallback (e.g. bare `echo xxxxxx`). - -function startgroup { - # Start a foldable group of log lines - # Pass a single argument, quoted - case ${CI:-} in - azure ) - echo "##[group]$1";; - travis ) - echo "$1" - echo -en 'travis_fold:start:'"${1// /}"'\\r';; - * ) - echo "$1";; - esac -} - -function endgroup { - # End a foldable group of log lines - # Pass a single argument, quoted - case ${CI:-} in - azure ) - echo "##[endgroup]";; - travis ) - echo -en 'travis_fold:end:'"${1// /}"'\\r';; - esac -} diff --git a/dev/tasks/conda-recipes/README.md b/dev/tasks/conda-recipes/README.md deleted file mode 100644 index 39f82f1b01a9..000000000000 --- a/dev/tasks/conda-recipes/README.md +++ /dev/null @@ -1,67 +0,0 @@ - - -# Conda Forge recipes - -This directory must be migrated periodically with the upstrem updates of -[arrow-cpp-feedstock][arrow-cpp-feedstock], -[parquet-cpp-feedstock][parquet-cpp-feedstock]. -conda-forge repositories because of multiple vendored files. - -## Keeping the recipes synchronized - -The recipes here are tested on nightly basis, so they follow the development -versions of arrow instead of the upstream recipes, which are suitable for the -latest releases. - -### Backporting from the upstream feedstocks - -In most of the cases these recipes are more accurate, then the upstream -feedstocks. Although the upstream feedstocks regularly receive automatic updates -by the conda-forge team so we need to backport those changes to the crossbow -recipes. Most of these updates are touching the version pinning files -(under `.ci_support`) and other CI related configuration files. - -Because all three recipes must be built in the same continuous integration -job prefer porting from the [arrpw-cpp feedstock][arrow-cpp-feedstock]. - -#### Updating the variants: - -Copy the configuration files from `arrow-cpp-feedstock/.ci_support` to the -`.ci_support` folder. - -#### Updating the CI configurations: - -The `.azure-pipelines/azure-pipelines-[linux|osx|win].yml` should be ported -to the local counterparts under `.azure-pipelines` with keeping the crossbow -related parts (the cloning of arrow and the jinja templated variables) and -moving the matrix definitions like [this][matrix-definition] to the crossbow -[tasks.yml][../tasks.yml] config file. - - -### Porting recipes from crossbow to the upstream feedstocks - -Theoretically these recipes should be up to date with the actual version of -Arrow, so during the release procedure the content of these recipes should be -copied to the upstream feedstocks. - - -[arrow-cpp-feedstock]: https://github.com/conda-forge/arrow-cpp-feedstock -[parquet-cpp-feedstock]: https://github.com/conda-forge/parquet-cpp-feedstock -[matrix-definition]: https://github.com/conda-forge/arrow-cpp-feedstock/blob/master/.azure-pipelines/azure-pipelines-linux.yml#L12 diff --git a/dev/tasks/conda-recipes/arrow-cpp/LLVM_LICENSE.txt b/dev/tasks/conda-recipes/arrow-cpp/LLVM_LICENSE.txt deleted file mode 100644 index 461398bab7a7..000000000000 --- a/dev/tasks/conda-recipes/arrow-cpp/LLVM_LICENSE.txt +++ /dev/null @@ -1,68 +0,0 @@ -============================================================================== -LLVM Release License -============================================================================== -University of Illinois/NCSA -Open Source License - -Copyright (c) 2003-2018 University of Illinois at Urbana-Champaign. -All rights reserved. - -Developed by: - - LLVM Team - - University of Illinois at Urbana-Champaign - - http://llvm.org - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal with -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimers. - - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimers in the - documentation and/or other materials provided with the distribution. - - * Neither the names of the LLVM Team, University of Illinois at - Urbana-Champaign, nor the names of its contributors may be used to - endorse or promote products derived from this Software without specific - prior written permission. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE -SOFTWARE. - -============================================================================== -Copyrights and Licenses for Third Party Software Distributed with LLVM: -============================================================================== -The LLVM software contains code written by third parties. Such software will -have its own individual LICENSE.TXT file in the directory in which it appears. -This file will describe the copyrights, license, and restrictions which apply -to that code. - -The disclaimer of warranty in the University of Illinois Open Source License -applies to all code in the LLVM Distribution, and nothing in any of the -other licenses gives permission to use the names of the LLVM Team or the -University of Illinois to endorse or promote products derived from this -Software. - -The following pieces of software have additional or alternate copyrights, -licenses, and/or restrictions: - -Program Directory -------- --------- -Google Test llvm/utils/unittest/googletest -OpenBSD regex llvm/lib/Support/{reg*, COPYRIGHT.regex} -pyyaml tests llvm/test/YAMLParser/{*.data, LICENSE.TXT} -ARM contributions llvm/lib/Target/ARM/LICENSE.TXT -md5 contributions llvm/lib/Support/MD5.cpp llvm/include/llvm/Support/MD5.h diff --git a/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat b/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat deleted file mode 100644 index 2cc6ed1ba3ec..000000000000 --- a/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat +++ /dev/null @@ -1,54 +0,0 @@ -@echo on - -mkdir "%SRC_DIR%"\cpp\build -pushd "%SRC_DIR%"\cpp\build - -:: Enable CUDA support -if "%cuda_compiler_version%"=="None" ( - set "EXTRA_CMAKE_ARGS=-DARROW_CUDA=OFF" -) else ( - REM this should move to nvcc-feedstock - set "CUDA_PATH=%CUDA_PATH:\=/%" - set "CUDA_HOME=%CUDA_HOME:\=/%" - - set "EXTRA_CMAKE_ARGS=-DARROW_CUDA=ON" -) - -cmake -G "Ninja" ^ - -DBUILD_SHARED_LIBS=ON ^ - -DCMAKE_INSTALL_PREFIX="%LIBRARY_PREFIX%" ^ - -DARROW_DEPENDENCY_SOURCE=SYSTEM ^ - -DARROW_PACKAGE_PREFIX="%LIBRARY_PREFIX%" ^ - -DLLVM_TOOLS_BINARY_DIR="%LIBRARY_BIN%" ^ - -DPython3_EXECUTABLE="%PYTHON%" ^ - -DARROW_WITH_BZ2:BOOL=ON ^ - -DARROW_WITH_ZLIB:BOOL=ON ^ - -DARROW_WITH_ZSTD:BOOL=ON ^ - -DARROW_WITH_LZ4:BOOL=ON ^ - -DARROW_WITH_SNAPPY:BOOL=ON ^ - -DARROW_WITH_BROTLI:BOOL=ON ^ - -DARROW_BOOST_USE_SHARED:BOOL=ON ^ - -DARROW_BUILD_TESTS:BOOL=OFF ^ - -DARROW_BUILD_UTILITIES:BOOL=OFF ^ - -DARROW_BUILD_STATIC:BOOL=OFF ^ - -DCMAKE_BUILD_TYPE=release ^ - -DARROW_SSE42:BOOL=OFF ^ - -DARROW_PYTHON:BOOL=ON ^ - -DARROW_MIMALLOC:BOOL=ON ^ - -DARROW_DATASET:BOOL=ON ^ - -DARROW_FLIGHT:BOOL=ON ^ - -DARROW_HDFS:BOOL=ON ^ - -DARROW_PARQUET:BOOL=ON ^ - -DARROW_GANDIVA:BOOL=ON ^ - -DARROW_ORC:BOOL=ON ^ - -DARROW_S3:BOOL=ON ^ - -DBoost_NO_BOOST_CMAKE=ON ^ - -DCMAKE_UNITY_BUILD=ON ^ - %EXTRA_CMAKE_ARGS% ^ - .. -if errorlevel 1 exit 1 - -cmake --build . --target install --config Release -if errorlevel 1 exit 1 - -popd diff --git a/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat b/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat deleted file mode 100644 index 89cec3710c3e..000000000000 --- a/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat +++ /dev/null @@ -1,44 +0,0 @@ -@echo on -pushd "%SRC_DIR%"\python - -@rem the symlinks for cmake modules don't work here -@rem NOTE: In contrast to conda-forge, they work here as we clone from git. -@rem del cmake_modules\BuildUtils.cmake -@rem del cmake_modules\SetupCxxFlags.cmake -@rem del cmake_modules\CompilerInfo.cmake -@rem del cmake_modules\FindNumPy.cmake -@rem del cmake_modules\FindPythonLibsNew.cmake -@rem copy /Y "%SRC_DIR%\cpp\cmake_modules\BuildUtils.cmake" cmake_modules\ -@rem copy /Y "%SRC_DIR%\cpp\cmake_modules\SetupCxxFlags.cmake" cmake_modules\ -@rem copy /Y "%SRC_DIR%\cpp\cmake_modules\CompilerInfo.cmake" cmake_modules\ -@rem copy /Y "%SRC_DIR%\cpp\cmake_modules\FindNumPy.cmake" cmake_modules\ -@rem copy /Y "%SRC_DIR%\cpp\cmake_modules\FindPythonLibsNew.cmake" cmake_modules\ - -SET ARROW_HOME=%LIBRARY_PREFIX% -SET SETUPTOOLS_SCM_PRETEND_VERSION=%PKG_VERSION% -SET PYARROW_BUILD_TYPE=release -SET PYARROW_WITH_S3=1 -SET PYARROW_WITH_HDFS=1 -SET PYARROW_WITH_DATASET=1 -SET PYARROW_WITH_FLIGHT=1 -SET PYARROW_WITH_GANDIVA=1 -SET PYARROW_WITH_PARQUET=1 -SET PYARROW_CMAKE_GENERATOR=Ninja - -:: Enable CUDA support -if "%cuda_compiler_version%"=="None" ( - set "PYARROW_WITH_CUDA=0" -) else ( - set "PYARROW_WITH_CUDA=1" -) - -%PYTHON% setup.py ^ - build_ext ^ - install --single-version-externally-managed ^ - --record=record.txt -if errorlevel 1 exit 1 -popd - -if [%PKG_NAME%] == [pyarrow] ( - rd /s /q %SP_DIR%\pyarrow\tests -) diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh deleted file mode 100644 index f9c1d975ec34..000000000000 --- a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/bin/bash - -set -e -set -x - -mkdir cpp/build -pushd cpp/build - -EXTRA_CMAKE_ARGS="" - -# Include g++'s system headers -if [ "$(uname)" == "Linux" ]; then - SYSTEM_INCLUDES=$(echo | ${CXX} -E -Wp,-v -xc++ - 2>&1 | grep '^ ' | awk '{print "-isystem;" substr($1, 1)}' | tr '\n' ';') - EXTRA_CMAKE_ARGS=" -DARROW_GANDIVA_PC_CXX_FLAGS=${SYSTEM_INCLUDES}" -fi - -# Enable CUDA support -if [[ ! -z "${cuda_compiler_version+x}" && "${cuda_compiler_version}" != "None" ]] -then - if [[ -z "${CUDA_HOME+x}" ]] - then - echo "cuda_compiler_version=${cuda_compiler_version} CUDA_HOME=$CUDA_HOME" - CUDA_GDB_EXECUTABLE=$(which cuda-gdb || exit 0) - if [[ -n "$CUDA_GDB_EXECUTABLE" ]] - then - CUDA_HOME=$(dirname $(dirname $CUDA_GDB_EXECUTABLE)) - else - echo "Cannot determine CUDA_HOME: cuda-gdb not in PATH" - return 1 - fi - fi - EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_CUDA=ON -DCUDA_TOOLKIT_ROOT_DIR=${CUDA_HOME} -DCMAKE_LIBRARY_PATH=${CUDA_HOME}/lib64/stubs" -else - EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_CUDA=OFF" -fi - -if [[ "${target_platform}" == "osx-arm64" ]]; then - # We need llvm 11+ support in Arrow for this - EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_GANDIVA=OFF" - sed -ie "s;protoc-gen-grpc.*$;protoc-gen-grpc=${BUILD_PREFIX}/bin/grpc_cpp_plugin\";g" ../src/arrow/flight/CMakeLists.txt - sed -ie 's;"--with-jemalloc-prefix\=je_arrow_";"--with-jemalloc-prefix\=je_arrow_" "--with-lg-page\=14";g' ../cmake_modules/ThirdpartyToolchain.cmake -else - EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_GANDIVA=ON" -fi - -cmake \ - -DARROW_BOOST_USE_SHARED=ON \ - -DARROW_BUILD_BENCHMARKS=OFF \ - -DARROW_BUILD_STATIC=OFF \ - -DARROW_BUILD_TESTS=OFF \ - -DARROW_BUILD_UTILITIES=OFF \ - -DBUILD_SHARED_LIBS=ON \ - -DARROW_DATASET=ON \ - -DARROW_DEPENDENCY_SOURCE=SYSTEM \ - -DARROW_FLIGHT=ON \ - -DARROW_FLIGHT_REQUIRE_TLSCREDENTIALSOPTIONS=ON \ - -DARROW_HDFS=ON \ - -DARROW_JEMALLOC=ON \ - -DARROW_MIMALLOC=ON \ - -DARROW_ORC=ON \ - -DARROW_PACKAGE_PREFIX=$PREFIX \ - -DARROW_PARQUET=ON \ - -DARROW_PLASMA=ON \ - -DARROW_PYTHON=ON \ - -DARROW_S3=ON \ - -DARROW_SIMD_LEVEL=NONE \ - -DARROW_USE_LD_GOLD=ON \ - -DARROW_WITH_BROTLI=ON \ - -DARROW_WITH_BZ2=ON \ - -DARROW_WITH_LZ4=ON \ - -DARROW_WITH_SNAPPY=ON \ - -DARROW_WITH_ZLIB=ON \ - -DARROW_WITH_ZSTD=ON \ - -DCMAKE_BUILD_TYPE=release \ - -DCMAKE_INSTALL_LIBDIR=lib \ - -DCMAKE_INSTALL_PREFIX=$PREFIX \ - -DLLVM_TOOLS_BINARY_DIR=$PREFIX/bin \ - -DPython3_EXECUTABLE=${PYTHON} \ - -DProtobuf_PROTOC_EXECUTABLE=$BUILD_PREFIX/bin/protoc \ - -GNinja \ - ${EXTRA_CMAKE_ARGS} \ - .. - -# Commented out until jemalloc and mimalloc are fixed upstream -if [[ "${target_platform}" == "osx-arm64" ]]; then - ninja jemalloc_ep-prefix/src/jemalloc_ep-stamp/jemalloc_ep-patch mimalloc_ep-prefix/src/mimalloc_ep-stamp/mimalloc_ep-patch - cp $BUILD_PREFIX/share/gnuconfig/config.* jemalloc_ep-prefix/src/jemalloc_ep/build-aux/ - sed -ie 's/list(APPEND mi_cflags -march=native)//g' mimalloc_ep-prefix/src/mimalloc_ep/CMakeLists.txt - # Use the correct register for thread-local storage - sed -ie 's/tpidr_el0/tpidrro_el0/g' mimalloc_ep-prefix/src/mimalloc_ep/include/mimalloc-internal.h -fi - -ninja install - -popd diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh b/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh deleted file mode 100644 index a394e999f7b2..000000000000 --- a/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/sh - -set -e -set -x - -# Build dependencies -export ARROW_HOME=$PREFIX -export PARQUET_HOME=$PREFIX -export SETUPTOOLS_SCM_PRETEND_VERSION=$PKG_VERSION -export PYARROW_BUILD_TYPE=release -export PYARROW_BUNDLE_ARROW_CPP_HEADERS=0 -export PYARROW_WITH_DATASET=1 -export PYARROW_WITH_FLIGHT=1 -if [[ "${target_platform}" == "osx-arm64" ]]; then - # We need llvm 11+ support in Arrow for this - export PYARROW_WITH_GANDIVA=0 -else - export PYARROW_WITH_GANDIVA=1 -fi -export PYARROW_WITH_HDFS=1 -export PYARROW_WITH_ORC=1 -export PYARROW_WITH_PARQUET=1 -export PYARROW_WITH_PLASMA=1 -export PYARROW_WITH_S3=1 -export PYARROW_CMAKE_GENERATOR=Ninja -BUILD_EXT_FLAGS="" - -# Enable CUDA support -if [[ ! -z "${cuda_compiler_version+x}" && "${cuda_compiler_version}" != "None" ]]; then - export PYARROW_WITH_CUDA=1 -else - export PYARROW_WITH_CUDA=0 -fi - -# Resolve: Make Error at cmake_modules/SetupCxxFlags.cmake:338 (message): Unsupported arch flag: -march=. -if [[ "${target_platform}" == "linux-aarch64" ]]; then - export PYARROW_CMAKE_OPTIONS="-DARROW_ARMV8_ARCH=armv8-a" -fi - -cd python - -$PYTHON setup.py \ - build_ext \ - install --single-version-externally-managed \ - --record=record.txt - -if [[ "$PKG_NAME" == "pyarrow" ]]; then - rm -r ${SP_DIR}/pyarrow/tests -fi diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml deleted file mode 100644 index 48a8629866d9..000000000000 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ /dev/null @@ -1,302 +0,0 @@ -# NOTE: In constrast to the conda-forge recipe, ARROW_VERSION is a templated variable here. -{% set version = ARROW_VERSION %} -{% set cuda_enabled = cuda_compiler_version != "None" %} -{% set build_ext_version = ARROW_VERSION %} -{% set build_ext = "cuda" if cuda_enabled else "cpu" %} -{% set proc_build_number = "0" %} - -package: - name: arrow-cpp-ext - version: {{ version }} - -source: - path: ../../../../ - -build: - number: 0 - # for cuda on win/linux, building with 9.2 is enough to be compatible with all later versions, - # since arrow is only using libcuda, and not libcudart. - skip: true # [(win or linux) and cuda_compiler_version not in ("None", "10.2")] - skip: true # [osx and cuda_compiler_version != "None"] - run_exports: - - {{ pin_subpackage("arrow-cpp", max_pin="x.x.x") }} - -outputs: - - name: arrow-cpp-proc - version: {{ build_ext_version }} - build: - number: {{ proc_build_number }} - string: "{{ build_ext }}" - test: - commands: - - exit 0 - about: - home: http://github.com/apache/arrow - license: Apache-2.0 - license_file: - - LICENSE.txt - summary: 'A meta-package to select Arrow build variant' - - - name: arrow-cpp - script: build-arrow.sh # [not win] - script: bld-arrow.bat # [win] - version: {{ version }} - build: - string: py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }} - run_exports: - - {{ pin_subpackage("arrow-cpp", max_pin="x.x.x") }} - ignore_run_exports: - - cudatoolkit - track_features: - {{ "- arrow-cuda" if cuda_enabled else "" }} - requirements: - build: - - python # [build_platform != target_platform] - - cross-python_{{ target_platform }} # [build_platform != target_platform] - - cython # [build_platform != target_platform] - - numpy # [build_platform != target_platform] - - gnuconfig # [osx and arm64] - - libprotobuf - - grpc-cpp - - cmake - - autoconf # [unix] - - ninja - - make # [unix] - - {{ compiler('c') }} - - {{ compiler('cxx') }} - - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] - host: - - aws-sdk-cpp - - boost-cpp >=1.70 - - brotli - - bzip2 - - c-ares - - gflags - - glog - - grpc-cpp - - libprotobuf - - clangdev 10 # [not (osx and arm64)] - - llvmdev 10 # [not (osx and arm64)] - - libutf8proc - - lz4-c - - numpy - - orc # [unix] - - python - - rapidjson - - re2 - - snappy - - thrift-cpp - - zlib - - zstd - run: - - {{ pin_compatible('numpy', lower_bound='1.16') }} - - python - run_constrained: - - arrow-cpp-proc * {{ build_ext }} - - cudatoolkit >=9.2 # [cuda_compiler_version != "None"] - - about: - home: http://github.com/apache/arrow - license: Apache-2.0 - license_file: - - LICENSE.txt - summary: C++ libraries for Apache Arrow - - test: - commands: - # headers - - test -f $PREFIX/include/arrow/api.h # [unix] - - test -f $PREFIX/include/arrow/flight/types.h # [unix] - - test -f $PREFIX/include/plasma/client.h # [unix] - - test -f $PREFIX/include/gandiva/engine.h # [unix and not (osx and arm64)] - - test -f $PREFIX/include/parquet/api/reader.h # [unix] - - if not exist %LIBRARY_INC%\\arrow\\api.h exit 1 # [win] - - if not exist %LIBRARY_INC%\\gandiva\\engine.h exit 1 # [win] - - if not exist %LIBRARY_INC%\\parquet\\api\\reader.h exit 1 # [win] - - # shared - - test -f $PREFIX/lib/libarrow.so # [linux] - - test -f $PREFIX/lib/libarrow_dataset.so # [linux] - - test -f $PREFIX/lib/libarrow_flight.so # [linux] - - test -f $PREFIX/lib/libarrow_python.so # [linux] - - test -f $PREFIX/lib/libparquet.so # [linux] - - test -f $PREFIX/lib/libgandiva.so # [linux] - - test -f $PREFIX/lib/libplasma.so # [linux] - - test -f $PREFIX/lib/libarrow_cuda${SHLIB_EXT} # [(cuda_compiler_version != "None") and unix] - - test ! -f $PREFIX/lib/libarrow_cuda${SHLIB_EXT} # [(cuda_compiler_version == "None") and unix] - - if not exist %PREFIX%\\Library\\bin\\arrow_cuda.dll exit 1 # [(cuda_compiler_version != "None") and win] - - if exist %PREFIX%\\Library\\bin\\arrow_cuda.dll exit 1 # [(cuda_compiler_version == "None") and win] - - test -f $PREFIX/lib/libarrow.dylib # [osx] - - test -f $PREFIX/lib/libarrow_dataset.dylib # [osx] - - test -f $PREFIX/lib/libarrow_python.dylib # [osx] - - test -f $PREFIX/lib/libgandiva.dylib # [osx and not arm64] - - test -f $PREFIX/lib/libparquet.dylib # [osx] - - test -f $PREFIX/lib/libplasma.dylib # [osx] - - if not exist %PREFIX%\\Library\\bin\\arrow.dll exit 1 # [win] - - if not exist %PREFIX%\\Library\\bin\\arrow_dataset.dll exit 1 # [win] - - if not exist %PREFIX%\\Library\\bin\\arrow_flight.dll exit 1 # [win] - - if not exist %PREFIX%\\Library\\bin\\arrow_python.dll exit 1 # [win] - - if not exist %PREFIX%\\Library\\bin\\parquet.dll exit 1 # [win] - - if not exist %PREFIX%\\Library\\bin\\gandiva.dll exit 1 # [win] - - # absence of static libraries - - test ! -f $PREFIX/lib/libarrow.a # [unix] - - test ! -f $PREFIX/lib/libarrow_dataset.a # [unix] - - test ! -f $PREFIX/lib/libarrow_flight.a # [unix] - - test ! -f $PREFIX/lib/libarrow_python.a # [unix] - - test ! -f $PREFIX/lib/libplasma.a # [unix] - - test ! -f $PREFIX/lib/libparquet.a # [unix] - - test ! -f $PREFIX/lib/libgandiva.a # [unix] - - if exist %PREFIX%\\Library\\lib\\arrow_static.lib exit 1 # [win] - - if exist %PREFIX%\\Library\\lib\\arrow_dataset_static.lib exit 1 # [win] - - if exist %PREFIX%\\Library\\lib\\arrow_flight_static.lib exit 1 # [win] - - if exist %PREFIX%\\Library\\lib\\arrow_python_static.lib exit 1 # [win] - - if exist %PREFIX%\\Library\\lib\\parquet_static.lib exit 1 # [win] - - if exist %PREFIX%\\Library\\lib\\gandiva_static.lib exit 1 # [win] - - - name: pyarrow - script: build-pyarrow.sh # [not win] - script: bld-pyarrow.bat # [win] - version: {{ version }} - build: - string: py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }} - ignore_run_exports: - - cudatoolkit - track_features: - {{ "- arrow-cuda" if cuda_enabled else "" }} - requirements: - build: - - python # [build_platform != target_platform] - - cross-python_{{ target_platform }} # [build_platform != target_platform] - - cython # [build_platform != target_platform] - - numpy # [build_platform != target_platform] - - cmake - - ninja - - make # [unix] - - {{ compiler('c') }} - - {{ compiler('cxx') }} - # pyarrow does not require nvcc but it needs to link against libraries in arrow-cpp=*=*cuda - - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] - host: - - {{ pin_subpackage('arrow-cpp', exact=True) }} - - cython - - numpy - - python - - setuptools - - setuptools_scm - - six - run: - - {{ pin_subpackage('arrow-cpp', exact=True) }} - - {{ pin_compatible('numpy', lower_bound='1.16') }} - # empty parquet-cpp metapackage, force old versions to be uninstalled - - parquet-cpp 1.5.1.* - - python - run_constrained: - - arrow-cpp-proc * {{ build_ext }} - - cudatoolkit >=9.2 # [cuda_compiler_version != "None"] - - about: - home: http://github.com/apache/arrow - license: Apache-2.0 - license_file: - - LICENSE.txt - summary: Python libraries for Apache Arrow - - test: - imports: - - pyarrow - - pyarrow.dataset - - pyarrow.flight - - pyarrow.gandiva # [not (osx and arm64)] - - pyarrow.orc # [unix] - - pyarrow.parquet - - pyarrow.plasma # [unix] - - pyarrow.fs - - pyarrow._s3fs - - pyarrow._hdfs - # We can only test importing cuda package but cannot run when a - # CUDA device is not available, for instance, when building from CI. - # On Windows, we cannot even do that due to `nvcuda.dll` not being found, see - # https://conda-forge.org/docs/maintainer/knowledge_base.html#nvcuda-dll-cannot-be-found-on-windows - # However, we check below for (at least) the presence of a correctly-compiled module - - pyarrow.cuda # [cuda_compiler_version != "None" and not win] - commands: - - test ! -f ${SP_DIR}/pyarrow/tests/test_array.py # [unix] - - if exist %SP_DIR%/pyarrow/tests/test_array.py exit 1 # [win] - # Need to remove dot from PY_VER; %MYVAR:x=y% replaces "x" in %MYVAR% with "y" - - if not exist %SP_DIR%/pyarrow/_cuda.cp%PY_VER:.=%-win_amd64.pyd exit 1 # [win and cuda_compiler_version != "None"] - - - name: pyarrow-tests - script: build-pyarrow.sh # [not win] - script: bld-pyarrow.bat # [win] - version: {{ version }} - build: - string: py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }} - ignore_run_exports: - - cudatoolkit - track_features: - {{ "- arrow-cuda" if cuda_enabled else "" }} - requirements: - build: - - python # [build_platform != target_platform] - - cross-python_{{ target_platform }} # [build_platform != target_platform] - - cython # [build_platform != target_platform] - - numpy # [build_platform != target_platform] - - cmake - - ninja - - make # [unix] - - {{ compiler('c') }} - - {{ compiler('cxx') }} - # pyarrow does not require nvcc but it needs to link against libraries in arrow-cpp=*=*cuda - - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] - host: - - {{ pin_subpackage('arrow-cpp', exact=True) }} - - {{ pin_subpackage('pyarrow', exact=True) }} - - cython - - numpy - - python - - setuptools - - setuptools_scm - - six - run: - - {{ pin_subpackage('pyarrow', exact=True) }} - - python - run_constrained: - - arrow-cpp-proc * {{ build_ext }} - - cudatoolkit >=9.2 # [cuda_compiler_version != "None"] - - about: - home: http://github.com/apache/arrow - license: Apache-2.0 - license_file: - - LICENSE.txt - summary: Python test files for Apache Arrow - - test: - commands: - - test -f ${SP_DIR}/pyarrow/tests/test_array.py # [unix] - - if not exist %SP_DIR%/pyarrow/tests/test_array.py exit 1 # [win] - -about: - home: http://github.com/apache/arrow - license: Apache-2.0 - license_file: - - LICENSE.txt - summary: C++ and Python libraries for Apache Arrow - -extra: - recipe-maintainers: - - wesm - - xhochy - - leifwalsh - - jreback - - cpcloud - - pcmoritz - - robertnishihara - - siddharthteotia - - kou - - kszucs - - pitrou - - pearu - - nealrichardson - - jakirkham diff --git a/dev/tasks/conda-recipes/azure.clean.yml b/dev/tasks/conda-recipes/azure.clean.yml deleted file mode 100644 index 55ac36528add..000000000000 --- a/dev/tasks/conda-recipes/azure.clean.yml +++ /dev/null @@ -1,28 +0,0 @@ -jobs: -- job: linux - pool: - vmImage: ubuntu-18.04 - timeoutInMinutes: 360 - - steps: - - script: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive - displayName: Clone arrow - - - script: | - conda install -y -c conda-forge pandas anaconda-client packaging - displayName: Install requirements - - - script: | - {% if arrow.branch == 'master' %} - mkdir -p $HOME/.continuum/anaconda-client/tokens/ - echo $(CROSSBOW_ANACONDA_TOKEN) > $HOME/.continuum/anaconda-client/tokens/https%3A%2F%2Fapi.anaconda.org.token - {% endif %} - eval "$(conda shell.bash hook)" - conda activate base - python3 arrow/dev/tasks/conda-recipes/clean.py {% if arrow.branch == 'master' %}FORCE{% endif %} - displayName: Delete outdated packages - diff --git a/dev/tasks/conda-recipes/azure.linux.yml b/dev/tasks/conda-recipes/azure.linux.yml deleted file mode 100755 index 161fd14e90cb..000000000000 --- a/dev/tasks/conda-recipes/azure.linux.yml +++ /dev/null @@ -1,38 +0,0 @@ -{% import 'macros.jinja' as macros with context %} - -jobs: -- job: linux - pool: - vmImage: ubuntu-16.04 - timeoutInMinutes: 360 - - variables: - CONFIG: {{ config }} - R_CONFIG: {{ r_config|default("") }} - ARROW_VERSION: {{ arrow.no_rc_version }} - UPLOAD_PACKAGES: False - - steps: - # configure qemu binfmt-misc running. This allows us to run docker containers - # embedded qemu-static - - script: | - docker run --rm --privileged multiarch/qemu-user-static:register --reset --credential yes - ls /proc/sys/fs/binfmt_misc/ - displayName: Configure binfmt_misc - condition: not(startsWith(variables['CONFIG'], 'linux_64')) - - {{ macros.azure_checkout_arrow() }} - - - task: CondaEnvironment@1 - inputs: - packageSpecs: 'anaconda-client shyaml' - installOptions: '-c conda-forge' - updateConda: false - - - script: | - mkdir build_artifacts - CI=azure arrow/dev/tasks/conda-recipes/run_docker_build.sh $(pwd)/build_artifacts - displayName: Run docker build - - {{ macros.azure_upload_releases("build_artifacts/linux-64/*.tar.bz2") }} - {{ macros.azure_upload_anaconda("build_artifacts/linux-64/*.tar.bz2") }} diff --git a/dev/tasks/conda-recipes/azure.osx.yml b/dev/tasks/conda-recipes/azure.osx.yml deleted file mode 100755 index dbb1a68aca6f..000000000000 --- a/dev/tasks/conda-recipes/azure.osx.yml +++ /dev/null @@ -1,80 +0,0 @@ -{% import 'macros.jinja' as macros with context %} - -jobs: -- job: osx - pool: - vmImage: macOS-10.14 - timeoutInMinutes: 360 - variables: - CONFIG: {{ config }} - R_CONFIG: {{ r_config|default("") }} - ARROW_VERSION: {{ arrow.no_rc_version }} - UPLOAD_PACKAGES: False - steps: - - script: | - echo "Removing homebrew from Azure to avoid conflicts." - curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/uninstall > ~/uninstall_homebrew - chmod +x ~/uninstall_homebrew - ~/uninstall_homebrew -fq - rm ~/uninstall_homebrew - displayName: Remove homebrew - - - bash: | - echo "##vso[task.prependpath]$CONDA/bin" - sudo chown -R $USER $CONDA - displayName: Add conda to PATH - - - script: | - source activate base - conda install -n base -c conda-forge --quiet --yes conda-forge-ci-setup=3 conda-build - displayName: 'Add conda-forge-ci-setup=3' - - {{ macros.azure_checkout_arrow() }} - - - script: | - source activate base - echo "Configuring conda." - - setup_conda_rc ./ ./ ./.ci_support/${CONFIG}.yaml - export CI=azure - source run_conda_forge_build_setup - conda update --yes --quiet --override-channels -c conda-forge -c defaults --all - displayName: Configure conda and conda-build - workingDirectory: arrow/dev/tasks/conda-recipes - env: - OSX_FORCE_SDK_DOWNLOAD: "1" - - - script: | - source activate base - mangle_compiler ./ ./ ./.ci_support/${CONFIG}.yaml - workingDirectory: arrow/dev/tasks/conda-recipes - displayName: Mangle compiler - - - script: | - source activate base - make_build_number ./ ./ ./.ci_support/${CONFIG}.yaml - workingDirectory: arrow/dev/tasks/conda-recipes - displayName: Generate build number clobber file - - - script: | - source activate base - set +x - if [[ "${CONFIG}" == osx_arm* ]]; then - EXTRA_CB_OPTIONS="${EXTRA_CB_OPTIONS:-} --no-test" - fi - conda build arrow-cpp \ - -m ./.ci_support/${CONFIG}.yaml \ - --clobber-file ./.ci_support/clobber_${CONFIG}.yaml \ - ${EXTRA_CB_OPTIONS:-} \ - --output-folder ./build_artifacts - - if [ ! -z "${R_CONFIG}" ]; then - conda build r-arrow \ - -m ./.ci_support/r/${R_CONFIG}.yaml \ - --output-folder ./build_artifacts - fi - workingDirectory: arrow/dev/tasks/conda-recipes - displayName: Build recipes - - {{ macros.azure_upload_releases("arrow/dev/tasks/conda-recipes/build_artifacts/osx-*/*.tar.bz2") }} - {{ macros.azure_upload_anaconda("arrow/dev/tasks/conda-recipes/build_artifacts/osx-*/*.tar.bz2") }} diff --git a/dev/tasks/conda-recipes/azure.win.yml b/dev/tasks/conda-recipes/azure.win.yml deleted file mode 100755 index a3ec6931caf2..000000000000 --- a/dev/tasks/conda-recipes/azure.win.yml +++ /dev/null @@ -1,77 +0,0 @@ -{% import 'macros.jinja' as macros with context %} - -jobs: -- job: win - pool: - vmImage: vs2017-win2016 - timeoutInMinutes: 360 - variables: - CONFIG: {{ config }} - R_CONFIG: {{ r_config|default("") }} - ARROW_VERSION: {{ arrow.no_rc_version }} - CONDA_BLD_PATH: D:\\bld\\ - UPLOAD_PACKAGES: False - - steps: - - script: | - choco install vcpython27 -fdv -y --debug - condition: contains(variables['CONFIG'], 'vs2008') - displayName: Install vcpython27.msi (if needed) - - - powershell: | - Set-PSDebug -Trace 1 - $batchcontent = @" - ECHO ON - SET vcpython=C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python\9.0 - DIR "%vcpython%" - CALL "%vcpython%\vcvarsall.bat" %* - "@ - $batchDir = "C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python\9.0\VC" - $batchPath = "$batchDir" + "\vcvarsall.bat" - New-Item -Path $batchPath -ItemType "file" -Force - Set-Content -Value $batchcontent -Path $batchPath - Get-ChildItem -Path $batchDir - Get-ChildItem -Path ($batchDir + '\..') - condition: contains(variables['CONFIG'], 'vs2008') - displayName: Patch vs2008 (if needed) - - - task: CondaEnvironment@1 - inputs: - packageSpecs: 'python=3.6 conda-build conda conda-forge::conda-forge-ci-setup=3 pip' # Optional - installOptions: "-c conda-forge" - updateConda: true - displayName: Install conda-build and activate environment - - script: set PYTHONUNBUFFERED=1 - - {{ macros.azure_checkout_arrow()|indent(2) }} - - # Configure the VM - - script: setup_conda_rc .\ .\ .\.ci_support\%CONFIG%.yaml - workingDirectory: arrow\dev\tasks\conda-recipes - - # Configure the VM. - - script: | - set "CI=azure" - call activate base - run_conda_forge_build_setup - displayName: conda-forge build setup - workingDirectory: arrow\dev\tasks\conda-recipes - - - script: | - conda.exe build arrow-cpp parquet-cpp -m .ci_support\%CONFIG%.yaml - displayName: Build recipe - workingDirectory: arrow\dev\tasks\conda-recipes - env: - PYTHONUNBUFFERED: 1 - condition: not(contains(variables['CONFIG'], 'vs2008')) - - - script: | - conda.exe build r-arrow -m .ci_support\r\%R_CONFIG%.yaml - displayName: Build recipe - workingDirectory: arrow\dev\tasks\conda-recipes - env: - PYTHONUNBUFFERED: 1 - condition: contains(variables['R_CONFIG'], 'win') - - {{ macros.azure_upload_releases("D:\bld\win-64\*.tar.bz2")|indent(2) }} - {{ macros.azure_upload_anaconda("D:\bld\win-64\*.tar.bz2")|indent(2) }} diff --git a/dev/tasks/conda-recipes/build_steps.sh b/dev/tasks/conda-recipes/build_steps.sh deleted file mode 100755 index 25864c08a708..000000000000 --- a/dev/tasks/conda-recipes/build_steps.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash - -# NOTE: This script has been slightly adopted to suite the Apache Arrow / crossbow CI -# setup. The next time this is updated to the current version on conda-forge, -# you will also make this additions afterwards. - -# PLEASE NOTE: This script has been automatically generated by conda-smithy. Any changes here -# will be lost next time ``conda smithy rerender`` is run. If you would like to make permanent -# changes to this script, consider a proposal to conda-smithy so that other feedstocks can also -# benefit from the improvement. - -set -xeuo pipefail - -output_dir=${1} - -export PYTHONUNBUFFERED=1 -export FEEDSTOCK_ROOT="${FEEDSTOCK_ROOT:-/home/conda/feedstock_root}" -export CI_SUPPORT="${FEEDSTOCK_ROOT}/.ci_support" -export CONFIG_FILE="${CI_SUPPORT}/${CONFIG}.yaml" - -cat >~/.condarc < List[str]: - env = os.environ.copy() - env["CONDA_SUBDIR"] = platform - pkgs_json = check_output( - [ - "conda", - "search", - "--json", - "-c", - "arrow-nightlies", - "--override-channels", - package_name, - ], - env=env, - ) - pkgs = pd.DataFrame(json.loads(pkgs_json)[package_name]) - pkgs["version"] = pkgs["version"].map(Version) - pkgs["py_version"] = pkgs["build"].str.slice(0, 4) - - to_delete = [] - - for (subdir, python), group in pkgs.groupby(["subdir", "py_version"]): - group = group.sort_values(by="version", ascending=False) - - if len(group) > VERSIONS_TO_KEEP: - del_candidates = group[VERSIONS_TO_KEEP:] - to_delete += ( - f"arrow-nightlies/{package_name}/" - + del_candidates["version"].astype(str) - + del_candidates["url"].str.replace( - "https://conda.anaconda.org/arrow-nightlies", "" - ) - ).to_list() - - return to_delete - - -if __name__ == "__main__": - to_delete = [] - for package in PACKAGES: - for platform in PLATFORMS: - if [package, platform] in EXCLUDED_PATTERNS: - continue - to_delete += packages_to_delete(package, platform) - - for name in to_delete: - print(f"Deleting {name} …") - if "FORCE" in sys.argv: - check_call(["anaconda", "remove", "-f", name]) diff --git a/dev/tasks/conda-recipes/conda-forge.yml b/dev/tasks/conda-recipes/conda-forge.yml deleted file mode 100644 index 4c07b5dd3e0b..000000000000 --- a/dev/tasks/conda-recipes/conda-forge.yml +++ /dev/null @@ -1 +0,0 @@ -channel_priority: strict diff --git a/dev/tasks/conda-recipes/drone-steps.sh b/dev/tasks/conda-recipes/drone-steps.sh deleted file mode 100755 index dffdb41b0887..000000000000 --- a/dev/tasks/conda-recipes/drone-steps.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -set -ex - -OUTPUT_DIR=$1 -QUEUE_REMOTE_URL=$2 -TASK_BRANCH=$3 -TASK_TAG=$4 -UPLOAD_TO_ANACONDA=$5 - -conda install -y mamba -$FEEDSTOCK_ROOT/build_steps.sh ${OUTPUT_DIR} - -# Upload as Github release -mamba install -y anaconda-client shyaml -c conda-forge - -pushd $DRONE_WORKSPACE -pip install -e arrow/dev/archery[crossbow] -archery crossbow \ - --queue-path . \ - --queue-remote ${QUEUE_REMOTE_URL} \ - upload-artifacts \ - --sha ${TASK_BRANCH} \ - --tag ${TASK_TAG} \ - --pattern "${OUTPUT_DIR}/linux-aarch64/*.tar.bz2" - -if [[ "${UPLOAD_TO_ANACONDA}" == "1" ]]; then - anaconda -t ${CROSSBOW_ANACONDA_TOKEN} upload --force build_artifacts/linux-aarch64/*.tar.bz2 -fi diff --git a/dev/tasks/conda-recipes/drone.yml b/dev/tasks/conda-recipes/drone.yml deleted file mode 100644 index a461c79b9d4c..000000000000 --- a/dev/tasks/conda-recipes/drone.yml +++ /dev/null @@ -1,43 +0,0 @@ ---- -kind: pipeline -name: {{ config }} - -platform: - os: linux - arch: arm64 - -# Omit double builds with crossbow -trigger: - event: - - push - -steps: -- name: Install and build - image: condaforge/linux-anvil-aarch64 - environment: - CONFIG: {{ config }} - UPLOAD_PACKAGES: False - ARROW_VERSION: {{ arrow.no_rc_version }} - PLATFORM: linux-aarch64 - BINSTAR_TOKEN: - from_secret: BINSTAR_TOKEN - FEEDSTOCK_TOKEN: - from_secret: FEEDSTOCK_TOKEN - STAGING_BINSTAR_TOKEN: - from_secret: STAGING_BINSTAR_TOKEN - CROSSBOW_GITHUB_TOKEN: - from_secret: CROSSBOW_GITHUB_TOKEN - CROSSBOW_ANACONDA_TOKEN: - from_secret: CROSSBOW_ANACONDA_TOKEN - commands: - - export RECIPE_ROOT="$FEEDSTOCK_ROOT/arrow-cpp" - - export CI=drone - - export GIT_BRANCH="{{ arrow.branch }}" - - export FEEDSTOCK_NAME=arrow-cpp - - export FEEDSTOCK_ROOT="$DRONE_WORKSPACE/arrow/dev/tasks/conda-recipes" - - sed -i '$ichown -R conda:conda "$FEEDSTOCK_ROOT"' /opt/docker/bin/entrypoint - - yum install -y git - - git clone --no-checkout {{ arrow.remote }} arrow - - pushd arrow && git fetch -t {{ arrow.remote }} {{ arrow.branch }} && git checkout FETCH_HEAD && git submodule update --init --recursive && popd - - mkdir -p $(pwd)/build_artifacts && chmod a+rwx $(pwd)/build_artifacts - - /opt/docker/bin/entrypoint $FEEDSTOCK_ROOT/drone-steps.sh $(pwd)/build_artifacts {{ queue_remote_url }} {{ task.branch }} {{ task.tag }} {% if arrow.branch == 'master' %}1{% else %}0{% endif %} diff --git a/dev/tasks/conda-recipes/parquet-cpp/meta.yaml b/dev/tasks/conda-recipes/parquet-cpp/meta.yaml deleted file mode 100644 index 5de06c32b1df..000000000000 --- a/dev/tasks/conda-recipes/parquet-cpp/meta.yaml +++ /dev/null @@ -1,51 +0,0 @@ -# ARROW-3229: this is a meta-package to prevent conflicts in the future - -{% set parquet_version = "1.5.1" %} - -package: - name: parquet-cpp - version: {{ parquet_version }} - -build: - number: 0 - skip: true # [win32] - skip: true # [win and py<35] - -requirements: - host: - # NOTE: in the upstream feedstock use >= instead of = - - arrow-cpp ={{ ARROW_VERSION }} - run: - - arrow-cpp ={{ ARROW_VERSION }} - -test: - commands: - # headers - - test -f $PREFIX/include/parquet/api/reader.h # [unix] - - if not exist %LIBRARY_INC%\\parquet\\api\\reader.h exit 1 # [win] - - # shared - - test -f $PREFIX/lib/libparquet.so # [linux] - - test -f $PREFIX/lib/libparquet.dylib # [osx] - - if not exist %PREFIX%\\Library\\bin\\parquet.dll exit 1 # [win] - - # absence of static libraries - - test ! -f $PREFIX/lib/libparquet.a # [unix] - - if exist %PREFIX%\\Library\\lib\\parquet_static.lib exit 1 # [win] - -about: - home: http://github.com/apache/arrow - license: Apache 2.0 - summary: 'C++ libraries for the Apache Parquet file format' - -extra: - recipe-maintainers: - - wesm - - xhochy - - leifwalsh - - jreback - - cpcloud - - siddharthteotia - - kou - - kszucs - - pitrou diff --git a/dev/tasks/conda-recipes/r-arrow/bld.bat b/dev/tasks/conda-recipes/r-arrow/bld.bat deleted file mode 100644 index a193ddc0a774..000000000000 --- a/dev/tasks/conda-recipes/r-arrow/bld.bat +++ /dev/null @@ -1,9 +0,0 @@ -bash %RECIPE_DIR%/build_win.sh -IF %ERRORLEVEL% NEQ 0 exit 1 -cp %RECIPE_DIR%/configure.win r -IF %ERRORLEVEL% NEQ 0 exit 1 -cp %RECIPE_DIR%/install.libs.R r/src -IF %ERRORLEVEL% NEQ 0 exit 1 -set "MAKEFLAGS=-j%CPU_COUNT%" -"%R%" CMD INSTALL --build r -IF %ERRORLEVEL% NEQ 0 exit 1 diff --git a/dev/tasks/conda-recipes/r-arrow/build.sh b/dev/tasks/conda-recipes/r-arrow/build.sh deleted file mode 100644 index 0a6c7961fc60..000000000000 --- a/dev/tasks/conda-recipes/r-arrow/build.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -export DISABLE_AUTOBREW=1 -$R CMD INSTALL --build r/. diff --git a/dev/tasks/conda-recipes/r-arrow/build_win.sh b/dev/tasks/conda-recipes/r-arrow/build_win.sh deleted file mode 100755 index 88e0462f6638..000000000000 --- a/dev/tasks/conda-recipes/r-arrow/build_win.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set -exuo pipefail - - -# Rename arrow.dll to lib_arrow.dll to avoid conflicts with the arrow-cpp arrow.dll -sed -i -e 's/void R_init_arrow/__declspec(dllexport) void R_init_lib_arrow/g' r/src/arrowExports.cpp -sed -i -e 's/useDynLib(arrow/useDynLib(lib_arrow/g' r/NAMESPACE diff --git a/dev/tasks/conda-recipes/r-arrow/configure.win b/dev/tasks/conda-recipes/r-arrow/configure.win deleted file mode 100755 index 82fa1795699d..000000000000 --- a/dev/tasks/conda-recipes/r-arrow/configure.win +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set -euxo pipefail - -echo "PKG_CPPFLAGS=-DNDEBUG -I\"${LIBRARY_PREFIX}/include\" -I\"${PREFIX}/include\" -DARROW_R_WITH_ARROW -DARROW_R_WITH_PARQUET -DARROW_R_WITH_DATASET -DARROW_R_WITH_S3" > src/Makevars.win -echo "PKG_CXXFLAGS=\$(CXX_VISIBILITY)" >> src/Makevars.win -echo 'CXX_STD=CXX11' >> src/Makevars.win -echo "PKG_LIBS=-L\"${LIBRARY_PREFIX}/lib\" -larrow_dataset -lparquet -larrow" >> src/Makevars.win diff --git a/dev/tasks/conda-recipes/r-arrow/install.libs.R b/dev/tasks/conda-recipes/r-arrow/install.libs.R deleted file mode 100644 index 005bbe16b998..000000000000 --- a/dev/tasks/conda-recipes/r-arrow/install.libs.R +++ /dev/null @@ -1,5 +0,0 @@ -src_dir <- file.path(R_PACKAGE_SOURCE, "src", fsep = "/") -dest_dir <- file.path(R_PACKAGE_DIR, paste0("libs", R_ARCH), fsep="/") - -dir.create(file.path(R_PACKAGE_DIR, paste0("libs", R_ARCH), fsep="/"), recursive = TRUE, showWarnings = FALSE) -file.copy(file.path(src_dir, "arrow.dll", fsep = "/"), file.path(dest_dir, "lib_arrow.dll", fsep = "/")) diff --git a/dev/tasks/conda-recipes/r-arrow/meta.yaml b/dev/tasks/conda-recipes/r-arrow/meta.yaml deleted file mode 100644 index 5f0643bef371..000000000000 --- a/dev/tasks/conda-recipes/r-arrow/meta.yaml +++ /dev/null @@ -1,66 +0,0 @@ -{% set version = ARROW_VERSION %} -{% set posix = 'm2-' if win else '' %} -{% set native = 'm2w64-' if win else '' %} - -package: - name: r-arrow - version: {{ version|replace("-", "_") }} - -source: - path: ../../../../ - -build: - merge_build_host: true # [win] - number: 0 - rpaths: - - lib/R/lib/ - - lib/ - -requirements: - build: - - {{ compiler('c') }} # [not win] - - {{ compiler('cxx') }} # [not win] - - {{ compiler('r_clang') }} # [win] - - pkg-config # [not win] - - {{ posix }}make - - {{ posix }}sed # [win] - - {{ posix }}coreutils # [win] - - {{ posix }}filesystem # [win] - - {{ posix }}zip # [win] - host: - # Needs to be here, otherwise merge_build_host runs into issues - - pkg-config # [win] - - r-base - - arrow-cpp {{ version }} - - r-cpp11 - - r-r6 - - r-assertthat - - r-bit64 - - r-purrr - - r-rlang - - r-tidyselect - run: - - r-base - - r-r6 - - r-assertthat - - r-bit64 - - r-purrr - - r-rlang - - r-tidyselect - -test: - commands: - - $R -e "library('arrow')" # [not win] - - "\"%R%\" -e \"library('arrow'); data(mtcars); write_parquet(mtcars, 'test.parquet')\"" # [win] - -about: - home: https://github.com/apache/arrow - license: Apache-2.0 - license_file: LICENSE.txt - summary: R Integration to 'Apache' 'Arrow'. - license_family: APACHE - -extra: - recipe-maintainers: - - conda-forge/r - - conda-forge/arrow-cpp diff --git a/dev/tasks/conda-recipes/run_docker_build.sh b/dev/tasks/conda-recipes/run_docker_build.sh deleted file mode 100755 index 7645c43e2fa7..000000000000 --- a/dev/tasks/conda-recipes/run_docker_build.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env bash - -# NOTE: This script has been slightly adopted to suite the Apache Arrow / crossbow CI -# setup. The next time this is updated to the current version on conda-forge, -# you will also make this additions afterwards. - -# PLEASE NOTE: This script has been automatically generated by conda-smithy. Any changes here -# will be lost next time ``conda smithy rerender`` is run. If you would like to make permanent -# changes to this script, consider a proposal to conda-smithy so that other feedstocks can also -# benefit from the improvement. - -set -xeo pipefail - -build_dir=${1} - -THISDIR="$( cd "$( dirname "$0" )" >/dev/null && pwd )" -ARROW_ROOT=$(cd "$THISDIR/../../.."; pwd;) -FEEDSTOCK_ROOT=$THISDIR - -docker info - -# In order for the conda-build process in the container to write to the mounted -# volumes, we need to run with the same id as the host machine, which is -# normally the owner of the mounted volumes, or at least has write permission -export HOST_USER_ID=$(id -u) -# Check if docker-machine is being used (normally on OSX) and get the uid from -# the VM -if hash docker-machine 2> /dev/null && docker-machine active > /dev/null; then - export HOST_USER_ID=$(docker-machine ssh $(docker-machine active) id -u) -fi - -if [ -z "$CONFIG" ]; then - set +x - FILES=`ls .ci_support/linux_*` - CONFIGS="" - for file in $FILES; do - CONFIGS="${CONFIGS}'${file:12:-5}' or "; - done - echo "Need to set CONFIG env variable. Value can be one of ${CONFIGS:0:-4}" - exit 1 -fi - -if [ -z "${DOCKER_IMAGE}" ]; then - SHYAML_INSTALLED="$(shyaml -h || echo NO)" - if [ "${SHYAML_INSTALLED}" == "NO" ]; then - echo "WARNING: DOCKER_IMAGE variable not set and shyaml not installed. Falling back to condaforge/linux-anvil-comp7" - DOCKER_IMAGE="condaforge/linux-anvil-comp7" - else - DOCKER_IMAGE="$(cat "${FEEDSTOCK_ROOT}/.ci_support/${CONFIG}.yaml" | shyaml get-value docker_image.0 condaforge/linux-anvil-comp7 )" - fi -fi - -mkdir -p "${build_dir}" -DONE_CANARY="${build_dir}/conda-forge-build-done-${CONFIG}" -rm -f "$DONE_CANARY" - -if [ -z "${CI}" ]; then - DOCKER_RUN_ARGS="-it " -fi - -export UPLOAD_PACKAGES="${UPLOAD_PACKAGES:-True}" -docker run ${DOCKER_RUN_ARGS} \ - --shm-size=2G \ - -v "${ARROW_ROOT}":/arrow:rw,z \ - -v "${build_dir}":/build:rw \ - -e FEEDSTOCK_ROOT="/arrow/dev/tasks/conda-recipes" \ - -e CONFIG \ - -e R_CONFIG \ - -e HOST_USER_ID \ - -e UPLOAD_PACKAGES \ - -e ARROW_VERSION \ - -e CI \ - $DOCKER_IMAGE \ - bash /arrow/dev/tasks/conda-recipes/build_steps.sh /build - -# verify that the end of the script was reached -test -f "$DONE_CANARY" diff --git a/dev/tasks/cpp-examples/github.linux.yml b/dev/tasks/cpp-examples/github.linux.yml deleted file mode 100644 index 717d3c44302a..000000000000 --- a/dev/tasks/cpp-examples/github.linux.yml +++ /dev/null @@ -1,46 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# NOTE: must set "Crossbow" as name to have the badge links working in the -# github comment reports! -name: Crossbow - -on: - push: - branches: - - "*-github-*" - -jobs: - test: - name: C++ Example - runs-on: ubuntu-latest - steps: - - name: Checkout Arrow - shell: bash - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive - - name: Free Up Disk Space - shell: bash - run: arrow/ci/scripts/util_cleanup.sh - - name: Run - shell: bash - run: | - cd arrow/cpp/examples/{{ type }} - docker-compose run --rm {{ run }} diff --git a/dev/tasks/docker-tests/azure.linux.yml b/dev/tasks/docker-tests/azure.linux.yml deleted file mode 100644 index c3706be44346..000000000000 --- a/dev/tasks/docker-tests/azure.linux.yml +++ /dev/null @@ -1,52 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -jobs: -- job: linux - pool: - vmImage: ubuntu-16.04 - timeoutInMinutes: 360 - {% if env is defined %} - variables: - {% for key, value in env.items() %} - {{ key }}: {{ value }} - {% endfor %} - {% endif %} - - steps: - - task: DockerInstaller@0 - displayName: Docker Installer - inputs: - dockerVersion: 17.09.0-ce - releaseType: stable - - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.6' - - - script: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive - displayName: Clone arrow - - - script: pip install -e arrow/dev/archery[docker] - displayName: Setup Archery - - - script: archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION="{{ arrow.no_rc_version }}" {{ run }} - displayName: Execute Docker Build diff --git a/dev/tasks/docker-tests/circle.linux.yml b/dev/tasks/docker-tests/circle.linux.yml deleted file mode 100644 index 3ddb93dc95ef..000000000000 --- a/dev/tasks/docker-tests/circle.linux.yml +++ /dev/null @@ -1,51 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -version: 2 -jobs: - build: - machine: - image: ubuntu-1604:202004-01 - {%- if env is defined %} - environment: - {%- for key, value in env.items() %} - {{ key }}: {{ value }} - {%- endfor %} - {%- endif %} - steps: - - run: | - docker -v - docker-compose -v - - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive - - run: - name: Execute Docker Build - command: | - pyenv versions - pyenv global 3.6.10 - pip install -e arrow/dev/archery[docker] - archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION="{{ arrow.no_rc_version }}" {{ run }} - no_output_timeout: "1h" - -workflows: - version: 2 - build: - jobs: - - build diff --git a/dev/tasks/docker-tests/github.linux.yml b/dev/tasks/docker-tests/github.linux.yml deleted file mode 100644 index 255c9ac14c4c..000000000000 --- a/dev/tasks/docker-tests/github.linux.yml +++ /dev/null @@ -1,42 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -{% import 'macros.jinja' as macros with context %} - -{{ macros.github_header() }} - -jobs: - test: - name: Docker Test - runs-on: ubuntu-latest - steps: - {{ macros.github_checkout_arrow()|indent }} - {{ macros.github_install_archery()|indent }} - - - name: Free Up Disk Space - shell: bash - run: arrow/ci/scripts/util_cleanup.sh - - - name: Execute Docker Build - shell: bash - {% if env is defined %} - env: - {% for key, value in env.items() %} - {{ key }}: {{ value }} - {% endfor %} - {% endif %} - run: archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION="{{ arrow.no_rc_version }}" {{ run }} diff --git a/dev/tasks/gandiva-jars/README.md b/dev/tasks/gandiva-jars/README.md deleted file mode 100644 index 2f4c694d799b..000000000000 --- a/dev/tasks/gandiva-jars/README.md +++ /dev/null @@ -1,29 +0,0 @@ - - -# Updating manylinux for Gandiva Jar Build. - -Do the following to update arrow manylinux docker image for building Gandiva Jars - -- Install java in the manylinux image. -- To do above, update Dockerfile-x86_64_base under python/manylinux1 to install java. -- Please note only upto java7 is available in CentOS5, so install java7 in the base. -- Export JAVA_HOME environment variable. -- Then update build_boost.sh under python/manylinux1/scripts to build boost statically. - -Please look at https://github.com/praveenbingo/arrow/tree/buildGandivaDocker that already has these changes. \ No newline at end of file diff --git a/dev/tasks/gandiva-jars/build-cpp-linux.sh b/dev/tasks/gandiva-jars/build-cpp-linux.sh deleted file mode 100755 index 42651739f840..000000000000 --- a/dev/tasks/gandiva-jars/build-cpp-linux.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -source /multibuild/manylinux_utils.sh - -# Quit on failure -set -e - -PYTHON_VERSION=3.6 -CPYTHON_PATH="$(cpython_path ${PYTHON_VERSION})" -PYTHON_INTERPRETER="${CPYTHON_PATH}/bin/python" -PIP="${CPYTHON_PATH}/bin/pip" - -ARROW_BUILD_DIR=/tmp/arrow-build -mkdir -p "${ARROW_BUILD_DIR}" -pushd "${ARROW_BUILD_DIR}" - -PATH="${CPYTHON_PATH}/bin:${PATH}" -export ARROW_TEST_DATA="/arrow/testing/data" - -cmake -DCMAKE_BUILD_TYPE=Release \ - -DARROW_DEPENDENCY_SOURCE="SYSTEM" \ - -DZLIB_ROOT=/usr/local \ - -DCMAKE_INSTALL_PREFIX=/arrow-dist \ - -DCMAKE_INSTALL_LIBDIR=lib \ - -DARROW_BUILD_TESTS=ON \ - -DARROW_BUILD_SHARED=ON \ - -DARROW_BOOST_USE_SHARED=OFF \ - -DARROW_PROTOBUF_USE_SHARED=OFF \ - -DARROW_OPENSSL_USE_SHARED=OFF \ - -DARROW_GANDIVA_PC_CXX_FLAGS="-isystem;/opt/rh/devtoolset-2/root/usr/include/c++/4.8.2;-isystem;/opt/rh/devtoolset-2/root/usr/include/c++/4.8.2/x86_64-CentOS-linux/" \ - -DARROW_JEMALLOC=ON \ - -DARROW_RPATH_ORIGIN=ON \ - -DARROW_PYTHON=OFF \ - -DARROW_PARQUET=OFF \ - -DARROW_DATASET=OFF \ - -DARROW_FILESYSTEM=OFF \ - -DPARQUET_BUILD_ENCRYPTION=OFF \ - -DPythonInterp_FIND_VERSION=${PYTHON_VERSION} \ - -DARROW_GANDIVA=ON \ - -DARROW_GANDIVA_JAVA=ON \ - -DARROW_GANDIVA_JAVA7=ON \ - -DBoost_NAMESPACE=arrow_boost \ - -Dgflags_SOURCE=BUNDLED \ - -DRapidJSON_SOURCE=BUNDLED \ - -DRE2_SOURCE=BUNDLED \ - -DARROW_BUILD_UTILITIES=OFF \ - -DBoost_NAMESPACE=arrow_boost \ - -DBOOST_ROOT=/arrow_boost_dist \ - -GNinja /arrow/cpp -ninja install -CTEST_OUTPUT_ON_FAILURE=1 ninja test -popd - - -# copy the library to distribution -cp -L /arrow-dist/lib/libgandiva_jni.so /arrow/dist diff --git a/dev/tasks/gandiva-jars/build-cpp-osx.sh b/dev/tasks/gandiva-jars/build-cpp-osx.sh deleted file mode 100755 index cc6ab246d963..000000000000 --- a/dev/tasks/gandiva-jars/build-cpp-osx.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -set -x - -# Builds arrow + gandiva and tests the same. -pushd cpp - mkdir build - pushd build - CMAKE_FLAGS="-DCMAKE_BUILD_TYPE=Release \ - -DARROW_GANDIVA=ON \ - -DARROW_GANDIVA_JAVA=ON \ - -DARROW_GANDIVA_STATIC_LIBSTDCPP=ON \ - -DARROW_BUILD_TESTS=ON \ - -DARROW_BUILD_UTILITIES=OFF \ - -DPARQUET_BUILD_ENCRYPTION=OFF \ - -DARROW_PARQUET=OFF \ - -DARROW_FILESYSTEM=OFF \ - -DARROW_DATASET=OFF \ - -DARROW_BOOST_USE_SHARED=OFF \ - -DARROW_PROTOBUF_USE_SHARED=OFF \ - -DARROW_GFLAGS_USE_SHARED=OFF \ - -DARROW_OPENSSL_USE_SHARED=OFF" - - cmake $CMAKE_FLAGS .. - make -j4 - ctest - - cp -L release/libgandiva_jni.dylib $GITHUB_WORKSPACE/arrow/dist - popd -popd diff --git a/dev/tasks/gandiva-jars/build-java.sh b/dev/tasks/gandiva-jars/build-java.sh deleted file mode 100755 index 7dec07115a3e..000000000000 --- a/dev/tasks/gandiva-jars/build-java.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -CPP_BUILD_DIR=$GITHUB_WORKSPACE/arrow/dist/ - -pushd java - if [[ $OS_NAME == "linux" ]]; then - SO_DEP=ldd - GANDIVA_LIB="$CPP_BUILD_DIR"libgandiva_jni.so - WHITELIST=(linux-vdso libz librt libdl libpthread libstdc++ libm libgcc_s libc ld-linux-x86-64) - else - SO_DEP="otool -L" - GANDIVA_LIB="$CPP_BUILD_DIR"libgandiva_jni.dylib - WHITELIST=(libgandiva_jni libz libncurses libSystem libc++) - fi - - # print the shared library dependencies - eval "$SO_DEP" "$GANDIVA_LIB" - - if [[ $CHECK_SHARED_DEPENDENCIES ]] ; then - # exit if any shared library not in whitelisted set is found - echo "Checking shared dependencies" - while read -r line - do - found=false - for item in "${WHITELIST[@]}" - do - if [[ "$line" == *"$item"* ]] ; then - found=true - fi - done - if [[ "$found" == false ]] ; then - echo "Unexpected shared dependency found" - exit 1 - fi - done < <(eval "$SO_DEP" "$GANDIVA_LIB" | awk '{print $1}') - fi - - # build the entire project - mvn clean install -q -DskipTests -P arrow-jni -Darrow.cpp.build.dir=$CPP_BUILD_DIR - # test only gandiva - mvn test -q -P arrow-jni -pl gandiva -Dgandiva.cpp.build.dir=$CPP_BUILD_DIR - - # copy the jars to distribution folder - find gandiva/target/ -name "*.jar" -not -name "*tests*" -exec cp {} $CPP_BUILD_DIR \; -popd diff --git a/dev/tasks/gandiva-jars/github.linux.yml b/dev/tasks/gandiva-jars/github.linux.yml deleted file mode 100644 index aabcdbee0efc..000000000000 --- a/dev/tasks/gandiva-jars/github.linux.yml +++ /dev/null @@ -1,47 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -{% import 'macros.jinja' as macros with context %} - -{{ macros.github_header() }} - -jobs: - package: - name: Package Gandiva - runs-on: ubuntu-18.04 - steps: - - name: Checkout Arrow - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - if [ $CROSSBOW_USE_COMMIT_ID = true ]; then git -C arrow checkout {{ arrow.head }}; else git -C arrow checkout FETCH_HEAD; fi - git -C arrow submodule update --init --recursive - - name: Build Gandiva - run: | - python3 -VV - cd arrow - mkdir -p dist - export CC="gcc-4.9" CXX="g++-4.9" - ulimit -c unlimited -S - set -e - docker run -v $PWD:/arrow quay.io/anthonylouisbsb/arrow:gandivadocker /arrow/dev/tasks/gandiva-jars/build-cpp-linux.sh - dev/tasks/gandiva-jars/build-java.sh - env: - OS_NAME: "linux" - CHECK_SHARED_DEPENDENCIES: true - - {{ macros.github_upload_releases("arrow/dist/*.jar")|indent }} diff --git a/dev/tasks/gandiva-jars/github.osx.yml b/dev/tasks/gandiva-jars/github.osx.yml deleted file mode 100644 index 3dd6fe46bb65..000000000000 --- a/dev/tasks/gandiva-jars/github.osx.yml +++ /dev/null @@ -1,46 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -{% import 'macros.jinja' as macros with context %} - -{{ macros.github_header() }} - -jobs: - package: - name: Package Gandiva - runs-on: macos-latest - steps: - - name: Checkout Arrow - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - if [ $CROSSBOW_USE_COMMIT_ID = true ]; then git -C arrow checkout {{ arrow.head }}; else git -C arrow checkout FETCH_HEAD; fi - git -C arrow submodule update --init --recursive - - name: Build Gandiva - run: | - cd arrow - mkdir -p dist - export ARROW_TEST_DATA=$PWD/testing/data - set -e - dev/tasks/gandiva-jars/build-cpp-osx.sh - dev/tasks/gandiva-jars/build-java.sh - env: - OS_NAME: "osx" - CHECK_SHARED_DEPENDENCIES: true - MACOSX_DEPLOYMENT_TARGET: "10.11" - - {{ macros.github_upload_releases("arrow/dist/*.jar")|indent }} diff --git a/dev/tasks/homebrew-formulae/apache-arrow.rb b/dev/tasks/homebrew-formulae/apache-arrow.rb deleted file mode 100644 index 953f1eea1c4b..000000000000 --- a/dev/tasks/homebrew-formulae/apache-arrow.rb +++ /dev/null @@ -1,69 +0,0 @@ -class ApacheArrow < Formula - desc "Columnar in-memory analytics layer designed to accelerate big data" - homepage "https://arrow.apache.org/" - url "https://www.apache.org/dyn/closer.lua?path=arrow/arrow-4.0.0-SNAPSHOT/apache-arrow-4.0.0-SNAPSHOT.tar.gz" - sha256 "9948ddb6d4798b51552d0dca3252dd6e3a7d0f9702714fc6f5a1b59397ce1d28" - license "Apache-2.0" - head "https://github.com/apache/arrow.git" - - depends_on "boost" => :build - depends_on "cmake" => :build - depends_on "llvm" => :build - depends_on "brotli" - depends_on "glog" - depends_on "grpc" - depends_on "lz4" - depends_on "numpy" - depends_on "openssl@1.1" - depends_on "protobuf" - depends_on "python@3.9" - depends_on "rapidjson" - depends_on "snappy" - depends_on "thrift" - depends_on "zstd" - - def install - ENV.cxx11 - # link against system libc++ instead of llvm provided libc++ - ENV.remove "HOMEBREW_LIBRARY_PATHS", Formula["llvm"].opt_lib - args = %W[ - -DARROW_FLIGHT=ON - -DARROW_GANDIVA=ON - -DARROW_JEMALLOC=ON - -DARROW_MIMALLOC=ON - -DARROW_ORC=ON - -DARROW_PARQUET=ON - -DARROW_PLASMA=ON - -DARROW_PROTOBUF_USE_SHARED=ON - -DARROW_PYTHON=ON - -DARROW_WITH_BZ2=ON - -DARROW_WITH_ZLIB=ON - -DARROW_WITH_ZSTD=ON - -DARROW_WITH_LZ4=ON - -DARROW_WITH_SNAPPY=ON - -DARROW_WITH_BROTLI=ON - -DARROW_INSTALL_NAME_RPATH=OFF - -DPython3_EXECUTABLE=#{Formula["python@3.9"].bin/"python3"} - ] - # Re-enable -DARROW_S3=ON and add back aws-sdk-cpp to depends_on in ARROW-6437 - - mkdir "build" - cd "build" do - system "cmake", "../cpp", *std_cmake_args, *args - system "make" - system "make", "install" - end - end - - test do - (testpath/"test.cpp").write <<~EOS - #include "arrow/api.h" - int main(void) { - arrow::int64(); - return 0; - } - EOS - system ENV.cxx, "test.cpp", "-std=c++11", "-I#{include}", "-L#{lib}", "-larrow", "-o", "test" - system "./test" - end -end diff --git a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb b/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb deleted file mode 100644 index 351d7764603e..000000000000 --- a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb +++ /dev/null @@ -1,88 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# https://github.com/autobrew/homebrew-core/blob/master/Formula/apache-arrow.rb -class ApacheArrow < Formula - desc "Columnar in-memory analytics layer designed to accelerate big data" - homepage "https://arrow.apache.org/" - url "https://www.apache.org/dyn/closer.lua?path=arrow/arrow-3.0.0.9000/apache-arrow-3.0.0.9000.tar.gz" - sha256 "9948ddb6d4798b51552d0dca3252dd6e3a7d0f9702714fc6f5a1b59397ce1d28" - head "https://github.com/apache/arrow.git" - - bottle do - cellar :any - sha256 "a55211ba6f464681b7ca1b48defdad9cfbe1cf6fad8ff9ec875dc5a3c8f3c5ed" => :el_capitan_or_later - root_url "https://autobrew.github.io/bottles" - end - - # NOTE: if you add something here, be sure to add to PKG_LIBS in r/tools/autobrew - depends_on "boost" => :build - depends_on "cmake" => :build - depends_on "aws-sdk-cpp" - depends_on "lz4" - depends_on "snappy" - depends_on "thrift" - depends_on "zstd" - - def install - ENV.cxx11 - args = %W[ - -DARROW_BUILD_SHARED=OFF - -DARROW_BUILD_UTILITIES=ON - -DARROW_COMPUTE=ON - -DARROW_CSV=ON - -DARROW_DATASET=ON - -DARROW_FILESYSTEM=ON - -DARROW_HDFS=OFF - -DARROW_JEMALLOC=ON - -DARROW_JSON=ON - -DARROW_MIMALLOC=ON - -DARROW_PARQUET=ON - -DARROW_PYTHON=OFF - -DARROW_S3=ON - -DARROW_USE_GLOG=OFF - -DARROW_VERBOSE_THIRDPARTY_BUILD=ON - -DARROW_WITH_LZ4=ON - -DARROW_WITH_SNAPPY=ON - -DARROW_WITH_ZLIB=ON - -DARROW_WITH_ZSTD=ON - -DCMAKE_UNITY_BUILD=OFF - -DPARQUET_BUILD_EXECUTABLES=ON - -DLZ4_HOME=#{Formula["lz4"].prefix} - -DTHRIFT_HOME=#{Formula["thrift"].prefix} - ] - - mkdir "build" - cd "build" do - system "cmake", "../cpp", *std_cmake_args, *args - system "make" - system "make", "install" - end - end - - test do - (testpath/"test.cpp").write <<~EOS - #include "arrow/api.h" - int main(void) { - arrow::int64(); - return 0; - } - EOS - system ENV.cxx, "test.cpp", "-std=c++11", "-I#{include}", "-L#{lib}", "-larrow", "-o", "test" - system "./test" - end -end diff --git a/dev/tasks/homebrew-formulae/github.macos.yml b/dev/tasks/homebrew-formulae/github.macos.yml deleted file mode 100644 index 232cc38a91f3..000000000000 --- a/dev/tasks/homebrew-formulae/github.macos.yml +++ /dev/null @@ -1,56 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# NOTE: must set "Crossbow" as name to have the badge links working in the -# github comment reports! -name: Crossbow - -on: - push: - branches: - - "*-github-*" - -jobs: - autobrew: - name: "Autobrew" - runs-on: macOS-latest - steps: - - name: Checkout Arrow - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive - - name: Configure homebrew formula for testing - env: - ARROW_FORMULA: ./arrow/dev/tasks/homebrew-formulae/{{ formula }} - run: | - # Pin the current commit in the formula to test so that we're not always pulling from master - sed -i.bak -E -e 's@https://github.com/apache/arrow.git"$@{{ arrow.remote }}.git", revision: "{{ arrow.head }}"@' $ARROW_FORMULA && rm -f $ARROW_FORMULA.bak - # Sometimes crossbow gives a remote URL with .git and sometimes not. Make sure there's only one - sed -i.bak -E -e 's@.git.git@.git@' $ARROW_FORMULA && rm -f $ARROW_FORMULA.bak - brew update - brew --version - brew unlink python@2 || true - brew config - brew doctor || true - cp $ARROW_FORMULA $(brew --repository homebrew/core)/Formula/apache-arrow.rb - - name: Test formula - run: | - brew install -v --HEAD apache-arrow - brew test apache-arrow - brew audit --strict apache-arrow diff --git a/dev/tasks/linux-packages/.gitignore b/dev/tasks/linux-packages/.gitignore deleted file mode 100644 index 0e49a90c1eb9..000000000000 --- a/dev/tasks/linux-packages/.gitignore +++ /dev/null @@ -1,28 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -/*/*.tar.gz -/*/apt/repositories/ -/*/apt/tmp/ -/*/apt/build.sh -/*/apt/env.sh -/*/yum/repositories/ -/*/yum/tmp/ -/*/yum/build.sh -/*/yum/env.sh -/apt/repositories/ -/yum/repositories/ diff --git a/dev/tasks/linux-packages/README.md b/dev/tasks/linux-packages/README.md deleted file mode 100644 index cafcc04ed046..000000000000 --- a/dev/tasks/linux-packages/README.md +++ /dev/null @@ -1,40 +0,0 @@ - - -# Linux packages for Apache Arrow C++ and GLib - -## Requirements - - * Ruby - * Docker - * Tools to build tar.gz for Apache Arrow C++ and GLib - -## How to build .deb packages - -```console -% rake version:update -% rake apt -``` - -## How to build .rpm packages - -```console -% rake version:update -% rake yum -``` diff --git a/dev/tasks/linux-packages/Rakefile b/dev/tasks/linux-packages/Rakefile deleted file mode 100644 index a84a43ae517b..000000000000 --- a/dev/tasks/linux-packages/Rakefile +++ /dev/null @@ -1,234 +0,0 @@ -# -*- ruby -*- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -require "English" - -require_relative "../../release/binary-task" -require_relative "helper" - -packages = [ - "apache-arrow", - "apache-arrow-apt-source", - "apache-arrow-release", -] - - -namespace :apt do - desc "Build deb packages" - task :build do - packages.each do |package| - cd(package) do - ruby("-S", "rake", "apt:build") - end - end - end -end - -namespace :yum do - desc "Build RPM packages" - task :build do - packages.each do |package| - cd(package) do - ruby("-S", "rake", "yum:build") - end - end - end -end - -namespace :version do - desc "Update versions" - task :update do - packages.each do |package| - cd(package) do - ruby("-S", "rake", "version:update") - end - end - end -end - -namespace :docker do - desc "Pull built images" - task :pull do - packages.each do |package| - cd(package) do - ruby("-S", "rake", "docker:pull") - end - end - end - - desc "Push built images" - task :push do - packages.each do |package| - cd(package) do - ruby("-S", "rake", "docker:push") - end - end - end -end - - -class LocalBinaryTask < BinaryTask - include Helper::ApacheArrow - - def initialize(packages) - @packages = packages - super() - end - - def define - define_apt_test_task - define_yum_test_task - end - - private - def latest_commit_time(git_directory) - cd(git_directory) do - return Time.iso8601(`git log -n 1 --format=%aI`.chomp).utc - end - end - - def version - @version ||= detect_version(detect_release_time) - end - - def resolve_docker_image(target) - image = "" - case target - when /-(?:arm64|aarch64)\z/ - target = $PREMATCH - image << "arm64v8/" - end - image << target.gsub(/-/, ":") - end - - def verify(target) - verify_command_line = [ - "docker", - "run", - "--rm", - "--log-driver", "none", - "--volume", "#{File.expand_path(arrow_source_dir)}:/arrow:delegated", - ] - if $stdin.tty? - verify_command_line << "--interactive" - verify_command_line << "--tty" - else - verify_command_line.concat(["--attach", "STDOUT"]) - verify_command_line.concat(["--attach", "STDERR"]) - end - verify_command_line << resolve_docker_image(target) - case target - when /\Adebian-/, /\Aubuntu-/ - verify_command_line << "/arrow/dev/release/verify-apt.sh" - else - verify_command_line << "/arrow/dev/release/verify-yum.sh" - end - verify_command_line << version - verify_command_line << "local" - sh(*verify_command_line) - end - - def apt_test_targets - targets = (ENV["APT_TARGETS"] || "").split(",") - targets = apt_test_targets_default if targets.empty? - targets - end - - def apt_test_targets_default - # Disable arm64 targets by default for now - # because they require some setups on host. - [ - "debian-buster", - # "debian-buster-arm64", - "debian-bullseye", - # "debian-bullseye-arm64", - "ubuntu-xenial", - # "ubuntu-xenial-arm64", - "ubuntu-bionic", - # "ubuntu-bionic-arm64", - "ubuntu-focal", - # "ubuntu-focal-arm64", - "ubuntu-groovy", - # "ubuntu-groovy-arm64", - ] - end - - def define_apt_test_task - namespace :apt do - desc "Test deb packages" - task :test do - repositories_dir = "apt/repositories" - rm_rf(repositories_dir) - @packages.each do |package| - package_repositories = "#{package}/apt/repositories" - next unless File.exist?(package_repositories) - sh("rsync", "-a", "#{package_repositories}/", repositories_dir) - end - Dir.glob("#{repositories_dir}/ubuntu/pool/*") do |code_name_dir| - universe_dir = "#{code_name_dir}/universe" - next unless File.exist?(universe_dir) - mv(universe_dir, "#{code_name_dir}/main") - end - apt_update(repositories_dir) - apt_test_targets.each do |target| - verify(target) - end - end - end - end - - def yum_test_targets - targets = (ENV["YUM_TARGETS"] || "").split(",") - targets = yum_test_targets_default if targets.empty? - targets - end - - def yum_test_targets_default - # Disable aarch64 targets by default for now - # because they require some setups on host. - [ - "centos-7", - "centos-8", - # "centos-8-aarch64", - ] - end - - def define_yum_test_task - namespace :yum do - desc "Test RPM packages" - task :test do - repositories_dir = "yum/repositories" - rm_rf(repositories_dir) - @packages.each do |package| - package_repositories = "#{package}/yum/repositories" - next unless File.exist?(package_repositories) - sh("rsync", "-a", "#{package_repositories}/", repositories_dir) - end - rpm_sign(repositories_dir) - yum_update(repositories_dir) - yum_test_targets.each do |target| - verify(target) - end - end - end - end -end - -local_binary_task = LocalBinaryTask.new(packages) -local_binary_task.define diff --git a/dev/tasks/linux-packages/apache-arrow-apt-source/Rakefile b/dev/tasks/linux-packages/apache-arrow-apt-source/Rakefile deleted file mode 100644 index 210fa951ee40..000000000000 --- a/dev/tasks/linux-packages/apache-arrow-apt-source/Rakefile +++ /dev/null @@ -1,64 +0,0 @@ -# -*- ruby -*- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -require_relative "../helper" -require_relative "../package-task" - -class ApacheArrowAptSourcePackageTask < PackageTask - include Helper::ApacheArrow - - def initialize - release_time = detect_release_time - super("apache-arrow-apt-source", - detect_version(release_time), - release_time, - :rc_build_type => :release) - end - - private - def define_archive_task - file @archive_name do - rm_rf(@archive_base_name) - mkdir(@archive_base_name) - download("https://downloads.apache.org/arrow/KEYS", - "#{@archive_base_name}/KEYS") - sh("tar", "czf", @archive_name, @archive_base_name) - rm_rf(@archive_base_name) - end - - if deb_archive_name != @archive_name - file deb_archive_name => @archive_name do - if @archive_base_name == deb_archive_base_name - cp(@archive_name, deb_archive_name) - else - sh("tar", "xf", @archive_name) - mv(@archive_base_name, deb_archive_base_name) - sh("tar", "czf", deb_archive_name, deb_archive_base_name) - end - end - end - end - - def enable_yum? - false - end -end - -task = ApacheArrowAptSourcePackageTask.new -task.define diff --git a/dev/tasks/linux-packages/apache-arrow-apt-source/apt/debian-bullseye/Dockerfile b/dev/tasks/linux-packages/apache-arrow-apt-source/apt/debian-bullseye/Dockerfile deleted file mode 100644 index 3193daaef081..000000000000 --- a/dev/tasks/linux-packages/apache-arrow-apt-source/apt/debian-bullseye/Dockerfile +++ /dev/null @@ -1,40 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -FROM debian:bullseye - -RUN \ - echo "debconf debconf/frontend select Noninteractive" | \ - debconf-set-selections - -RUN \ - echo 'APT::Install-Recommends "false";' > \ - /etc/apt/apt.conf.d/disable-install-recommends - -ARG DEBUG - -RUN \ - quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ - apt update ${quiet} && \ - apt install -y -V ${quiet} \ - build-essential \ - debhelper \ - devscripts \ - fakeroot \ - gnupg \ - lsb-release && \ - apt clean diff --git a/dev/tasks/linux-packages/apache-arrow-apt-source/apt/debian-buster/Dockerfile b/dev/tasks/linux-packages/apache-arrow-apt-source/apt/debian-buster/Dockerfile deleted file mode 100644 index 0d37f5dee71b..000000000000 --- a/dev/tasks/linux-packages/apache-arrow-apt-source/apt/debian-buster/Dockerfile +++ /dev/null @@ -1,41 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -FROM debian:buster - -RUN \ - echo "debconf debconf/frontend select Noninteractive" | \ - debconf-set-selections - -RUN \ - echo 'APT::Install-Recommends "false";' > \ - /etc/apt/apt.conf.d/disable-install-recommends - -ARG DEBUG - -RUN \ - quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ - apt update ${quiet} && \ - apt install -y -V ${quiet} \ - build-essential \ - debhelper \ - devscripts \ - fakeroot \ - gnupg \ - lsb-release && \ - apt clean && \ - rm -rf /var/lib/apt/lists/* diff --git a/dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-bionic/Dockerfile b/dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-bionic/Dockerfile deleted file mode 100644 index 53e11fb7eb7b..000000000000 --- a/dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-bionic/Dockerfile +++ /dev/null @@ -1,41 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -FROM ubuntu:bionic - -RUN \ - echo "debconf debconf/frontend select Noninteractive" | \ - debconf-set-selections - -RUN \ - echo 'APT::Install-Recommends "false";' > \ - /etc/apt/apt.conf.d/disable-install-recommends - -ARG DEBUG - -RUN \ - quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ - apt update ${quiet} && \ - apt install -y -V ${quiet} \ - build-essential \ - debhelper \ - devscripts \ - fakeroot \ - gnupg \ - lsb-release && \ - apt clean && \ - rm -rf /var/lib/apt/lists/* diff --git a/dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-focal/Dockerfile b/dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-focal/Dockerfile deleted file mode 100644 index dc902d14d3df..000000000000 --- a/dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-focal/Dockerfile +++ /dev/null @@ -1,41 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -FROM ubuntu:focal - -RUN \ - echo "debconf debconf/frontend select Noninteractive" | \ - debconf-set-selections - -RUN \ - echo 'APT::Install-Recommends "false";' > \ - /etc/apt/apt.conf.d/disable-install-recommends - -ARG DEBUG - -RUN \ - quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ - apt update ${quiet} && \ - apt install -y -V ${quiet} \ - build-essential \ - debhelper \ - devscripts \ - fakeroot \ - gnupg \ - lsb-release && \ - apt clean && \ - rm -rf /var/lib/apt/lists/* diff --git a/dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-groovy/Dockerfile b/dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-groovy/Dockerfile deleted file mode 100644 index 7efd5d1df323..000000000000 --- a/dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-groovy/Dockerfile +++ /dev/null @@ -1,41 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -FROM ubuntu:groovy - -RUN \ - echo "debconf debconf/frontend select Noninteractive" | \ - debconf-set-selections - -RUN \ - echo 'APT::Install-Recommends "false";' > \ - /etc/apt/apt.conf.d/disable-install-recommends - -ARG DEBUG - -RUN \ - quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ - apt update ${quiet} && \ - apt install -y -V ${quiet} \ - build-essential \ - debhelper \ - devscripts \ - fakeroot \ - gnupg \ - lsb-release && \ - apt clean && \ - rm -rf /var/lib/apt/lists/* diff --git a/dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-xenial/Dockerfile b/dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-xenial/Dockerfile deleted file mode 100644 index e05843081eef..000000000000 --- a/dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-xenial/Dockerfile +++ /dev/null @@ -1,41 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -FROM ubuntu:xenial - -RUN \ - echo "debconf debconf/frontend select Noninteractive" | \ - debconf-set-selections - -RUN \ - echo 'APT::Install-Recommends "false";' > \ - /etc/apt/apt.conf.d/disable-install-recommends - -ARG DEBUG - -RUN \ - quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ - apt update ${quiet} && \ - apt install -y -V ${quiet} \ - build-essential \ - debhelper \ - devscripts \ - fakeroot \ - gnupg \ - lsb-release && \ - apt clean && \ - rm -rf /var/lib/apt/lists/* diff --git a/dev/tasks/linux-packages/apache-arrow-apt-source/debian/apache-arrow-apt-source.install b/dev/tasks/linux-packages/apache-arrow-apt-source/debian/apache-arrow-apt-source.install deleted file mode 100644 index 7bcb2ecc9f05..000000000000 --- a/dev/tasks/linux-packages/apache-arrow-apt-source/debian/apache-arrow-apt-source.install +++ /dev/null @@ -1,2 +0,0 @@ -etc/apt/sources.list.d/* -usr/share/keyrings/* diff --git a/dev/tasks/linux-packages/apache-arrow-apt-source/debian/changelog b/dev/tasks/linux-packages/apache-arrow-apt-source/debian/changelog deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/dev/tasks/linux-packages/apache-arrow-apt-source/debian/compat b/dev/tasks/linux-packages/apache-arrow-apt-source/debian/compat deleted file mode 100644 index ec635144f600..000000000000 --- a/dev/tasks/linux-packages/apache-arrow-apt-source/debian/compat +++ /dev/null @@ -1 +0,0 @@ -9 diff --git a/dev/tasks/linux-packages/apache-arrow-apt-source/debian/control b/dev/tasks/linux-packages/apache-arrow-apt-source/debian/control deleted file mode 100644 index f54d52f98a2e..000000000000 --- a/dev/tasks/linux-packages/apache-arrow-apt-source/debian/control +++ /dev/null @@ -1,23 +0,0 @@ -Source: apache-arrow-apt-source -Section: misc -Priority: important -Maintainer: Apache Arrow Developers -Build-Depends: - debhelper (>= 9), - gnupg, - lsb-release -Standards-Version: 3.9.7 -Homepage: https://arrow.apache.org/ - -Package: apache-arrow-apt-source -Section: misc -Architecture: all -Replaces: apache-arrow-archive-keyring -Breaks: apache-arrow-archive-keyring -Depends: - ${misc:Depends}, - apt-transport-https, - gnupg -Description: GnuPG archive key of the Apache Arrow archive - The Apache Arrow project digitally signs its Release files. This - package contains the archive key used for that. diff --git a/dev/tasks/linux-packages/apache-arrow-apt-source/debian/copyright b/dev/tasks/linux-packages/apache-arrow-apt-source/debian/copyright deleted file mode 100644 index 274d64ca06b9..000000000000 --- a/dev/tasks/linux-packages/apache-arrow-apt-source/debian/copyright +++ /dev/null @@ -1,26 +0,0 @@ -Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ -Upstream-Name: Apache Arrow -Upstream-Contact: Apache Arrow Developers - -Files: * -Copyright: 2016 The Apache Software Foundation -License: Apache-2.0 - -License: Apache-2.0 - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - . - http://www.apache.org/licenses/LICENSE-2.0 - . - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - . - On Debian systems, the full text of the Apache Software License version 2 can - be found in the file `/usr/share/common-licenses/Apache-2.0'. diff --git a/dev/tasks/linux-packages/apache-arrow-apt-source/debian/rules b/dev/tasks/linux-packages/apache-arrow-apt-source/debian/rules deleted file mode 100755 index bf7a85c8c8bc..000000000000 --- a/dev/tasks/linux-packages/apache-arrow-apt-source/debian/rules +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/make -f -# -*- makefile-gmake -*- -# -# Uncomment this to turn on verbose mode. -#export DH_VERBOSE=1 -# This has to be exported to make some magic below work. -export DH_OPTIONS - -%: - dh $@ - -override_dh_auto_build: - gpg \ - --no-default-keyring \ - --keyring ./apache-arrow-apt-source.gpg \ - --import KEYS - - ( \ - distribution=$$(lsb_release --id --short | tr 'A-Z' 'a-z'); \ - code_name=$$(lsb_release --codename --short); \ - echo "Types: deb deb-src"; \ - echo "URIs: https://apache.jfrog.io/artifactory/arrow/$${distribution}/"; \ - echo "Suites: $${code_name}"; \ - echo "Components: main"; \ - echo "Signed-By: /usr/share/keyrings/apache-arrow-apt-source.gpg"; \ - ) > apache-arrow.sources - -override_dh_install: - install -d debian/tmp/usr/share/keyrings/ - install -m 0644 apache-arrow-apt-source.gpg \ - debian/tmp/usr/share/keyrings/ - - install -d debian/tmp/etc/apt/sources.list.d/ - install -m 0644 apache-arrow.sources \ - debian/tmp/etc/apt/sources.list.d/ - - dh_install diff --git a/dev/tasks/linux-packages/apache-arrow-apt-source/debian/source/format b/dev/tasks/linux-packages/apache-arrow-apt-source/debian/source/format deleted file mode 100644 index 163aaf8d82b6..000000000000 --- a/dev/tasks/linux-packages/apache-arrow-apt-source/debian/source/format +++ /dev/null @@ -1 +0,0 @@ -3.0 (quilt) diff --git a/dev/tasks/linux-packages/apache-arrow-release/Rakefile b/dev/tasks/linux-packages/apache-arrow-release/Rakefile deleted file mode 100644 index 4a341c6f10b6..000000000000 --- a/dev/tasks/linux-packages/apache-arrow-release/Rakefile +++ /dev/null @@ -1,66 +0,0 @@ -# -*- ruby -*- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -require_relative "../helper" -require_relative "../package-task" - -class ApacheArrowReleasePackageTask < PackageTask - include Helper::ApacheArrow - - def initialize - release_time = detect_release_time - super("apache-arrow-release", - detect_version(release_time), - release_time, - :rc_build_type => :release) - end - - private - def repo_path - "#{yum_dir}/Apache-Arrow.repo" - end - - def define_archive_task - file @archive_name => [repo_path] do - rm_rf(@archive_base_name) - mkdir(@archive_base_name) - download("https://downloads.apache.org/arrow/KEYS", - "#{@archive_base_name}/KEYS") - cp(repo_path, @archive_base_name) - sh("tar", "czf", @archive_name, @archive_base_name) - rm_rf(@archive_base_name) - end - - if rpm_archive_name != @archive_name - file rpm_archive_name => @archive_name do - sh("tar", "xf", @archive_name) - rpm_archive_base_name = File.basename(rpm_archive_name, ".tar.gz") - mv(@archive_base_name, rpm_archive_base_name) - sh("tar", "czf", rpm_archive_name, rpm_archive_base_name) - end - end - end - - def enable_apt? - false - end -end - -task = ApacheArrowReleasePackageTask.new -task.define diff --git a/dev/tasks/linux-packages/apache-arrow-release/yum/Apache-Arrow.repo b/dev/tasks/linux-packages/apache-arrow-release/yum/Apache-Arrow.repo deleted file mode 100644 index fd77306e6f53..000000000000 --- a/dev/tasks/linux-packages/apache-arrow-release/yum/Apache-Arrow.repo +++ /dev/null @@ -1,37 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[apache-arrow-amazon-linux] -name=Apache Arrow for Amazon Linux 2 - $basearch -baseurl=https://apache.jfrog.io/artifactory/arrow/centos/7/$basearch/ -gpgcheck=1 -enabled=1 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Apache-Arrow - -[apache-arrow-centos] -name=Apache Arrow for CentOS $releasever - $basearch -baseurl=https://apache.jfrog.io/artifactory/arrow/centos/$releasever/$basearch/ -gpgcheck=1 -enabled=1 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Apache-Arrow - -[apache-arrow-rhel] -name=Apache Arrow for RHEL $releasever - $basearch -baseurl=https://apache.jfrog.io/artifactory/arrow/centos/$releasever/$basearch/ -gpgcheck=1 -enabled=1 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Apache-Arrow diff --git a/dev/tasks/linux-packages/apache-arrow-release/yum/apache-arrow-release.spec.in b/dev/tasks/linux-packages/apache-arrow-release/yum/apache-arrow-release.spec.in deleted file mode 100644 index 9f546569e868..000000000000 --- a/dev/tasks/linux-packages/apache-arrow-release/yum/apache-arrow-release.spec.in +++ /dev/null @@ -1,110 +0,0 @@ -# -*- sh-shell: rpm -*- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -%define _centos_ver %{?centos_ver:%{centos_ver}}%{!?centos_ver:8} - -%define use_dnf (%{_centos_ver} >= 8) -%if %{use_dnf} -%define yum_repository_enable() (dnf config-manager --set-enabled %1) -%define yum_repository_disable() (dnf config-manager --set-disabled %1) -%else -%define yum_repository_enable() (yum-config-manager --enable %1) -%define yum_repository_disable() (yum-config-manager --disable %1) -%endif - -Name: @PACKAGE@ -Version: @VERSION@ -Release: @RELEASE@%{?dist} -Summary: Apache Arrow release files - -License: Apache-2.0 -URL: https://arrow.apache.org/ -Source0: @PACKAGE@-%{version}.tar.gz - -BuildArch: noarch - -Requires: epel-release -%if %{use_dnf} -Requires: dnf-command(config-manager) -%else -Requires: yum-utils -%endif - -%description -Apache Arrow release files. - -%prep -%setup -q - -%build -# We use distribution version explicitly because we can't use symbolic link -# on Bintray. CentOS uses 7 and 8 but RHEL uses 7Server and 8Server -# for $releasever. If we can use symbolic link on Bintray, we can use -# $releasever directly. -distribution_version=$(cut -d: -f5 /etc/system-release-cpe) -sed -i'' -e "s/\\\$releasever/${distribution_version}/g" Apache-Arrow.repo - -%install -rm -rf $RPM_BUILD_ROOT - -%{__install} -Dp -m0644 KEYS \ - $RPM_BUILD_ROOT%{_sysconfdir}/pki/rpm-gpg/RPM-GPG-KEY-Apache-Arrow - -%{__install} -d $RPM_BUILD_ROOT%{_sysconfdir}/yum.repos.d/ -%{__install} -Dp -m0644 Apache-Arrow.repo \ - $RPM_BUILD_ROOT%{_sysconfdir}/yum.repos.d/Apache-Arrow.repo - -%files -%defattr(-, root, root, 0755) -%doc -%dir %{_sysconfdir}/yum.repos.d/ -%dir %{_sysconfdir}/pki/rpm-gpg/ -%{_sysconfdir}/pki/rpm-gpg/RPM-GPG-KEY-Apache-Arrow -%config(noreplace) %{_sysconfdir}/yum.repos.d/Apache-Arrow.repo - -%post -if grep -q 'Amazon Linux release 2' /etc/system-release 2>/dev/null; then - %{yum_repository_enable apache-arrow-amazon-linux} - %{yum_repository_disable apache-arrow-centos} - %{yum_repository_disable apache-arrow-rhel} -elif grep -q 'Red Hat Enterprise Linux' /etc/system-release 2>/dev/null; then - %{yum_repository_disable apache-arrow-amazon-linux} - %{yum_repository_disable apache-arrow-centos} - %{yum_repository_enable apache-arrow-rhel} -else - %{yum_repository_disable apache-arrow-amazon-linux} - %{yum_repository_enable apache-arrow-centos} - %{yum_repository_disable apache-arrow-rhel} -fi - -%changelog -* Mon Jan 18 2021 Krisztián Szűcs - 3.0.0-1 -- New upstream release. - -* Mon Oct 12 2020 Krisztián Szűcs - 2.0.0-1 -- New upstream release. - -* Mon Jul 20 2020 Krisztián Szűcs - 1.0.0-1 -- New upstream release. - -* Thu Apr 16 2020 Krisztián Szűcs - 0.17.0-1 -- New upstream release. - -* Thu Jan 30 2020 Krisztián Szűcs - 0.16.0-1 -- New upstream release. diff --git a/dev/tasks/linux-packages/apache-arrow-release/yum/centos-7/Dockerfile b/dev/tasks/linux-packages/apache-arrow-release/yum/centos-7/Dockerfile deleted file mode 100644 index 0396593d7d52..000000000000 --- a/dev/tasks/linux-packages/apache-arrow-release/yum/centos-7/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -FROM centos:7 - -ARG DEBUG - -RUN \ - quiet=$([ "${DEBUG}" = "yes" ] || echo "--quiet") && \ - yum install -y ${quiet} epel-release && \ - yum install -y ${quiet} \ - rpm-build \ - rpmdevtools && \ - yum clean ${quiet} all diff --git a/dev/tasks/linux-packages/apache-arrow-release/yum/centos-8/Dockerfile b/dev/tasks/linux-packages/apache-arrow-release/yum/centos-8/Dockerfile deleted file mode 100644 index c2131bf84126..000000000000 --- a/dev/tasks/linux-packages/apache-arrow-release/yum/centos-8/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -FROM centos:8 - -ARG DEBUG - -RUN \ - quiet=$([ "${DEBUG}" = "yes" ] || echo "--quiet") && \ - dnf install -y ${quiet} epel-release && \ - dnf install --enablerepo=powertools -y ${quiet} \ - rpm-build \ - rpmdevtools && \ - dnf clean ${quiet} all diff --git a/dev/tasks/linux-packages/apache-arrow/Rakefile b/dev/tasks/linux-packages/apache-arrow/Rakefile deleted file mode 100644 index d4848e417be2..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/Rakefile +++ /dev/null @@ -1,120 +0,0 @@ -# -*- ruby -*- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -require_relative "../helper" -require_relative "../package-task" - -class ApacheArrowPackageTask < PackageTask - include Helper::ApacheArrow - - def initialize - release_time = detect_release_time - super("apache-arrow", - detect_version(release_time), - release_time, - :rc_build_type => :release) - @rpm_package = "arrow" - end - - private - def define_archive_task - file @archive_name do - case @version - when /\A\d+\.\d+\.\d+-rc\d+\z/ - download_rc_archive - when /\A\d+\.\d+\.\d+\z/ - download_released_archive - else - build_archive - end - end - - if deb_archive_name != @archive_name - file deb_archive_name => @archive_name do - cp(@archive_name, deb_archive_name) - end - end - - if rpm_archive_name != @archive_name - file rpm_archive_name => @archive_name do - cp(@archive_name, rpm_archive_name) - end - end - end - - def download_rc_archive - base_url = "https://dist.apache.org/repos/dist/dev/arrow" - archive_name_no_rc = @archive_name.gsub(/-rc\d+(\.tar\.gz)\z/, "\\1") - url = "#{base_url}/#{@package}-#{@version}/#{archive_name_no_rc}" - download(url, @archive_name) - end - - def download_released_archive - mirror_base_url = "https://www.apache.org/dyn/closer.lua/arrow" - mirror_list_url = "#{mirror_base_url}/arrow-#{@version}/#{@archive_name}" - open(mirror_list_url) do |response| - if /href="(.+?\/#{Regexp.escape(@archive_name)})"/ =~ response.read - download($1, ".") - end - end - end - - def build_archive - cd(arrow_source_dir) do - sh("git", "archive", "HEAD", - "--prefix", "#{@archive_base_name}/", - "--output", @full_archive_name) - end - end - - def apt_arm64_cuda_available_target?(target) - # ubuntu-20.10 has navidia-cuda-toolkit but not libcuda1. - # ubuntu-21.04 may support this. - false - end - - def apt_prepare_debian_control_cuda_architecture(control, target) - if apt_arm64_cuda_available_target?(target) - cuda_architecture = "any" - else - cuda_architecture = "i386 amd64" - end - control.gsub(/@CUDA_ARCHITECTURE@/, cuda_architecture) - end - - def apt_prepare_debian_control_grpc(control, target) - case target - when /\Adebian-buster/, /\Aubuntu-(?:bionic|focal)/ - use_system_grpc = "#" - else - use_system_grpc = "" - end - control.gsub(/@USE_SYSTEM_GRPC@/, use_system_grpc) - end - - def apt_prepare_debian_control(control_in, target) - control = control_in.dup - control = apt_prepare_debian_control_cuda_architecture(control, target) - control = apt_prepare_debian_control_grpc(control, target) - control - end -end - -task = ApacheArrowPackageTask.new -task.define diff --git a/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye-arm64/from b/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye-arm64/from deleted file mode 100644 index 34187b2af5a7..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye-arm64/from +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -arm64v8/debian:bullseye diff --git a/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye/Dockerfile deleted file mode 100644 index fa4961bc97e0..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye/Dockerfile +++ /dev/null @@ -1,81 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG FROM=debian:bullseye -FROM ${FROM} - -COPY qemu-* /usr/bin/ - -RUN \ - echo "debconf debconf/frontend select Noninteractive" | \ - debconf-set-selections - -RUN \ - echo 'APT::Install-Recommends "false";' > \ - /etc/apt/apt.conf.d/disable-install-recommends - -RUN sed -i'' -e 's/main$/main contrib non-free/g' /etc/apt/sources.list - -ARG DEBUG -RUN \ - quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ - apt update ${quiet} && \ - apt install -y -V ${quiet} \ - build-essential \ - ccache \ - clang-11 \ - cmake \ - debhelper \ - devscripts \ - git \ - gtk-doc-tools \ - libboost-filesystem-dev \ - libboost-system-dev \ - libbrotli-dev \ - libbz2-dev \ - libc-ares-dev \ - libcurl4-openssl-dev \ - libgirepository1.0-dev \ - libglib2.0-doc \ - libgmock-dev \ - libgoogle-glog-dev \ - libgrpc++-dev \ - libgtest-dev \ - liblz4-dev \ - libre2-dev \ - libsnappy-dev \ - libssl-dev \ - libthrift-dev \ - libutf8proc-dev \ - libzstd-dev \ - llvm-11-dev \ - lsb-release \ - ninja-build \ - pkg-config \ - protobuf-compiler-grpc \ - python3-dev \ - python3-numpy \ - python3-pip \ - rapidjson-dev \ - tzdata \ - zlib1g-dev && \ - if apt list | grep '^nvidia-cuda-toolkit/'; then \ - apt install -y -V ${quiet} nvidia-cuda-toolkit; \ - fi && \ - pip3 install --upgrade meson && \ - ln -s /usr/local/bin/meson /usr/bin/ && \ - apt clean diff --git a/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye/qemu-dummy-static b/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye/qemu-dummy-static deleted file mode 100755 index c42e0962def3..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye/qemu-dummy-static +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Do nothing. This exists only for not requiring qemu-aarch64-static copy. -# Recent Debian (buster or later) and Ubuntu (18.10 or later) on amd64 hosts or -# arm64 host don't require qemu-aarch64-static in Docker image. But old Debian -# and Ubuntu hosts on amd64 require qemu-aarch64-static in Docker image. -# -# We use "COPY qemu* /usr/bin/" in Dockerfile. If we don't put any "qemnu*", -# the "COPY" is failed. It means that we always require "qemu*" even if we -# use recent Debian/Ubuntu or arm64 host. If we have this dummy "qemu*" file, -# the "COPY" isn't failed. It means that we can copy "qemu*" only when we -# need. -# -# See also "script" in dev/tasks/linux-packages/azure.linux.arm64.yml. -# Azure Pipelines uses old Ubuntu (18.04). -# So we need to put "qemu-aarch64-static" into this directory. diff --git a/dev/tasks/linux-packages/apache-arrow/apt/debian-buster-arm64/from b/dev/tasks/linux-packages/apache-arrow/apt/debian-buster-arm64/from deleted file mode 100644 index 8da222b86182..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/apt/debian-buster-arm64/from +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -arm64v8/debian:buster diff --git a/dev/tasks/linux-packages/apache-arrow/apt/debian-buster/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/debian-buster/Dockerfile deleted file mode 100644 index 5dcc1b46b2dd..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/apt/debian-buster/Dockerfile +++ /dev/null @@ -1,85 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG FROM=debian:buster -FROM ${FROM} - -COPY qemu-* /usr/bin/ - -RUN \ - echo "debconf debconf/frontend select Noninteractive" | \ - debconf-set-selections - -RUN \ - echo 'APT::Install-Recommends "false";' > \ - /etc/apt/apt.conf.d/disable-install-recommends - -RUN sed -i'' -e 's/main$/main contrib non-free/g' /etc/apt/sources.list - -RUN \ - echo "deb http://deb.debian.org/debian buster-backports main" > \ - /etc/apt/sources.list.d/backports.list - -ARG DEBUG -RUN \ - quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ - apt update ${quiet} && \ - apt install -y -V ${quiet} \ - build-essential \ - ccache \ - cmake \ - debhelper \ - devscripts \ - git \ - gtk-doc-tools \ - libboost-filesystem-dev \ - libboost-system-dev \ - libbrotli-dev \ - libbz2-dev \ - libc-ares-dev \ - libcurl4-openssl-dev \ - libgirepository1.0-dev \ - libglib2.0-doc \ - libgmock-dev \ - libgoogle-glog-dev \ - libgtest-dev \ - liblz4-dev \ - libre2-dev \ - libsnappy-dev \ - libssl-dev \ - libthrift-dev \ - libutf8proc-dev \ - libzstd-dev \ - lsb-release \ - ninja-build \ - pkg-config \ - python3-dev \ - python3-numpy \ - python3-pip \ - rapidjson-dev \ - tzdata \ - zlib1g-dev && \ - apt install -y -V -t buster-backports ${quiet} \ - clang-8 \ - llvm-8-dev && \ - if apt list | grep '^nvidia-cuda-toolkit/'; then \ - apt install -y -V ${quiet} nvidia-cuda-toolkit; \ - fi && \ - pip3 install --upgrade meson && \ - ln -s /usr/local/bin/meson /usr/bin/ && \ - apt clean && \ - rm -rf /var/lib/apt/lists/* diff --git a/dev/tasks/linux-packages/apache-arrow/apt/debian-buster/qemu-dummy-static b/dev/tasks/linux-packages/apache-arrow/apt/debian-buster/qemu-dummy-static deleted file mode 100755 index c42e0962def3..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/apt/debian-buster/qemu-dummy-static +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Do nothing. This exists only for not requiring qemu-aarch64-static copy. -# Recent Debian (buster or later) and Ubuntu (18.10 or later) on amd64 hosts or -# arm64 host don't require qemu-aarch64-static in Docker image. But old Debian -# and Ubuntu hosts on amd64 require qemu-aarch64-static in Docker image. -# -# We use "COPY qemu* /usr/bin/" in Dockerfile. If we don't put any "qemnu*", -# the "COPY" is failed. It means that we always require "qemu*" even if we -# use recent Debian/Ubuntu or arm64 host. If we have this dummy "qemu*" file, -# the "COPY" isn't failed. It means that we can copy "qemu*" only when we -# need. -# -# See also "script" in dev/tasks/linux-packages/azure.linux.arm64.yml. -# Azure Pipelines uses old Ubuntu (18.04). -# So we need to put "qemu-aarch64-static" into this directory. diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic-arm64/from b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic-arm64/from deleted file mode 100644 index c3ba00cf01c9..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic-arm64/from +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -arm64v8/ubuntu:bionic diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile deleted file mode 100644 index 60be92951944..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile +++ /dev/null @@ -1,88 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG FROM=ubuntu:bionic -FROM ${FROM} - -COPY qemu-* /usr/bin/ - -RUN \ - echo "debconf debconf/frontend select Noninteractive" | \ - debconf-set-selections - -RUN \ - echo 'APT::Install-Recommends "false";' > \ - /etc/apt/apt.conf.d/disable-install-recommends - -ARG DEBUG -RUN \ - quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ - apt update ${quiet} && \ - apt install -y -V ${quiet} \ - build-essential \ - ccache \ - clang-10 \ - cmake \ - devscripts \ - fakeroot \ - git \ - gtk-doc-tools \ - libboost-filesystem-dev \ - libboost-system-dev \ - libbrotli-dev \ - libbz2-dev \ - libcurl4-openssl-dev \ - libgirepository1.0-dev \ - libglib2.0-doc \ - libgoogle-glog-dev \ - libgtest-dev \ - liblz4-dev \ - libre2-dev \ - libsnappy-dev \ - libssl-dev \ - libutf8proc-dev \ - libzstd-dev \ - llvm-10-dev \ - lsb-release \ - ninja-build \ - pkg-config \ - python3-dev \ - python3-numpy \ - python3-pip \ - python3-setuptools \ - python3-wheel \ - rapidjson-dev \ - tzdata \ - zlib1g-dev && \ - (echo "includedir=/usr/include" && \ - echo "libdir=/usr/lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH)" && \ - echo "" && \ - echo "Name: re2" && \ - echo "Description: RE2 is a fast, safe, thread-friendly regular expression engine." && \ - echo "Version: 0.0.0" && \ - echo "Cflags: -std=c++11 -pthread -I\${includedir}" && \ - echo "Libs: -pthread -L\${libdir} -lre2") | \ - tee "/usr/lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH)/pkgconfig/re2.pc" && \ - if apt list | grep '^nvidia-cuda-toolkit/'; then \ - apt install -y -V ${quiet} nvidia-cuda-toolkit; \ - fi && \ - apt install -y -V -t bionic-backports ${quiet} \ - debhelper && \ - pip3 install --upgrade meson && \ - ln -s /usr/local/bin/meson /usr/bin/ && \ - apt clean && \ - rm -rf /var/lib/apt/lists/* diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/qemu-dummy-static b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/qemu-dummy-static deleted file mode 100755 index c42e0962def3..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/qemu-dummy-static +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Do nothing. This exists only for not requiring qemu-aarch64-static copy. -# Recent Debian (buster or later) and Ubuntu (18.10 or later) on amd64 hosts or -# arm64 host don't require qemu-aarch64-static in Docker image. But old Debian -# and Ubuntu hosts on amd64 require qemu-aarch64-static in Docker image. -# -# We use "COPY qemu* /usr/bin/" in Dockerfile. If we don't put any "qemnu*", -# the "COPY" is failed. It means that we always require "qemu*" even if we -# use recent Debian/Ubuntu or arm64 host. If we have this dummy "qemu*" file, -# the "COPY" isn't failed. It means that we can copy "qemu*" only when we -# need. -# -# See also "script" in dev/tasks/linux-packages/azure.linux.arm64.yml. -# Azure Pipelines uses old Ubuntu (18.04). -# So we need to put "qemu-aarch64-static" into this directory. diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal-arm64/from b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal-arm64/from deleted file mode 100644 index 52ab48b66f22..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal-arm64/from +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -arm64v8/ubuntu:focal diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile deleted file mode 100644 index ad83bfa90025..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile +++ /dev/null @@ -1,78 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG FROM=ubuntu:focal -FROM ${FROM} - -COPY qemu-* /usr/bin/ - -RUN \ - echo "debconf debconf/frontend select Noninteractive" | \ - debconf-set-selections - -RUN \ - echo 'APT::Install-Recommends "false";' > \ - /etc/apt/apt.conf.d/disable-install-recommends - -ARG DEBUG -RUN \ - quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ - apt update ${quiet} && \ - apt install -y -V ${quiet} \ - build-essential \ - ccache \ - clang-10 \ - cmake \ - debhelper \ - devscripts \ - git \ - gtk-doc-tools \ - libboost-filesystem-dev \ - libboost-system-dev \ - libbrotli-dev \ - libbz2-dev \ - libcurl4-openssl-dev \ - libgirepository1.0-dev \ - libglib2.0-doc \ - libgmock-dev \ - libgoogle-glog-dev \ - libgtest-dev \ - liblz4-dev \ - libre2-dev \ - libsnappy-dev \ - libssl-dev \ - libthrift-dev \ - libutf8proc-dev \ - libzstd-dev \ - llvm-10-dev \ - lsb-release \ - ninja-build \ - pkg-config \ - python3-dev \ - python3-numpy \ - python3-pip \ - python3-setuptools \ - rapidjson-dev \ - tzdata \ - zlib1g-dev && \ - if apt list | grep '^nvidia-cuda-toolkit/'; then \ - apt install -y -V ${quiet} nvidia-cuda-toolkit; \ - fi && \ - apt clean && \ - python3 -m pip install --no-use-pep517 meson && \ - ln -s /usr/local/bin/meson /usr/bin/ && \ - rm -rf /var/lib/apt/lists/* diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/qemu-dummy-static b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/qemu-dummy-static deleted file mode 100755 index c42e0962def3..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/qemu-dummy-static +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Do nothing. This exists only for not requiring qemu-aarch64-static copy. -# Recent Debian (buster or later) and Ubuntu (18.10 or later) on amd64 hosts or -# arm64 host don't require qemu-aarch64-static in Docker image. But old Debian -# and Ubuntu hosts on amd64 require qemu-aarch64-static in Docker image. -# -# We use "COPY qemu* /usr/bin/" in Dockerfile. If we don't put any "qemnu*", -# the "COPY" is failed. It means that we always require "qemu*" even if we -# use recent Debian/Ubuntu or arm64 host. If we have this dummy "qemu*" file, -# the "COPY" isn't failed. It means that we can copy "qemu*" only when we -# need. -# -# See also "script" in dev/tasks/linux-packages/azure.linux.arm64.yml. -# Azure Pipelines uses old Ubuntu (18.04). -# So we need to put "qemu-aarch64-static" into this directory. diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-groovy-arm64/from b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-groovy-arm64/from deleted file mode 100644 index d1f6aa9a854c..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-groovy-arm64/from +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -arm64v8/ubuntu:groovy diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-groovy/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-groovy/Dockerfile deleted file mode 100644 index d60e6320e365..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-groovy/Dockerfile +++ /dev/null @@ -1,79 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG FROM=ubuntu:groovy -FROM ${FROM} - -COPY qemu-* /usr/bin/ - -RUN \ - echo "debconf debconf/frontend select Noninteractive" | \ - debconf-set-selections - -RUN \ - echo 'APT::Install-Recommends "false";' > \ - /etc/apt/apt.conf.d/disable-install-recommends - -ARG DEBUG -RUN \ - quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ - apt update ${quiet} && \ - apt install -y -V ${quiet} \ - build-essential \ - ccache \ - clang-11 \ - cmake \ - debhelper \ - devscripts \ - git \ - gtk-doc-tools \ - libboost-filesystem-dev \ - libboost-system-dev \ - libbrotli-dev \ - libbz2-dev \ - libcurl4-openssl-dev \ - libgirepository1.0-dev \ - libglib2.0-doc \ - libgmock-dev \ - libgoogle-glog-dev \ - libgrpc++-dev \ - libgtest-dev \ - liblz4-dev \ - libre2-dev \ - libsnappy-dev \ - libssl-dev \ - libthrift-dev \ - libutf8proc-dev \ - libzstd-dev \ - llvm-11-dev \ - lsb-release \ - ninja-build \ - pkg-config \ - protobuf-compiler-grpc \ - python3-dev \ - python3-numpy \ - python3-pip \ - python3-setuptools \ - rapidjson-dev \ - tzdata \ - zlib1g-dev && \ - ! apt list | grep -q '^libcuda1' || \ - apt install -y -V ${quiet} nvidia-cuda-toolkit && \ - apt clean && \ - python3 -m pip install --no-use-pep517 meson && \ - ln -s /usr/local/bin/meson /usr/bin/ && \ - rm -rf /var/lib/apt/lists/* diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-groovy/qemu-dummy-static b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-groovy/qemu-dummy-static deleted file mode 100755 index c42e0962def3..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-groovy/qemu-dummy-static +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Do nothing. This exists only for not requiring qemu-aarch64-static copy. -# Recent Debian (buster or later) and Ubuntu (18.10 or later) on amd64 hosts or -# arm64 host don't require qemu-aarch64-static in Docker image. But old Debian -# and Ubuntu hosts on amd64 require qemu-aarch64-static in Docker image. -# -# We use "COPY qemu* /usr/bin/" in Dockerfile. If we don't put any "qemnu*", -# the "COPY" is failed. It means that we always require "qemu*" even if we -# use recent Debian/Ubuntu or arm64 host. If we have this dummy "qemu*" file, -# the "COPY" isn't failed. It means that we can copy "qemu*" only when we -# need. -# -# See also "script" in dev/tasks/linux-packages/azure.linux.arm64.yml. -# Azure Pipelines uses old Ubuntu (18.04). -# So we need to put "qemu-aarch64-static" into this directory. diff --git a/dev/tasks/linux-packages/apache-arrow/debian/changelog b/dev/tasks/linux-packages/apache-arrow/debian/changelog deleted file mode 100644 index 2adfc442de82..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/changelog +++ /dev/null @@ -1,111 +0,0 @@ -apache-arrow (3.0.0-1) unstable; urgency=low - - * New upstream release. - - -- Krisztián Szűcs Mon, 18 Jan 2021 21:33:18 -0000 - -apache-arrow (2.0.0-1) unstable; urgency=low - - * New upstream release. - - -- Krisztián Szűcs Mon, 12 Oct 2020 23:38:01 -0000 - -apache-arrow (1.0.0-1) unstable; urgency=low - - * New upstream release. - - -- Krisztián Szűcs Mon, 20 Jul 2020 20:41:07 -0000 - -apache-arrow (0.17.0-1) unstable; urgency=low - - * New upstream release. - - -- Krisztián Szűcs Thu, 16 Apr 2020 12:05:43 -0000 - -apache-arrow (0.16.0-1) unstable; urgency=low - - * New upstream release. - - -- Krisztián Szűcs Thu, 30 Jan 2020 20:21:44 -0000 - -apache-arrow (0.15.0-1) unstable; urgency=low - - * New upstream release. - - -- Krisztián Szűcs Mon, 30 Sep 2019 17:19:02 -0000 - -apache-arrow (0.14.0-1) unstable; urgency=low - - * New upstream release. - - -- Sutou Kouhei Fri, 28 Jun 2019 22:22:35 -0000 - -apache-arrow (0.13.0-1) unstable; urgency=low - - * New upstream release. - - -- Kouhei Sutou Thu, 28 Mar 2019 02:24:58 -0000 - -apache-arrow (0.12.0-1) unstable; urgency=low - - * New upstream release. - - -- Krisztián Szűcs Wed, 16 Jan 2019 03:29:25 -0000 - -apache-arrow (0.11.0-1) unstable; urgency=low - - * New upstream release. - - -- Kouhei Sutou Thu, 04 Oct 2018 00:33:42 -0000 - -apache-arrow (0.10.0-1) unstable; urgency=low - - * New upstream release. - - -- Phillip Cloud Thu, 02 Aug 2018 23:58:23 -0000 - -apache-arrow (0.9.0-1) unstable; urgency=low - - * New upstream release. - - -- Kouhei Sutou Fri, 16 Mar 2018 16:56:31 -0000 - -apache-arrow (0.8.0-1) unstable; urgency=low - - * New upstream release. - - * Add libarrow-gpu-glib0, libarrow-gpu-glib-dev and gir1.2-arrow-gpu-1.0. - - -- Uwe L. Korn Sun, 17 Dec 2017 20:24:44 -0000 - -apache-arrow (0.7.1-2) unstable; urgency=low - - * Add libarrow-gpu0 and libarrow-gpu-dev. - - * Add libarrow-python-dev. - - -- Kouhei Sutou Sun, 29 Oct 2017 21:59:13 +0900 - -apache-arrow (0.7.1-1) unstable; urgency=low - - * New upstream release. - - -- Kouhei Sutou Wed, 27 Sep 2017 13:19:05 -0000 - -apache-arrow (0.7.0-1) unstable; urgency=low - - * New upstream release. - - -- Wes McKinney Tue, 12 Sep 2017 22:01:14 -0000 - -apache-arrow (0.6.0-1) unstable; urgency=low - - * New upstream release. - - -- Kouhei Sutou Fri, 11 Aug 2017 21:27:51 -0000 - -apache-arrow (0.6.0.20170802-1) unstable; urgency=low - - * New upstream release. - - -- Kouhei Sutou Wed, 02 Aug 2017 22:28:18 -0000 diff --git a/dev/tasks/linux-packages/apache-arrow/debian/compat b/dev/tasks/linux-packages/apache-arrow/debian/compat deleted file mode 100644 index 48082f72f087..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/compat +++ /dev/null @@ -1 +0,0 @@ -12 diff --git a/dev/tasks/linux-packages/apache-arrow/debian/control.in b/dev/tasks/linux-packages/apache-arrow/debian/control.in deleted file mode 100644 index f50b09e60438..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/control.in +++ /dev/null @@ -1,583 +0,0 @@ -Source: apache-arrow -Section: devel -Priority: optional -Maintainer: Apache Arrow Developers -Build-Depends: - cmake, - debhelper (>= 12), - git, - gobject-introspection, - gtk-doc-tools, - libboost-filesystem-dev, - libboost-system-dev, - libbrotli-dev, - libbz2-dev, - libcurl4-openssl-dev, - libgirepository1.0-dev, - libgoogle-glog-dev, -@USE_SYSTEM_GRPC@ libgrpc++-dev, - libgtest-dev, - liblz4-dev, - libre2-dev, - libsnappy-dev, - libssl-dev, - libutf8proc-dev, - libzstd-dev, - ninja-build, - nvidia-cuda-toolkit [!arm64], - pkg-config, -@USE_SYSTEM_GRPC@ protobuf-compiler-grpc, - python3-dev, - python3-numpy, - tzdata, - zlib1g-dev -Build-Depends-Indep: libglib2.0-doc -Standards-Version: 3.9.8 -Homepage: https://arrow.apache.org/ - -Package: libarrow400 -Section: libs -Architecture: any -Multi-Arch: same -Pre-Depends: ${misc:Pre-Depends} -Depends: - ${misc:Depends}, - ${shlibs:Depends} -Description: Apache Arrow is a data processing library for analysis - . - This package provides C++ library files. - -Package: libarrow-cuda400 -Section: libs -Architecture: @CUDA_ARCHITECTURE@ -Multi-Arch: same -Pre-Depends: ${misc:Pre-Depends} -Depends: - ${misc:Depends}, - ${shlibs:Depends}, - libarrow400 (= ${binary:Version}) -Description: Apache Arrow is a data processing library for analysis - . - This package provides C++ library files for CUDA support. - -Package: libarrow-dataset400 -Section: libs -Architecture: any -Multi-Arch: same -Pre-Depends: ${misc:Pre-Depends} -Depends: - ${misc:Depends}, - ${shlibs:Depends}, - libarrow400 (= ${binary:Version}), - libparquet400 (= ${binary:Version}) -Description: Apache Arrow is a data processing library for analysis - . - This package provides C++ library files for Dataset module. - -Package: libarrow-flight400 -Section: libs -Architecture: any -Multi-Arch: same -Pre-Depends: ${misc:Pre-Depends} -Depends: - ${misc:Depends}, - ${shlibs:Depends}, - libarrow400 (= ${binary:Version}) -Description: Apache Arrow is a data processing library for analysis - . - This package provides C++ library files for Flight RPC system. - -Package: libarrow-python400 -Section: libs -Architecture: any -Multi-Arch: same -Pre-Depends: ${misc:Pre-Depends} -Depends: - ${misc:Depends}, - ${shlibs:Depends}, - libarrow400 (= ${binary:Version}), - python3, - python3-numpy -Description: Apache Arrow is a data processing library for analysis - . - This package provides C++ library files for Python support. - -Package: libarrow-python-flight400 -Section: libs -Architecture: any -Multi-Arch: same -Pre-Depends: ${misc:Pre-Depends} -Depends: - ${misc:Depends}, - ${shlibs:Depends}, - libarrow-flight400 (= ${binary:Version}), - libarrow-python400 (= ${binary:Version}) -Description: Apache Arrow is a data processing library for analysis - . - This package provides C++ library files for Flight and Python support. - -Package: libarrow-dev -Section: libdevel -Architecture: any -Multi-Arch: same -Depends: - ${misc:Depends}, - libarrow400 (= ${binary:Version}), - libbrotli-dev, - libbz2-dev, -@USE_SYSTEM_GRPC@ libgrpc++-dev, - liblz4-dev, - libre2-dev, - libsnappy-dev, - libssl-dev, - libutf8proc-dev, - libzstd-dev, -@USE_SYSTEM_GRPC@ protobuf-compiler-grpc, - zlib1g-dev -Description: Apache Arrow is a data processing library for analysis - . - This package provides C++ header files. - -Package: libarrow-cuda-dev -Section: libdevel -Architecture: @CUDA_ARCHITECTURE@ -Multi-Arch: same -Depends: - ${misc:Depends}, - libarrow-dev (= ${binary:Version}), - libarrow-cuda400 (= ${binary:Version}) -Description: Apache Arrow is a data processing library for analysis - . - This package provides C++ header files for CUDA support. - -Package: libarrow-dataset-dev -Section: libdevel -Architecture: any -Multi-Arch: same -Depends: - ${misc:Depends}, - libarrow-dev (= ${binary:Version}), - libarrow-dataset400 (= ${binary:Version}), - libparquet-dev (= ${binary:Version}) -Description: Apache Arrow is a data processing library for analysis - . - This package provides C++ header files for dataset module. - -Package: libarrow-flight-dev -Section: libdevel -Architecture: any -Multi-Arch: same -Depends: - ${misc:Depends}, - libarrow-dev (= ${binary:Version}), - libarrow-flight400 (= ${binary:Version}) -Description: Apache Arrow is a data processing library for analysis - . - This package provides C++ header files for Flight RPC system. - -Package: libarrow-python-dev -Section: libdevel -Architecture: any -Multi-Arch: same -Depends: - ${misc:Depends}, - libarrow-dev (= ${binary:Version}), - libarrow-python400 (= ${binary:Version}) -Description: Apache Arrow is a data processing library for analysis - . - This package provides C++ header files for Python support. - -Package: libarrow-python-flight-dev -Section: libdevel -Architecture: any -Multi-Arch: same -Depends: - ${misc:Depends}, - libarrow-flight-dev (= ${binary:Version}), - libarrow-python-dev (= ${binary:Version}), - libarrow-python-flight400 (= ${binary:Version}) -Description: Apache Arrow is a data processing library for analysis - . - This package provides C++ header files for Flight and Python support. - -Package: libgandiva400 -Section: libs -Architecture: any -Multi-Arch: same -Pre-Depends: ${misc:Pre-Depends} -Depends: - ${misc:Depends}, - ${shlibs:Depends}, - libarrow400 (= ${binary:Version}) -Description: Gandiva is a toolset for compiling and evaluating expressions - on Arrow Data. - . - This package provides C++ library files. - -Package: libgandiva-dev -Section: libdevel -Architecture: any -Multi-Arch: same -Depends: - ${misc:Depends}, - libarrow-dev (= ${binary:Version}), - libgandiva400 (= ${binary:Version}) -Description: Gandiva is a toolset for compiling and evaluating expressions - on Arrow Data. - . - This package provides C++ header files. - -Package: libplasma400 -Section: libs -Architecture: @CUDA_ARCHITECTURE@ -Multi-Arch: same -Pre-Depends: ${misc:Pre-Depends} -Depends: - ${misc:Depends}, - ${shlibs:Depends}, - libarrow-cuda400 (= ${binary:Version}) -Description: Plasma is an in-memory object store and cache for big data. - . - This package provides C++ library files to connect plasma-store-server. - -Package: plasma-store-server -Section: utils -Architecture: @CUDA_ARCHITECTURE@ -Pre-Depends: ${misc:Pre-Depends} -Depends: - ${misc:Depends}, - ${shlibs:Depends}, - libplasma400 (= ${binary:Version}) -Description: Plasma is an in-memory object store and cache for big data. - . - This package provides plasma-store-server. - -Package: libplasma-dev -Section: libdevel -Architecture: @CUDA_ARCHITECTURE@ -Multi-Arch: same -Depends: - ${misc:Depends}, - libarrow-cuda-dev (= ${binary:Version}), - libplasma400 (= ${binary:Version}) -Description: Plasma is an in-memory object store and cache for big data. - . - This package provides C++ header files. - -Package: libparquet400 -Section: libs -Architecture: any -Multi-Arch: same -Pre-Depends: ${misc:Pre-Depends} -Depends: - ${misc:Depends}, - ${shlibs:Depends} -Description: Apache Parquet is a columnar storage format - . - This package provides C++ library files to process Apache Parquet format. - -Package: libparquet-dev -Section: libdevel -Architecture: any -Multi-Arch: same -Depends: - ${misc:Depends}, - libarrow-dev (= ${binary:Version}), - libparquet400 (= ${binary:Version}) -Description: Apache Parquet is a columnar storage format - . - This package provides C++ header files. - -Package: libarrow-glib400 -Section: libs -Architecture: any -Multi-Arch: same -Pre-Depends: ${misc:Pre-Depends} -Depends: - ${misc:Depends}, - ${shlibs:Depends}, - libarrow400 (= ${binary:Version}) -Description: Apache Arrow is a data processing library for analysis - . - This package provides GLib based library files. - -Package: gir1.2-arrow-1.0 -Section: introspection -Architecture: any -Multi-Arch: same -Depends: - ${gir:Depends}, - ${misc:Depends} -Description: Apache Arrow is a data processing library for analysis - . - This package provides GObject Introspection typelib files. - -Package: libarrow-glib-dev -Section: libdevel -Architecture: any -Multi-Arch: same -Depends: - ${misc:Depends}, - libglib2.0-dev, - libarrow-dev (= ${binary:Version}), - libarrow-glib400 (= ${binary:Version}), - gir1.2-arrow-1.0 (= ${binary:Version}) -Suggests: libarrow-glib-doc -Description: Apache Arrow is a data processing library for analysis - . - This package provides GLib based header files. - -Package: libarrow-glib-doc -Section: doc -Architecture: all -Multi-Arch: foreign -Depends: - ${misc:Depends} -Recommends: libglib2.0-doc -Description: Apache Arrow is a data processing library for analysis - . - This package provides documentations. - -Package: libarrow-cuda-glib400 -Section: libs -Architecture: @CUDA_ARCHITECTURE@ -Multi-Arch: same -Pre-Depends: ${misc:Pre-Depends} -Depends: - ${misc:Depends}, - ${shlibs:Depends}, - libarrow-glib400 (= ${binary:Version}), - libarrow-cuda400 (= ${binary:Version}) -Description: Apache Arrow is a data processing library for analysis - . - This package provides GLib based library files for CUDA support. - -Package: gir1.2-arrow-cuda-1.0 -Section: introspection -Architecture: @CUDA_ARCHITECTURE@ -Multi-Arch: same -Depends: - ${gir:Depends}, - ${misc:Depends} -Description: Apache Arrow is a data processing library for analysis - . - This package provides GObject Introspection typelib files for CUDA support. - -Package: libarrow-cuda-glib-dev -Section: libdevel -Architecture: @CUDA_ARCHITECTURE@ -Multi-Arch: same -Depends: - ${misc:Depends}, - libarrow-cuda-dev (= ${binary:Version}), - libarrow-glib-dev (= ${binary:Version}), - libarrow-cuda-glib400 (= ${binary:Version}), - gir1.2-arrow-cuda-1.0 (= ${binary:Version}) -Description: Apache Arrow is a data processing library for analysis - . - This package provides GLib based header files for CUDA support. - -Package: libarrow-dataset-glib400 -Section: libs -Architecture: any -Multi-Arch: same -Pre-Depends: ${misc:Pre-Depends} -Depends: - ${misc:Depends}, - ${shlibs:Depends}, - libarrow-glib400 (= ${binary:Version}), - libarrow-dataset400 (= ${binary:Version}) -Description: Apache Arrow is a data processing library for analysis - . - This package provides GLib based library files for dataset module. - -Package: gir1.2-arrow-dataset-1.0 -Section: introspection -Architecture: any -Multi-Arch: same -Depends: - ${gir:Depends}, - ${misc:Depends} -Description: Apache Arrow is a data processing library for analysis - . - This package provides GObject Introspection typelib files for dataset module. - -Package: libarrow-dataset-glib-dev -Section: libdevel -Architecture: any -Multi-Arch: same -Depends: - ${misc:Depends}, - libarrow-dataset-dev (= ${binary:Version}), - libarrow-glib-dev (= ${binary:Version}), - libarrow-dataset-glib400 (= ${binary:Version}), - gir1.2-arrow-dataset-1.0 (= ${binary:Version}) -Description: Apache Arrow is a data processing library for analysis - . - This package provides GLib based header files for dataset module. - -Package: libarrow-dataset-glib-doc -Section: doc -Architecture: any -Multi-Arch: foreign -Depends: - ${misc:Depends} -Recommends: libarrow-glib-doc -Description: Apache Arrow is a data processing library for analysis - . - This package provides documentations for dataset module. - -Package: libgandiva-glib400 -Section: libs -Architecture: any -Multi-Arch: same -Pre-Depends: ${misc:Pre-Depends} -Depends: - ${misc:Depends}, - ${shlibs:Depends}, - libarrow-glib400 (= ${binary:Version}), - libgandiva400 (= ${binary:Version}) -Description: Gandiva is a toolset for compiling and evaluating expressions - on Arrow Data. - . - This package provides GLib based library files. - -Package: gir1.2-gandiva-1.0 -Section: introspection -Architecture: any -Multi-Arch: same -Depends: - ${gir:Depends}, - ${misc:Depends} -Description: Gandiva is a toolset for compiling and evaluating expressions - on Arrow Data. - . - This package provides GObject Introspection typelib files. - -Package: libgandiva-glib-dev -Section: libdevel -Architecture: any -Multi-Arch: same -Depends: - ${misc:Depends}, - libgandiva-dev (= ${binary:Version}), - libarrow-glib-dev (= ${binary:Version}), - libgandiva-glib400 (= ${binary:Version}), - gir1.2-gandiva-1.0 (= ${binary:Version}) -Description: Gandiva is a toolset for compiling and evaluating expressions - on Arrow Data. - . - This package provides GLib based header files. - -Package: libgandiva-glib-doc -Section: doc -Architecture: any -Multi-Arch: foreign -Depends: - ${misc:Depends} -Recommends: libglib2.0-doc -Description: Gandiva is a toolset for compiling and evaluating expressions - on Arrow Data. - . - This package provides documentations. - -Package: libplasma-glib400 -Section: libs -Architecture: @CUDA_ARCHITECTURE@ -Multi-Arch: same -Pre-Depends: ${misc:Pre-Depends} -Depends: - ${misc:Depends}, - ${shlibs:Depends}, - libarrow-cuda-glib400 (= ${binary:Version}), - libplasma400 (= ${binary:Version}) -Description: Plasma is an in-memory object store and cache for big data. - . - This package provides GLib based library files to connect plasma-store-server. - -Package: gir1.2-plasma-1.0 -Section: introspection -Architecture: @CUDA_ARCHITECTURE@ -Multi-Arch: same -Depends: - ${gir:Depends}, - ${misc:Depends} -Description: Plasma is an in-memory object store and cache for big data. - . - This package provides GObject Introspection typelib files. - -Package: libplasma-glib-dev -Section: libdevel -Architecture: @CUDA_ARCHITECTURE@ -Multi-Arch: same -Depends: - ${misc:Depends}, - libplasma-dev (= ${binary:Version}), - libarrow-cuda-glib-dev (= ${binary:Version}), - libplasma-glib400 (= ${binary:Version}), - gir1.2-plasma-1.0 (= ${binary:Version}) -Description: Plasma is an in-memory object store and cache for big data. - . - This package provides GLib based header files. - -Package: libplasma-glib-doc -Section: doc -Architecture: @CUDA_ARCHITECTURE@ -Multi-Arch: foreign -Depends: - ${misc:Depends} -Recommends: libglib2.0-doc -Description: Plasma is an in-memory object store and cache for big data. - . - This package provides documentations. - -Package: libparquet-glib400 -Section: libs -Architecture: any -Multi-Arch: same -Pre-Depends: ${misc:Pre-Depends} -Depends: - ${misc:Depends}, - ${shlibs:Depends}, - libarrow-glib400 (= ${binary:Version}), - libparquet400 (= ${binary:Version}) -Description: Apache Parquet is a columnar storage format - . - This package provides GLib based library files. - -Package: gir1.2-parquet-1.0 -Section: introspection -Architecture: any -Multi-Arch: same -Depends: - ${gir:Depends}, - ${misc:Depends} -Description: Apache Parquet is a columnar storage format - . - This package provides GObject Introspection typelib files. - -Package: libparquet-glib-dev -Section: libdevel -Architecture: any -Multi-Arch: same -Depends: - ${misc:Depends}, - libarrow-glib-dev (= ${binary:Version}), - libparquet-dev (= ${binary:Version}), - libparquet-glib400 (= ${binary:Version}), - gir1.2-parquet-1.0 (= ${binary:Version}) -Suggests: libparquet-glib-doc -Description: Apache Parquet is a columnar storage format - . - This package provides GLib based header files. - -Package: libparquet-glib-doc -Section: doc -Architecture: all -Multi-Arch: foreign -Depends: - ${misc:Depends} -Recommends: libglib2.0-doc -Description: Apache Parquet is a columnar storage format - . - This package provides documentations. diff --git a/dev/tasks/linux-packages/apache-arrow/debian/copyright b/dev/tasks/linux-packages/apache-arrow/debian/copyright deleted file mode 100644 index 9db0ea76f721..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/copyright +++ /dev/null @@ -1,193 +0,0 @@ -Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ -Upstream-Name: Apache Arrow -Upstream-Contact: -Source: https://dist.apache.org/repos/dist/release/arrow/ - -Files: * -Copyright: 2016 The Apache Software Foundation -License: Apache-2.0 - -Files: TODO for "This product includes software from the SFrame project" -Copyright: 2015 Dato, Inc. - 2009 Carnegie Mellon University. -License: BSD-3-clause - -Files: TODO for "This product includes software from the Numpy project" -Copyright: 1995, 1996, 1997 Jim Hugunin, hugunin@mit.edu - 2005 Travis E. Oliphant oliphant@ee.byu.edu Brigham Young University -License: BSD-3-clause - -Files: TODO for "This product includes software from the Feather project" -Copyright: TODO -License: Apache-2.0 - -Files: TODO for "This product includes software from the DyND project" -Copyright: TODO -License: BSD-2-clause - -Files: TODO for "This product includes software from the LLVM project" -Copyright: 2003-2007 University of Illinois at Urbana-Champaign. -License: U-OF-I-BSD-LIKE - -Files: TODO for "This product includes software from the google-lint project" -Copyright: 2009 Google Inc. All rights reserved. -License: BSD-3-clause - -Files: TODO for "This product includes software from the mman-win32 project" -Copyright: 2010 kutuzov.viktor.84 -License: MIT - -Files: TODO for "This product includes software from the LevelDB project" -Copyright: 2011 The LevelDB Authors. All rights reserved. -License: BSD-3-clause - -Files: TODO for "This product includes software from the CMake project" -Copyright: 2001-2009 Kitware, Inc. - 2012-2014 Continuum Analytics, Inc. -License: BSD-3-clause - -Files: TODO for "This product includes software from https://github.com/matthew-brett/multibuild" -Copyright: 2013-2016, Matt Terry and Matthew Brett; all rights reserved. -License: BSD-2-clause - -Files: TODO for "This product includes software from the Ibis project" -Copyright: 2015 Cloudera, Inc. -License: Apache-2.0 - -Files: TODO for "This product includes code from Apache Kudu" -Copyright: 2016 The Apache Software Foundation -License: Apache-2.0 - -License: Apache-2.0 - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - . - http://www.apache.org/licenses/LICENSE-2.0 - . - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - . - On Debian systems, the full text of the Apache Software License version 2 can - be found in the file `/usr/share/common-licenses/Apache-2.0'. - -License: BSD-3-clause - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - . - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. Neither the name of the University nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - . - THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - SUCH DAMAGE. - -License: BSD-2-clause - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - . - 1) Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - . - 2) Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - . - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY - WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - -License: U-OF-I-BSD-LIKE - ============================================================================== - LLVM Release License - ============================================================================== - University of Illinois/NCSA - Open Source License - . - Copyright (c) 2003-2013 University of Illinois at Urbana-Champaign. - All rights reserved. - . - Developed by: - . - LLVM Team - . - University of Illinois at Urbana-Champaign - . - http://llvm.org - . - Permission is hereby granted, free of charge, to any person obtaining a copy of - this software and associated documentation files (the "Software"), to deal with - the Software without restriction, including without limitation the rights to - use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies - of the Software, and to permit persons to whom the Software is furnished to do - so, subject to the following conditions: - . - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimers. - . - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimers in the - documentation and/or other materials provided with the distribution. - . - * Neither the names of the LLVM Team, University of Illinois at - Urbana-Champaign, nor the names of its contributors may be used to - endorse or promote products derived from this Software without specific - prior written permission. - . - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS - FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE - SOFTWARE. - -License: MIT - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - . - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - . - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. diff --git a/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-1.0.install b/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-1.0.install deleted file mode 100644 index e0197fcd327c..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-1.0.install +++ /dev/null @@ -1 +0,0 @@ -usr/lib/*/girepository-1.0/Arrow-1.0.typelib diff --git a/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-cuda-1.0.install b/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-cuda-1.0.install deleted file mode 100644 index ef0d9f56f9db..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-cuda-1.0.install +++ /dev/null @@ -1 +0,0 @@ -usr/lib/*/girepository-1.0/ArrowCUDA-1.0.typelib diff --git a/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-dataset-1.0.install b/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-dataset-1.0.install deleted file mode 100644 index 27091dab36e3..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-dataset-1.0.install +++ /dev/null @@ -1 +0,0 @@ -usr/lib/*/girepository-1.0/ArrowDataset-1.0.typelib diff --git a/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-gandiva-1.0.install b/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-gandiva-1.0.install deleted file mode 100644 index 0433b367a24c..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-gandiva-1.0.install +++ /dev/null @@ -1 +0,0 @@ -usr/lib/*/girepository-1.0/Gandiva-1.0.typelib diff --git a/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-parquet-1.0.install b/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-parquet-1.0.install deleted file mode 100644 index 13fde668132e..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-parquet-1.0.install +++ /dev/null @@ -1 +0,0 @@ -usr/lib/*/girepository-1.0/Parquet-1.0.typelib diff --git a/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-plasma-1.0.install b/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-plasma-1.0.install deleted file mode 100644 index 7b7ce21581df..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-plasma-1.0.install +++ /dev/null @@ -1 +0,0 @@ -usr/lib/*/girepository-1.0/Plasma-1.0.typelib diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-dev.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-dev.install deleted file mode 100644 index 77e0b70f672f..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-dev.install +++ /dev/null @@ -1,6 +0,0 @@ -usr/lib/*/cmake/arrow/ArrowCUDAConfig*.cmake -usr/lib/*/cmake/arrow/ArrowCUDATargets*.cmake -usr/lib/*/cmake/arrow/FindArrowCUDA.cmake -usr/lib/*/libarrow_cuda.a -usr/lib/*/libarrow_cuda.so -usr/lib/*/pkgconfig/arrow-cuda.pc diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib-dev.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib-dev.install deleted file mode 100644 index 778ae5fd7416..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib-dev.install +++ /dev/null @@ -1,4 +0,0 @@ -usr/include/arrow-cuda-glib/ -usr/lib/*/libarrow-cuda-glib.so -usr/lib/*/pkgconfig/arrow-cuda-glib.pc -usr/share/gir-1.0/ArrowCUDA-1.0.gir diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib400.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib400.install deleted file mode 100644 index a6d6375268d3..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib400.install +++ /dev/null @@ -1 +0,0 @@ -usr/lib/*/libarrow-cuda-glib.so.* diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda400.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda400.install deleted file mode 100644 index 5ae46468764f..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda400.install +++ /dev/null @@ -1 +0,0 @@ -usr/lib/*/libarrow_cuda.so.* diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-dev.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-dev.install deleted file mode 100644 index 53e727ae0526..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-dev.install +++ /dev/null @@ -1,6 +0,0 @@ -usr/lib/*/cmake/arrow/ArrowDatasetConfig*.cmake -usr/lib/*/cmake/arrow/ArrowDatasetTargets*.cmake -usr/lib/*/cmake/arrow/FindArrowDataset.cmake -usr/lib/*/libarrow_dataset.a -usr/lib/*/libarrow_dataset.so -usr/lib/*/pkgconfig/arrow-dataset.pc diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-dev.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-dev.install deleted file mode 100644 index 4c50bde975ad..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-dev.install +++ /dev/null @@ -1,4 +0,0 @@ -usr/include/arrow-dataset-glib/ -usr/lib/*/libarrow-dataset-glib.so -usr/lib/*/pkgconfig/arrow-dataset-glib.pc -usr/share/gir-1.0/ArrowDataset-1.0.gir diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.doc-base b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.doc-base deleted file mode 100644 index 5ec8156b05d6..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.doc-base +++ /dev/null @@ -1,9 +0,0 @@ -Document: arrow-dataset-glib -Title: Apache Arrow Dataset GLib Reference Manual -Author: The Apache Software Foundation -Abstract: Apache Arrow Dataset GLib provides an API to read and write semantic datasets stored in different locations and formats that uses GLib. -Section: Programming - -Format: HTML -Index: /usr/share/gtk-doc/html/arrow-dataset-glib/index.html -Files: /usr/share/gtk-doc/html/arrow-dataset-glib/*.html diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.install deleted file mode 100644 index 523bc206e1e0..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.install +++ /dev/null @@ -1 +0,0 @@ -usr/share/gtk-doc/html/arrow-dataset-glib diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.links b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.links deleted file mode 100644 index 3d880362b5e9..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.links +++ /dev/null @@ -1,3 +0,0 @@ -usr/share/gtk-doc/html/arrow-dataset-glib usr/share/doc/libarrow-dataset-glib-doc/arrow-dataset-glib -usr/share/doc/libglib2.0-doc/glib usr/share/doc/libarrow-dataset-glib-doc/glib -usr/share/doc/libglib2.0-doc/gobject usr/share/doc/libarrow-dataset-glib-doc/gobject diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib400.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib400.install deleted file mode 100644 index 10085f3a0a0a..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib400.install +++ /dev/null @@ -1 +0,0 @@ -usr/lib/*/libarrow-dataset-glib.so.* diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset400.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset400.install deleted file mode 100644 index 0146341652e1..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset400.install +++ /dev/null @@ -1 +0,0 @@ -usr/lib/*/libarrow_dataset.so.* diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install deleted file mode 100644 index 52fbbb32d816..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install +++ /dev/null @@ -1,21 +0,0 @@ -usr/include/arrow/ -usr/lib/*/cmake/arrow/ArrowConfig*.cmake -usr/lib/*/cmake/arrow/ArrowOptions.cmake -usr/lib/*/cmake/arrow/ArrowTargets*.cmake -usr/lib/*/cmake/arrow/Find*Alt.cmake -usr/lib/*/cmake/arrow/FindArrow.cmake -usr/lib/*/cmake/arrow/FindBrotli.cmake -usr/lib/*/cmake/arrow/FindLz4.cmake -usr/lib/*/cmake/arrow/FindSnappy.cmake -usr/lib/*/cmake/arrow/Findutf8proc.cmake -usr/lib/*/cmake/arrow/Findzstd.cmake -usr/lib/*/cmake/arrow/arrow-config.cmake -usr/lib/*/libarrow.a -usr/lib/*/libarrow.so -usr/lib/*/libarrow_bundled_dependencies.a -usr/lib/*/pkgconfig/arrow-compute.pc -usr/lib/*/pkgconfig/arrow-csv.pc -usr/lib/*/pkgconfig/arrow-filesystem.pc -usr/lib/*/pkgconfig/arrow-json.pc -usr/lib/*/pkgconfig/arrow-orc.pc -usr/lib/*/pkgconfig/arrow.pc diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-dev.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-dev.install deleted file mode 100644 index 20ca33d84362..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-dev.install +++ /dev/null @@ -1,6 +0,0 @@ -usr/lib/*/cmake/arrow/ArrowFlightConfig*.cmake -usr/lib/*/cmake/arrow/ArrowFlightTargets*.cmake -usr/lib/*/cmake/arrow/FindArrowFlight.cmake -usr/lib/*/libarrow_flight.a -usr/lib/*/libarrow_flight.so -usr/lib/*/pkgconfig/arrow-flight.pc diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight400.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight400.install deleted file mode 100644 index abdb96d4ca83..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight400.install +++ /dev/null @@ -1 +0,0 @@ -usr/lib/*/libarrow_flight.so.* diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-dev.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-dev.install deleted file mode 100644 index f6de7eedb6f1..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-dev.install +++ /dev/null @@ -1,6 +0,0 @@ -usr/include/arrow-glib/ -usr/lib/*/libarrow-glib.so -usr/lib/*/pkgconfig/arrow-glib.pc -usr/lib/*/pkgconfig/arrow-orc-glib.pc -usr/share/arrow-glib/example/ -usr/share/gir-1.0/Arrow-1.0.gir diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.doc-base b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.doc-base deleted file mode 100644 index 8ae4ffb6ddaa..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.doc-base +++ /dev/null @@ -1,9 +0,0 @@ -Document: arrow-glib -Title: Apache Arrow GLib Reference Manual -Author: The Apache Software Foundation -Abstract: Apache Arrow GLib is a data processing library for analysis that uses GLib. -Section: Programming - -Format: HTML -Index: /usr/share/gtk-doc/html/arrow-glib/index.html -Files: /usr/share/gtk-doc/html/arrow-glib/*.html diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.install deleted file mode 100644 index 912a29c58508..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.install +++ /dev/null @@ -1,2 +0,0 @@ -usr/share/doc/arrow-glib/ -usr/share/gtk-doc/html/arrow-glib diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.links b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.links deleted file mode 100644 index 556987d0a806..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.links +++ /dev/null @@ -1,3 +0,0 @@ -usr/share/gtk-doc/html/arrow-glib usr/share/doc/libarrow-glib-doc/arrow-glib -usr/share/doc/libglib2.0-doc/glib usr/share/doc/libarrow-glib-doc/glib -usr/share/doc/libglib2.0-doc/gobject usr/share/doc/libarrow-glib-doc/gobject diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib400.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib400.install deleted file mode 100644 index ec369d1536b5..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib400.install +++ /dev/null @@ -1 +0,0 @@ -usr/lib/*/libarrow-glib.so.* diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-dev.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-dev.install deleted file mode 100644 index 807583f9845e..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-dev.install +++ /dev/null @@ -1,6 +0,0 @@ -usr/lib/*/cmake/arrow/ArrowPythonConfig*.cmake -usr/lib/*/cmake/arrow/ArrowPythonTargets*.cmake -usr/lib/*/cmake/arrow/FindArrowPython.cmake -usr/lib/*/libarrow_python.a -usr/lib/*/libarrow_python.so -usr/lib/*/pkgconfig/arrow-python.pc diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight-dev.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight-dev.install deleted file mode 100644 index 6cf96e227e94..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight-dev.install +++ /dev/null @@ -1,6 +0,0 @@ -usr/lib/*/cmake/arrow/ArrowPythonFlightConfig*.cmake -usr/lib/*/cmake/arrow/ArrowPythonFlightTargets*.cmake -usr/lib/*/cmake/arrow/FindArrowPythonFlight.cmake -usr/lib/*/libarrow_python_flight.a -usr/lib/*/libarrow_python_flight.so -usr/lib/*/pkgconfig/arrow-python-flight.pc diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight400.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight400.install deleted file mode 100644 index b7cbfec1f050..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight400.install +++ /dev/null @@ -1 +0,0 @@ -usr/lib/*/libarrow_python_flight.so.* diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python400.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python400.install deleted file mode 100644 index eef3e6648373..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python400.install +++ /dev/null @@ -1 +0,0 @@ -usr/lib/*/libarrow_python.so.* diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow400.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow400.install deleted file mode 100644 index 98ef2139cb16..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow400.install +++ /dev/null @@ -1 +0,0 @@ -usr/lib/*/libarrow.so.* diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-dev.install b/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-dev.install deleted file mode 100644 index 26e7e76fb380..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-dev.install +++ /dev/null @@ -1,7 +0,0 @@ -usr/include/gandiva/ -usr/lib/*/cmake/arrow/GandivaConfig*.cmake -usr/lib/*/cmake/arrow/GandivaTargets*.cmake -usr/lib/*/cmake/arrow/FindGandiva.cmake -usr/lib/*/libgandiva.a -usr/lib/*/libgandiva.so -usr/lib/*/pkgconfig/gandiva.pc diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-dev.install b/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-dev.install deleted file mode 100644 index fe7d8bb793f8..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-dev.install +++ /dev/null @@ -1,4 +0,0 @@ -usr/include/gandiva-glib/ -usr/lib/*/libgandiva-glib.so -usr/lib/*/pkgconfig/gandiva-glib.pc -usr/share/gir-1.0/Gandiva-1.0.gir diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.doc-base b/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.doc-base deleted file mode 100644 index 2bf913062fb8..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.doc-base +++ /dev/null @@ -1,9 +0,0 @@ -Document: gandiva-glib -Title: Gandiva GLib Reference Manual -Author: The Apache Software Foundation -Abstract: Gandiva GLib is a toolset for compiling and evaluating expressions on Arrow Data that uses GLib. -Section: Programming - -Format: HTML -Index: /usr/share/gtk-doc/html/gandiva-glib/index.html -Files: /usr/share/gtk-doc/html/gandiva-glib/*.html diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.install b/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.install deleted file mode 100644 index 358e4e5c768b..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.install +++ /dev/null @@ -1 +0,0 @@ -usr/share/gtk-doc/html/gandiva-glib diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.links b/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.links deleted file mode 100644 index 234794e232ef..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.links +++ /dev/null @@ -1,3 +0,0 @@ -usr/share/gtk-doc/html/gandiva-glib usr/share/doc/libgandiva-glib-doc/gandiva-glib -usr/share/doc/libglib2.0-doc/glib usr/share/doc/libgandiva-glib-doc/glib -usr/share/doc/libglib2.0-doc/gobject usr/share/doc/libgandiva-glib-doc/gobject diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib400.install b/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib400.install deleted file mode 100644 index 6257fd43823c..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib400.install +++ /dev/null @@ -1 +0,0 @@ -usr/lib/*/libgandiva-glib.so.* diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libgandiva400.install b/dev/tasks/linux-packages/apache-arrow/debian/libgandiva400.install deleted file mode 100644 index 1475f49cf4c5..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libgandiva400.install +++ /dev/null @@ -1 +0,0 @@ -usr/lib/*/libgandiva.so.* diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libparquet-dev.install b/dev/tasks/linux-packages/apache-arrow/debian/libparquet-dev.install deleted file mode 100644 index e163115f0351..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libparquet-dev.install +++ /dev/null @@ -1,7 +0,0 @@ -usr/include/parquet/ -usr/lib/*/cmake/arrow/ParquetConfig*.cmake -usr/lib/*/cmake/arrow/ParquetTargets*.cmake -usr/lib/*/cmake/arrow/FindParquet.cmake -usr/lib/*/libparquet.a -usr/lib/*/libparquet.so -usr/lib/*/pkgconfig/parquet.pc diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-dev.install b/dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-dev.install deleted file mode 100644 index 9cce737a7106..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-dev.install +++ /dev/null @@ -1,4 +0,0 @@ -usr/include/parquet-glib/ -usr/lib/*/libparquet-glib.so -usr/lib/*/pkgconfig/parquet-glib.pc -usr/share/gir-1.0/Parquet-1.0.gir diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.doc-base b/dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.doc-base deleted file mode 100644 index cc68e2df6c11..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.doc-base +++ /dev/null @@ -1,9 +0,0 @@ -Document: parquet-glib -Title: Apache Parquet GLib Reference Manual -Author: The Apache Software Foundation -Abstract: Apache Parquet GLib is a columnar storage format processing library that uses GLib. -Section: Programming - -Format: HTML -Index: /usr/share/gtk-doc/html/parquet-glib/index.html -Files: /usr/share/gtk-doc/html/parquet-glib/*.html diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.install b/dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.install deleted file mode 100644 index 5843ea3dab8b..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.install +++ /dev/null @@ -1 +0,0 @@ -usr/share/gtk-doc/html/parquet-glib diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.links b/dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.links deleted file mode 100644 index c31f346b174d..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.links +++ /dev/null @@ -1,3 +0,0 @@ -usr/share/gtk-doc/html/parquet-glib usr/share/doc/libparquet-glib-doc/parquet-glib -usr/share/doc/libglib2.0-doc/glib usr/share/doc/libparquet-glib-doc/glib -usr/share/doc/libglib2.0-doc/gobject usr/share/doc/libparquet-glib-doc/gobject diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib400.install b/dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib400.install deleted file mode 100644 index 1c0e4419966b..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib400.install +++ /dev/null @@ -1 +0,0 @@ -usr/lib/*/libparquet-glib.so.* diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libparquet400.install b/dev/tasks/linux-packages/apache-arrow/debian/libparquet400.install deleted file mode 100644 index 540a91d5ef7a..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libparquet400.install +++ /dev/null @@ -1 +0,0 @@ -usr/lib/*/libparquet.so.* diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libplasma-dev.install b/dev/tasks/linux-packages/apache-arrow/debian/libplasma-dev.install deleted file mode 100644 index c315d4dfc770..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libplasma-dev.install +++ /dev/null @@ -1,7 +0,0 @@ -usr/include/plasma/ -usr/lib/*/cmake/arrow/PlasmaConfig*.cmake -usr/lib/*/cmake/arrow/PlasmaTargets*.cmake -usr/lib/*/cmake/arrow/FindPlasma.cmake -usr/lib/*/libplasma.a -usr/lib/*/libplasma.so -usr/lib/*/pkgconfig/plasma.pc diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-dev.install b/dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-dev.install deleted file mode 100644 index 7800681d20a2..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-dev.install +++ /dev/null @@ -1,4 +0,0 @@ -usr/include/plasma-glib/ -usr/lib/*/libplasma-glib.so -usr/lib/*/pkgconfig/plasma-glib.pc -usr/share/gir-1.0/Plasma-1.0.gir diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.doc-base b/dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.doc-base deleted file mode 100644 index a9d306d8bf58..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.doc-base +++ /dev/null @@ -1,9 +0,0 @@ -Document: plasma-glib -Title: Plasma GLib Reference Manual -Author: The Apache Software Foundation -Abstract: Plasma GLib is an in-memory object store and cache for big data that uses GLib. -Section: Programming - -Format: HTML -Index: /usr/share/gtk-doc/html/plasma-glib/index.html -Files: /usr/share/gtk-doc/html/plasma-glib/*.html diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.install b/dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.install deleted file mode 100644 index ad13b94cd72b..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.install +++ /dev/null @@ -1 +0,0 @@ -usr/share/gtk-doc/html/plasma-glib diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.links b/dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.links deleted file mode 100644 index 193262f9b173..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.links +++ /dev/null @@ -1,3 +0,0 @@ -usr/share/gtk-doc/html/plasma-glib usr/share/doc/libplasma-glib-doc/plasma-glib -usr/share/doc/libglib2.0-doc/glib usr/share/doc/libplasma-glib-doc/glib -usr/share/doc/libglib2.0-doc/gobject usr/share/doc/libplasma-glib-doc/gobject diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib400.install b/dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib400.install deleted file mode 100644 index 339bcca3e727..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib400.install +++ /dev/null @@ -1 +0,0 @@ -usr/lib/*/libplasma-glib.so.* diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libplasma400.install b/dev/tasks/linux-packages/apache-arrow/debian/libplasma400.install deleted file mode 100644 index f8a744b65975..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/libplasma400.install +++ /dev/null @@ -1 +0,0 @@ -usr/lib/*/libplasma.so.* diff --git a/dev/tasks/linux-packages/apache-arrow/debian/patches/series b/dev/tasks/linux-packages/apache-arrow/debian/patches/series deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/dev/tasks/linux-packages/apache-arrow/debian/plasma-store-server.install b/dev/tasks/linux-packages/apache-arrow/debian/plasma-store-server.install deleted file mode 100644 index bd13b0e8175f..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/plasma-store-server.install +++ /dev/null @@ -1 +0,0 @@ -usr/bin/plasma-store-server diff --git a/dev/tasks/linux-packages/apache-arrow/debian/rules b/dev/tasks/linux-packages/apache-arrow/debian/rules deleted file mode 100755 index 7b8dff26a086..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/rules +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/make -f -# -*- makefile-gmake -*- -# -# Uncomment this to turn on verbose mode. -#export DH_VERBOSE=1 -# This has to be exported to make some magic below work. -export DH_OPTIONS - -export DEB_BUILD_MAINT_OPTIONS=reproducible=-timeless - -BUILD_TYPE=release - -%: - dh $@ --with gir - -override_dh_auto_configure: - if dpkg -l nvidia-cuda-toolkit > /dev/null 2>&1; then \ - ARROW_CUDA=ON; \ - ARROW_PLASMA=ON; \ - else \ - ARROW_CUDA=OFF; \ - ARROW_PLASMA=OFF; \ - fi; \ - dh_auto_configure \ - --sourcedirectory=cpp \ - --builddirectory=cpp_build \ - --buildsystem=cmake+ninja \ - -- \ - -DARROW_CUDA=$${ARROW_CUDA} \ - -DARROW_FLIGHT=ON \ - -DARROW_GANDIVA=ON \ - -DARROW_GANDIVA_JAVA=OFF \ - -DARROW_MIMALLOC=ON \ - -DARROW_ORC=ON \ - -DARROW_PARQUET=ON \ - -DARROW_PLASMA=$${ARROW_PLASMA} \ - -DARROW_PYTHON=ON \ - -DARROW_S3=ON \ - -DARROW_USE_CCACHE=OFF \ - -DARROW_WITH_BROTLI=ON \ - -DARROW_WITH_BZ2=ON \ - -DARROW_WITH_LZ4=ON \ - -DARROW_WITH_SNAPPY=ON \ - -DARROW_WITH_ZLIB=ON \ - -DARROW_WITH_ZSTD=ON \ - -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \ - -DCMAKE_UNITY_BUILD=ON \ - -DPARQUET_REQUIRE_ENCRYPTION=ON \ - -DPythonInterp_FIND_VERSION=ON \ - -DPythonInterp_FIND_VERSION_MAJOR=3 - -override_dh_auto_build: - dh_auto_build \ - --sourcedirectory=cpp \ - --builddirectory=cpp_build - dh_auto_configure \ - --sourcedirectory=c_glib \ - --builddirectory=c_glib_build \ - --buildsystem=meson+ninja \ - -- \ - -Darrow_cpp_build_type=$(BUILD_TYPE) \ - -Darrow_cpp_build_dir=../cpp_build \ - -Dgtk_doc=true - env \ - LD_LIBRARY_PATH=$(CURDIR)/cpp_build/$(BUILD_TYPE) \ - dh_auto_build \ - --sourcedirectory=c_glib \ - --builddirectory=c_glib_build \ - --buildsystem=meson+ninja - -override_dh_auto_install: - dh_auto_install \ - --sourcedirectory=c_glib \ - --builddirectory=c_glib_build \ - --buildsystem=meson+ninja - # Remove built files to reduce disk usage - dh_auto_clean \ - --sourcedirectory=c_glib \ - --builddirectory=c_glib_build \ - --buildsystem=meson+ninja - - dh_auto_install \ - --sourcedirectory=cpp \ - --builddirectory=cpp_build - # Remove built files to reduce disk usage - dh_auto_clean \ - --sourcedirectory=cpp \ - --builddirectory=cpp_build - -override_dh_auto_test: - # TODO: We need Boost 1.64 or later to build tests for - # Apache Arrow Flight. - # git clone --depth 1 https://github.com/apache/arrow-testing.git - # git clone --depth 1 https://github.com/apache/parquet-testing.git - # cd cpp_build && \ - # env \ - # ARROW_TEST_DATA=$(CURDIR)/arrow-testing/data \ - # PARQUET_TEST_DATA=$(CURDIR)/parquet-testing/data \ - # ctest --exclude-regex 'arrow-cuda-test|plasma-client_tests' - -# skip file failing with "Unknown DWARF DW_OP_255" (see bug#949296) -override_dh_dwz: - dh_dwz --exclude=libgandiva.so diff --git a/dev/tasks/linux-packages/apache-arrow/debian/source/format b/dev/tasks/linux-packages/apache-arrow/debian/source/format deleted file mode 100644 index 163aaf8d82b6..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/source/format +++ /dev/null @@ -1 +0,0 @@ -3.0 (quilt) diff --git a/dev/tasks/linux-packages/apache-arrow/debian/watch b/dev/tasks/linux-packages/apache-arrow/debian/watch deleted file mode 100644 index 5cb3f00915f8..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/debian/watch +++ /dev/null @@ -1,2 +0,0 @@ -version=3 -https://dist.apache.org/repos/dist/release/arrow/arrow-(.+)/apache-arrow-(.+).tar.gz diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in deleted file mode 100644 index 842b3b0f0149..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in +++ /dev/null @@ -1,802 +0,0 @@ -# -*- sh-shell: rpm -*- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -%define _centos_ver %{?centos_ver:%{centos_ver}}%{!?centos_ver:8} - -%define boost_version %( \ - if [ "%{_centos_ver}" = 7 ]; then \ - echo 169; \ - fi) -%define cmake_version %( \ - if [ "%{_centos_ver}" -lt 8 ]; then \ - echo 3; \ - fi) -%define python_version %( \ - if [ "%{_centos_ver}" = 7 ]; then \ - echo 36; \ - else \ - echo 3; \ - fi) - -%define use_flight (%{_centos_ver} >= 8) -%define use_gandiva (%{_centos_ver} >= 8 && %{_arch} != "aarch64") -%define use_mimalloc (%{_centos_ver} >= 8) -%define use_ninja (%{_centos_ver} >= 8) -# TODO: Enable this. This works on local but is fragile on GitHub Actions and -# Travis CI. -# %define use_s3 (%{_centos_ver} >= 8) -%define use_s3 0 - -%define have_rapidjson (%{_centos_ver} == 7) -%define have_re2 (%{_centos_ver} >= 8) -%define have_utf8proc (%{_centos_ver} == 7) - -Name: @PACKAGE@ -Version: @VERSION@ -Release: @RELEASE@%{?dist} -Summary: Apache Arrow is a data processing library for analysis - -License: Apache-2.0 -URL: https://arrow.apache.org/ -Source0: https://dist.apache.org/repos/dist/release/@PACKAGE@/@PACKAGE@-%{version}/apache-@PACKAGE@-%{version}.tar.gz - -BuildRequires: bison -BuildRequires: boost%{boost_version}-devel -BuildRequires: brotli-devel -BuildRequires: bzip2-devel -BuildRequires: cmake%{cmake_version} -%if %{use_s3} -BuildRequires: curl-devel -%endif -BuildRequires: flex -BuildRequires: gcc-c++ -BuildRequires: gflags-devel -BuildRequires: git -BuildRequires: glog-devel -BuildRequires: libzstd-devel -BuildRequires: lz4-devel -BuildRequires: openssl-devel -BuildRequires: pkgconfig -BuildRequires: python%{python_version}-devel -BuildRequires: python%{python_version}-numpy -%if %{have_rapidjson} -BuildRequires: rapidjson-devel -%endif -%if %{have_re2} -BuildRequires: re2-devel -%endif -BuildRequires: snappy-devel -%if %{have_utf8proc} -BuildRequires: utf8proc-devel -%endif -BuildRequires: zlib-devel - -%if %{use_gandiva} -BuildRequires: llvm-devel -BuildRequires: ncurses-devel -%endif - -BuildRequires: gobject-introspection-devel -BuildRequires: gtk-doc - -%description -Apache Arrow is a data processing library for analysis. - -%prep -%setup -q -n apache-@PACKAGE@-%{version} - -%build -cpp_build_type=release -mkdir cpp/build -cd cpp/build -%cmake3 .. \ -%if %{use_flight} - -DARROW_FLIGHT=ON \ -%endif -%if %{use_gandiva} - -DARROW_GANDIVA=ON \ -%endif -%if %{use_mimalloc} - -DARROW_MIMALLOC=ON \ -%endif - -DARROW_ORC=ON \ - -DARROW_PARQUET=ON \ - -DARROW_PLASMA=ON \ - -DARROW_PYTHON=ON \ -%if %{use_s3} - -DARROW_S3=ON \ -%endif - -DARROW_WITH_BROTLI=ON \ - -DARROW_WITH_BZ2=ON \ - -DARROW_WITH_LZ4=ON \ - -DARROW_WITH_SNAPPY=ON \ - -DARROW_WITH_ZLIB=ON \ - -DARROW_WITH_ZSTD=ON \ - -DCMAKE_BUILD_TYPE=$cpp_build_type \ - -DARROW_USE_CCACHE=OFF \ - -DCMAKE_UNITY_BUILD=ON \ - -DPARQUET_REQUIRE_ENCRYPTION=ON \ - -DPythonInterp_FIND_VERSION=ON \ - -DPythonInterp_FIND_VERSION_MAJOR=3 \ -%if %{use_ninja} - -GNinja -%endif - -%if %{use_ninja} -ninja %{?_smp_mflags} -%else -make %{?_smp_mflags} -%endif -cd - - -cd c_glib -pip3 install meson -meson setup build \ - --default-library=both \ - --libdir=%{_libdir} \ - --prefix=%{_prefix} \ - -Darrow_cpp_build_dir=../cpp/build \ - -Darrow_cpp_build_type=$cpp_build_type \ - -Dgtk_doc=true -LD_LIBRARY_PATH=$PWD/../cpp/build/$cpp_build_type ninja -C build %{?_smp_mflags} -cd - - -%install -cpp_build_type=release - -cd c_glib -DESTDIR=$RPM_BUILD_ROOT ninja -C build install -ninja -C build clean -cd - - -cd cpp/build -%if %{use_ninja} -DESTDIR=$RPM_BUILD_ROOT ninja install -ninja clean -%else -make install DESTDIR=$RPM_BUILD_ROOT -make clean -%endif -cd - - -%package libs -Summary: Runtime libraries for Apache Arrow C++ -License: Apache-2.0 -Requires: boost%{boost_version}-system -Requires: boost%{boost_version}-filesystem -Requires: brotli -Requires: gflags -Requires: glog -Requires: libzstd -Requires: lz4 -%if %{have_re2} -Requires: re2 -%endif -Requires: snappy -%if %{have_utf8proc} -Requires: utf8proc -%endif -Requires: zlib - -%description libs -This package contains the libraries for Apache Arrow C++. - -%files libs -%defattr(-,root,root,-) -%doc README.md LICENSE.txt NOTICE.txt -%{_docdir}/arrow/ -%{_libdir}/libarrow.so.* - -%package devel -Summary: Libraries and header files for Apache Arrow C++ -License: Apache-2.0 -Requires: %{name}-libs = %{version}-%{release} -Requires: brotli-devel -Requires: bzip2-devel -Requires: libzstd-devel -Requires: lz4-devel -Requires: openssl-devel -%if %{have_rapidjson} -Requires: rapidjson-devel -%endif -%if %{have_re2} -Requires: re2-devel -%endif -Requires: snappy-devel -%if %{have_utf8proc} -Requires: utf8proc-devel -%endif -Requires: zlib-devel - -%description devel -Libraries and header files for Apache Arrow C++. - -%files devel -%defattr(-,root,root,-) -%doc README.md LICENSE.txt NOTICE.txt -%{_includedir}/arrow/ -%exclude %{_includedir}/arrow/dataset/ -%if %{use_flight} -%exclude %{_includedir}/arrow/flight/ -%endif -%exclude %{_includedir}/arrow/python/ -%{_libdir}/cmake/arrow/ArrowConfig*.cmake -%{_libdir}/cmake/arrow/ArrowOptions.cmake -%{_libdir}/cmake/arrow/ArrowTargets*.cmake -%{_libdir}/cmake/arrow/FindArrow.cmake -%{_libdir}/cmake/arrow/FindBrotli.cmake -%{_libdir}/cmake/arrow/FindLz4.cmake -%{_libdir}/cmake/arrow/FindSnappy.cmake -%if %{have_re2} -%{_libdir}/cmake/arrow/Findre2Alt.cmake -%endif -%if %{have_utf8proc} -%{_libdir}/cmake/arrow/Findutf8proc.cmake -%endif -%{_libdir}/cmake/arrow/Findzstd.cmake -%{_libdir}/cmake/arrow/arrow-config.cmake -%{_libdir}/libarrow.a -%{_libdir}/libarrow.so -%{_libdir}/libarrow_bundled_dependencies.a -%{_libdir}/pkgconfig/arrow-compute.pc -%{_libdir}/pkgconfig/arrow-csv.pc -%{_libdir}/pkgconfig/arrow-filesystem.pc -%{_libdir}/pkgconfig/arrow-json.pc -%{_libdir}/pkgconfig/arrow-orc.pc -%{_libdir}/pkgconfig/arrow.pc - -%package dataset-libs -Summary: C++ library to read and write semantic datasets stored in different locations and formats -License: Apache-2.0 -Requires: %{name}-libs = %{version}-%{release} - -%description dataset-libs -This package contains the libraries for Apache Arrow dataset. - -%files dataset-libs -%defattr(-,root,root,-) -%doc README.md LICENSE.txt NOTICE.txt -%{_libdir}/libarrow_dataset.so.* - -%package dataset-devel -Summary: Libraries and header files for Apache Arrow dataset. -License: Apache-2.0 -Requires: %{name}-dataset-libs = %{version}-%{release} - -%description dataset-devel -Libraries and header files for Apache Arrow dataset. - -%files dataset-devel -%defattr(-,root,root,-) -%doc README.md LICENSE.txt NOTICE.txt -%{_includedir}/arrow/dataset/ -%{_libdir}/cmake/arrow/ArrowDatasetConfig*.cmake -%{_libdir}/cmake/arrow/ArrowDatasetTargets*.cmake -%{_libdir}/cmake/arrow/FindArrowDataset.cmake -%{_libdir}/libarrow_dataset.a -%{_libdir}/libarrow_dataset.so -%{_libdir}/pkgconfig/arrow-dataset.pc - -%if %{use_flight} -%package flight-libs -Summary: C++ library for fast data transport. -License: Apache-2.0 -Requires: %{name}-libs = %{version}-%{release} -Requires: openssl - -%description flight-libs -This package contains the libraries for Apache Arrow Flight. - -%files flight-libs -%defattr(-,root,root,-) -%doc README.md LICENSE.txt NOTICE.txt -%{_libdir}/libarrow_flight.so.* - -%package flight-devel -Summary: Libraries and header files for Apache Arrow Flight. -License: Apache-2.0 -Requires: %{name}-flight-libs = %{version}-%{release} - -%description flight-devel -Libraries and header files for Apache Arrow Flight. - -%files flight-devel -%defattr(-,root,root,-) -%doc README.md LICENSE.txt NOTICE.txt -%{_includedir}/arrow/flight/ -%{_libdir}/cmake/arrow/ArrowFlightConfig*.cmake -%{_libdir}/cmake/arrow/ArrowFlightTargets*.cmake -%{_libdir}/cmake/arrow/FindArrowFlight.cmake -%{_libdir}/libarrow_flight.a -%{_libdir}/libarrow_flight.so -%{_libdir}/pkgconfig/arrow-flight.pc -%endif - -%if %{use_gandiva} -%package -n gandiva-libs -Summary: C++ library for compiling and evaluating expressions on Apache Arrow data. -License: Apache-2.0 -Requires: %{name}-libs = %{version}-%{release} -Requires: ncurses-libs - -%description -n gandiva-libs -This package contains the libraries for Gandiva. - -%files -n gandiva-libs -%defattr(-,root,root,-) -%doc README.md LICENSE.txt NOTICE.txt -%{_libdir}/libgandiva.so.* - -%package -n gandiva-devel -Summary: Libraries and header files for Gandiva. -License: Apache-2.0 -Requires: gandiva-libs = %{version}-%{release} -Requires: llvm-devel - -%description -n gandiva-devel -Libraries and header files for Gandiva. - -%files -n gandiva-devel -%defattr(-,root,root,-) -%doc README.md LICENSE.txt NOTICE.txt -%{_includedir}/gandiva/ -%{_libdir}/cmake/arrow/GandivaConfig*.cmake -%{_libdir}/cmake/arrow/GandivaTargets*.cmake -%{_libdir}/cmake/arrow/FindGandiva.cmake -%{_libdir}/libgandiva.a -%{_libdir}/libgandiva.so -%{_libdir}/pkgconfig/gandiva.pc -%endif - -%package python-libs -Summary: Python integration library for Apache Arrow -License: Apache-2.0 -Requires: %{name}-libs = %{version}-%{release} -Requires: python%{python_version}-numpy - -%description python-libs -This package contains the Python integration library for Apache Arrow. - -%files python-libs -%defattr(-,root,root,-) -%doc README.md LICENSE.txt NOTICE.txt -%{_libdir}/libarrow_python.so.* - -%package python-devel -Summary: Libraries and header files for Python integration library for Apache Arrow -License: Apache-2.0 -Requires: %{name}-devel = %{version}-%{release} -Requires: %{name}-libs = %{version}-%{release} -Requires: python%{python_version}-devel - -%description python-devel -Libraries and header files for Python integration library for Apache Arrow. - -%files python-devel -%defattr(-,root,root,-) -%doc README.md LICENSE.txt NOTICE.txt -%{_includedir}/arrow/python/ -%exclude %{_includedir}/arrow/python/flight.h -%{_libdir}/cmake/arrow/ArrowPythonConfig*.cmake -%{_libdir}/cmake/arrow/ArrowPythonTargets*.cmake -%{_libdir}/cmake/arrow/FindArrowPython.cmake -%{_libdir}/libarrow_python.a -%{_libdir}/libarrow_python.so -%{_libdir}/pkgconfig/arrow-python.pc - -%if %{use_flight} -%package python-flight-libs -Summary: Python integration library for Apache Arrow Flight -License: Apache-2.0 -Requires: %{name}-flight-libs = %{version}-%{release} -Requires: %{name}-python-libs = %{version}-%{release} - -%description python-flight-libs -This package contains the Python integration library for Apache Arrow Flight. - -%files python-flight-libs -%defattr(-,root,root,-) -%doc README.md LICENSE.txt NOTICE.txt -%{_libdir}/libarrow_python_flight.so.* - -%package python-flight-devel -Summary: Libraries and header files for Python integration library for Apache Arrow Flight. -License: Apache-2.0 -Requires: %{name}-flight-devel = %{version}-%{release} -Requires: %{name}-python-devel = %{version}-%{release} -Requires: %{name}-python-flight-libs = %{version}-%{release} - -%description python-flight-devel -Libraries and header files for Python integration library for -Apache Arrow Flight. - -%files python-flight-devel -%defattr(-,root,root,-) -%doc README.md LICENSE.txt NOTICE.txt -%{_includedir}/arrow/python/flight.h -%{_libdir}/cmake/arrow/ArrowPythonFlightConfig*.cmake -%{_libdir}/cmake/arrow/ArrowPythonFlightTargets*.cmake -%{_libdir}/cmake/arrow/FindArrowPythonFlight.cmake -%{_libdir}/libarrow_python_flight.a -%{_libdir}/libarrow_python_flight.so -%{_libdir}/pkgconfig/arrow-python-flight.pc -%endif - -%package -n plasma-libs -Summary: Runtime libraries for Plasma in-memory object store -License: Apache-2.0 -Requires: %{name}-libs = %{version}-%{release} - -%description -n plasma-libs -This package contains the libraries for Plasma in-memory object store. - -%files -n plasma-libs -%defattr(-,root,root,-) -%doc README.md LICENSE.txt NOTICE.txt -%{_libdir}/libplasma.so.* - -%package -n plasma-store-server -Summary: Server for Plasma in-memory object store -License: Apache-2.0 -Requires: plasma-libs = %{version}-%{release} - -%description -n plasma-store-server -This package contains the server for Plasma in-memory object store. - -%files -n plasma-store-server -%defattr(-,root,root,-) -%doc README.md LICENSE.txt NOTICE.txt -%{_bindir}/plasma-store-server - -%package -n plasma-devel -Summary: Libraries and header files for Plasma in-memory object store -License: Apache-2.0 -Requires: plasma-libs = %{version}-%{release} - -%description -n plasma-devel -Libraries and header files for Plasma in-memory object store. - -%files -n plasma-devel -%defattr(-,root,root,-) -%doc README.md LICENSE.txt NOTICE.txt -%{_includedir}/plasma/ -%{_libdir}/cmake/arrow/PlasmaConfig*.cmake -%{_libdir}/cmake/arrow/PlasmaTargets*.cmake -%{_libdir}/cmake/arrow/FindPlasma.cmake -%{_libdir}/libplasma.a -%{_libdir}/libplasma.so -%{_libdir}/pkgconfig/plasma*.pc - -%package -n parquet-libs -Summary: Runtime libraries for Apache Parquet C++ -License: Apache-2.0 -Requires: boost%{boost_version}-program-options -Requires: %{name}-libs = %{version}-%{release} -Requires: openssl - -%description -n parquet-libs -This package contains the libraries for Apache Parquet C++. - -%files -n parquet-libs -%defattr(-,root,root,-) -%doc README.md LICENSE.txt NOTICE.txt -%{_libdir}/libparquet.so.* - -%package -n parquet-devel -Summary: Libraries and header files for Apache Parquet C++ -License: Apache-2.0 -Requires: parquet-libs = %{version}-%{release} -Requires: zlib-devel - -%description -n parquet-devel -Libraries and header files for Apache Parquet C++. - -%files -n parquet-devel -%defattr(-,root,root,-) -%doc README.md LICENSE.txt NOTICE.txt -%{_includedir}/parquet/ -%{_libdir}/cmake/arrow/ParquetConfig*.cmake -%{_libdir}/cmake/arrow/ParquetTargets*.cmake -%{_libdir}/cmake/arrow/FindParquet.cmake -%{_libdir}/libparquet.a -%{_libdir}/libparquet.so -%{_libdir}/pkgconfig/parquet*.pc - -%package glib-libs -Summary: Runtime libraries for Apache Arrow GLib -License: Apache-2.0 -Requires: %{name}-libs = %{version}-%{release} -Requires: glib2 - -%description glib-libs -This package contains the libraries for Apache Arrow GLib. - -%files glib-libs -%defattr(-,root,root,-) -%doc README.md LICENSE.txt NOTICE.txt -%{_libdir}/libarrow-glib.so.* -%{_datadir}/gir-1.0/Arrow-1.0.gir - -%package glib-devel -Summary: Libraries and header files for Apache Arrow GLib -License: Apache-2.0 -Requires: %{name}-devel = %{version}-%{release} -Requires: glib2-devel -Requires: gobject-introspection-devel - -%description glib-devel -Libraries and header files for Apache Arrow GLib. - -%files glib-devel -%defattr(-,root,root,-) -%doc README.md LICENSE.txt NOTICE.txt -%{_includedir}/arrow-glib/ -%{_libdir}/libarrow-glib.a -%{_libdir}/libarrow-glib.so -%{_libdir}/pkgconfig/arrow-glib.pc -%{_libdir}/pkgconfig/arrow-orc-glib.pc -%{_libdir}/girepository-1.0/Arrow-1.0.typelib -%{_datadir}/arrow-glib/example/ - -%package glib-doc -Summary: Documentation for Apache Arrow GLib -License: Apache-2.0 - -%description glib-doc -Documentation for Apache Arrow GLib. - -%files glib-doc -%defattr(-,root,root,-) -%doc README.md LICENSE.txt NOTICE.txt -%{_docdir}/arrow-glib/ -%{_datadir}/gtk-doc/html/arrow-glib/ - -%package dataset-glib-libs -Summary: Runtime libraries for Apache Arrow dataset GLib -License: Apache-2.0 -Requires: %{name}-dataset-libs = %{version}-%{release} -Requires: %{name}-glib-libs = %{version}-%{release} - -%description dataset-glib-libs -This package contains the libraries for Apache Arrow dataset GLib. - -%files dataset-glib-libs -%defattr(-,root,root,-) -%doc README.md LICENSE.txt NOTICE.txt -%{_libdir}/libarrow-dataset-glib.so.* -%{_datadir}/gir-1.0/ArrowDataset-1.0.gir - -%package dataset-glib-devel -Summary: Libraries and header files for Apache Arrow dataset GLib -License: Apache-2.0 -Requires: %{name}-dataset-devel = %{version}-%{release} -Requires: %{name}-glib-devel = %{version}-%{release} - -%description dataset-glib-devel -Libraries and header files for Apache Arrow dataset GLib. - -%files dataset-glib-devel -%defattr(-,root,root,-) -%doc README.md LICENSE.txt NOTICE.txt -%{_includedir}/arrow-dataset-glib/ -%{_libdir}/libarrow-dataset-glib.a -%{_libdir}/libarrow-dataset-glib.so -%{_libdir}/pkgconfig/arrow-dataset-glib.pc -%{_libdir}/girepository-1.0/ArrowDataset-1.0.typelib - -%package dataset-glib-doc -Summary: Documentation for Apache Arrow dataset GLib -License: Apache-2.0 - -%description dataset-glib-doc -Documentation for Apache Arrow dataset GLib. - -%files dataset-glib-doc -%defattr(-,root,root,-) -%doc README.md LICENSE.txt NOTICE.txt -%{_datadir}/gtk-doc/html/arrow-dataset-glib/ - -%if %{use_gandiva} -%package -n gandiva-glib-libs -Summary: Runtime libraries for Gandiva GLib -License: Apache-2.0 -Requires: gandiva-libs = %{version}-%{release} -Requires: %{name}-glib-libs = %{version}-%{release} - -%description -n gandiva-glib-libs -This package contains the libraries for Gandiva GLib. - -%files -n gandiva-glib-libs -%defattr(-,root,root,-) -%doc README.md LICENSE.txt NOTICE.txt -%{_libdir}/libgandiva-glib.so.* -%{_datadir}/gir-1.0/Gandiva-1.0.gir - -%package -n gandiva-glib-devel -Summary: Libraries and header files for Gandiva GLib -License: Apache-2.0 -Requires: gandiva-devel = %{version}-%{release} -Requires: %{name}-glib-devel = %{version}-%{release} - -%description -n gandiva-glib-devel -Libraries and header files for Gandiva GLib. - -%files -n gandiva-glib-devel -%defattr(-,root,root,-) -%doc README.md LICENSE.txt NOTICE.txt -%{_includedir}/gandiva-glib/ -%{_libdir}/libgandiva-glib.a -%{_libdir}/libgandiva-glib.so -%{_libdir}/pkgconfig/gandiva-glib.pc -%{_libdir}/girepository-1.0/Gandiva-1.0.typelib - -%package -n gandiva-glib-doc -Summary: Documentation for Gandiva GLib -License: Apache-2.0 - -%description -n gandiva-glib-doc -Documentation for Gandiva GLib. - -%files -n gandiva-glib-doc -%defattr(-,root,root,-) -%doc README.md LICENSE.txt NOTICE.txt -%{_datadir}/gtk-doc/html/gandiva-glib/ -%endif - -%package -n plasma-glib-libs -Summary: Runtime libraries for Plasma GLib -License: Apache-2.0 -Requires: plasma-libs = %{version}-%{release} -Requires: %{name}-glib-libs = %{version}-%{release} - -%description -n plasma-glib-libs -This package contains the libraries for Plasma GLib. - -%files -n plasma-glib-libs -%defattr(-,root,root,-) -%doc README.md LICENSE.txt NOTICE.txt -%{_libdir}/libplasma-glib.so.* -%{_datadir}/gir-1.0/Plasma-1.0.gir - -%package -n plasma-glib-devel -Summary: Libraries and header files for Plasma GLib -License: Apache-2.0 -Requires: plasma-devel = %{version}-%{release} -Requires: %{name}-glib-devel = %{version}-%{release} - -%description -n plasma-glib-devel -Libraries and header files for Plasma GLib. - -%files -n plasma-glib-devel -%defattr(-,root,root,-) -%doc README.md LICENSE.txt NOTICE.txt -%{_includedir}/plasma-glib/ -%{_libdir}/libplasma-glib.a -%{_libdir}/libplasma-glib.so -%{_libdir}/pkgconfig/plasma-glib.pc -%{_libdir}/girepository-1.0/Plasma-1.0.typelib - -%package -n plasma-glib-doc -Summary: Documentation for Plasma GLib -License: Apache-2.0 - -%description -n plasma-glib-doc -Documentation for Plasma GLib. - -%files -n plasma-glib-doc -%defattr(-,root,root,-) -%doc README.md LICENSE.txt NOTICE.txt -%{_datadir}/gtk-doc/html/plasma-glib/ - -%package -n parquet-glib-libs -Summary: Runtime libraries for Apache Parquet GLib -License: Apache-2.0 -Requires: parquet-libs = %{version}-%{release} -Requires: %{name}-glib-libs = %{version}-%{release} - -%description -n parquet-glib-libs -This package contains the libraries for Apache Parquet GLib. - -%files -n parquet-glib-libs -%defattr(-,root,root,-) -%doc README.md LICENSE.txt NOTICE.txt -%{_libdir}/libparquet-glib.so.* -%{_datadir}/gir-1.0/Parquet-1.0.gir - -%package -n parquet-glib-devel -Summary: Libraries and header files for Apache Parquet GLib -License: Apache-2.0 -Requires: parquet-devel = %{version}-%{release} -Requires: %{name}-glib-devel = %{version}-%{release} - -%description -n parquet-glib-devel -Libraries and header files for Apache Parquet GLib. - -%files -n parquet-glib-devel -%defattr(-,root,root,-) -%doc README.md LICENSE.txt NOTICE.txt -%{_includedir}/parquet-glib/ -%{_libdir}/libparquet-glib.a -%{_libdir}/libparquet-glib.so -%{_libdir}/pkgconfig/parquet-glib.pc -%{_libdir}/girepository-1.0/Parquet-1.0.typelib - -%package -n parquet-glib-doc -Summary: Documentation for Apache Parquet GLib -License: Apache-2.0 - -%description -n parquet-glib-doc -Documentation for Apache Parquet GLib. - -%files -n parquet-glib-doc -%defattr(-,root,root,-) -%doc README.md LICENSE.txt NOTICE.txt -%{_datadir}/gtk-doc/html/parquet-glib/ - -%changelog -* Mon Jan 18 2021 Krisztián Szűcs - 3.0.0-1 -- New upstream release. - -* Mon Oct 12 2020 Krisztián Szűcs - 2.0.0-1 -- New upstream release. - -* Mon Jul 20 2020 Krisztián Szűcs - 1.0.0-1 -- New upstream release. - -* Thu Apr 16 2020 Krisztián Szűcs - 0.17.0-1 -- New upstream release. - -* Thu Jan 30 2020 Krisztián Szűcs - 0.16.0-1 -- New upstream release. - -* Mon Sep 30 2019 Krisztián Szűcs - 0.15.0-1 -- New upstream release. - -* Fri Jun 28 2019 Sutou Kouhei - 0.14.0-1 -- New upstream release. - -* Thu Mar 28 2019 Kouhei Sutou - 0.13.0-1 -- New upstream release. - -* Wed Jan 16 2019 Krisztián Szűcs - 0.12.0-1 -- New upstream release. - -* Thu Oct 04 2018 Kouhei Sutou - 0.11.0-1 -- New upstream release. - -* Thu Aug 02 2018 Phillip Cloud - 0.10.0-1 -- New upstream release. - -* Fri Mar 16 2018 Kouhei Sutou - 0.9.0-1 -- New upstream release. - -* Sun Dec 17 2017 Uwe Korn - 0.8.0-1 -- New upstream release. - -* Wed Sep 27 2017 Kouhei Sutou - 0.7.1-1 -- New upstream release. - -* Tue Sep 12 2017 Wes McKinney - 0.7.0-1 -- New upstream release. - -* Fri Aug 11 2017 Kouhei Sutou - 0.6.0-1 -- New upstream release. - -* Wed Aug 02 2017 Kouhei Sutou - 0.6.0.20170802-1 -- New upstream release. diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile deleted file mode 100644 index 8c6c9d66d255..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile +++ /dev/null @@ -1,62 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG FROM=centos:7 -FROM ${FROM} - -COPY qemu-* /usr/bin/ - -ARG DEBUG - -RUN \ - quiet=$([ "${DEBUG}" = "yes" ] || echo "--quiet") && \ - yum update -y ${quiet} && \ - yum install -y ${quiet} epel-release && \ - yum groupinstall -y ${quiet} "Development Tools" && \ - yum install -y ${quiet} \ - bison \ - boost169-devel \ - brotli-devel \ - bzip2-devel \ - ccache \ - cmake3 \ - flex \ - gflags-devel \ - git \ - glog-devel \ - gobject-introspection-devel \ - gtk-doc \ - libzstd-devel \ - lz4-devel \ - ninja-build \ - openssl-devel \ - pkg-config \ - python36 \ - python36-devel \ - python36-numpy \ - rapidjson-devel \ - rpm-build \ - rpmdevtools \ - snappy-devel \ - tar \ - utf8proc-devel \ - zlib-devel && \ - yum clean ${quiet} all - -ENV \ - BOOST_INCLUDEDIR=/usr/include/boost169 \ - BOOST_LIBRARYDIR=/usr/lib64/boost169 diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/qemu-dummy-static b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/qemu-dummy-static deleted file mode 100755 index c42e0962def3..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/qemu-dummy-static +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Do nothing. This exists only for not requiring qemu-aarch64-static copy. -# Recent Debian (buster or later) and Ubuntu (18.10 or later) on amd64 hosts or -# arm64 host don't require qemu-aarch64-static in Docker image. But old Debian -# and Ubuntu hosts on amd64 require qemu-aarch64-static in Docker image. -# -# We use "COPY qemu* /usr/bin/" in Dockerfile. If we don't put any "qemnu*", -# the "COPY" is failed. It means that we always require "qemu*" even if we -# use recent Debian/Ubuntu or arm64 host. If we have this dummy "qemu*" file, -# the "COPY" isn't failed. It means that we can copy "qemu*" only when we -# need. -# -# See also "script" in dev/tasks/linux-packages/azure.linux.arm64.yml. -# Azure Pipelines uses old Ubuntu (18.04). -# So we need to put "qemu-aarch64-static" into this directory. diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-8-aarch64/from b/dev/tasks/linux-packages/apache-arrow/yum/centos-8-aarch64/from deleted file mode 100644 index 587ce9d4a32d..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/yum/centos-8-aarch64/from +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -arm64v8/centos:8 diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-8/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/centos-8/Dockerfile deleted file mode 100644 index 66c435c333d4..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/yum/centos-8/Dockerfile +++ /dev/null @@ -1,65 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG FROM=centos:8 -FROM ${FROM} - -COPY qemu-* /usr/bin/ - -ARG DEBUG - -RUN \ - quiet=$([ "${DEBUG}" = "yes" ] || echo "--quiet") && \ - dnf install -y ${quiet} epel-release && \ - dnf install --enablerepo=powertools -y ${quiet} \ - bison \ - boost-devel \ - brotli-devel \ - bzip2-devel \ - ccache \ - clang \ - cmake \ - curl-devel \ - flex \ - gcc-c++ \ - gflags-devel \ - git \ - glog-devel \ - gobject-introspection-devel \ - gtk-doc \ - libzstd-devel \ - llvm-devel \ - llvm-static \ - lz4-devel \ - make \ - ncurses-devel \ - ninja-build \ - openssl-devel \ - pkg-config \ - python3 \ - python3-devel \ - python3-numpy \ - python3-pip \ - re2-devel \ - # rapidjson-devel \ - rpm-build \ - rpmdevtools \ - snappy-devel \ - tar \ - # utf8proc-devel \ - zlib-devel && \ - dnf clean ${quiet} all diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-8/qemu-dummy-static b/dev/tasks/linux-packages/apache-arrow/yum/centos-8/qemu-dummy-static deleted file mode 100755 index c42e0962def3..000000000000 --- a/dev/tasks/linux-packages/apache-arrow/yum/centos-8/qemu-dummy-static +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Do nothing. This exists only for not requiring qemu-aarch64-static copy. -# Recent Debian (buster or later) and Ubuntu (18.10 or later) on amd64 hosts or -# arm64 host don't require qemu-aarch64-static in Docker image. But old Debian -# and Ubuntu hosts on amd64 require qemu-aarch64-static in Docker image. -# -# We use "COPY qemu* /usr/bin/" in Dockerfile. If we don't put any "qemnu*", -# the "COPY" is failed. It means that we always require "qemu*" even if we -# use recent Debian/Ubuntu or arm64 host. If we have this dummy "qemu*" file, -# the "COPY" isn't failed. It means that we can copy "qemu*" only when we -# need. -# -# See also "script" in dev/tasks/linux-packages/azure.linux.arm64.yml. -# Azure Pipelines uses old Ubuntu (18.04). -# So we need to put "qemu-aarch64-static" into this directory. diff --git a/dev/tasks/linux-packages/apt/build.sh b/dev/tasks/linux-packages/apt/build.sh deleted file mode 100755 index 73538e7e8d2d..000000000000 --- a/dev/tasks/linux-packages/apt/build.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/bin/bash -# -*- sh-indentation: 2; sh-basic-offset: 2 -*- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -LANG=C - -set -u - -run() -{ - "$@" - if test $? -ne 0; then - echo "Failed $@" - exit 1 - fi -} - -. /host/env.sh - -distribution=$(lsb_release --id --short | tr 'A-Z' 'a-z') -code_name=$(lsb_release --codename --short) -case "${distribution}" in - debian) - component=main - ;; - ubuntu) - component=universe - ;; -esac -architecture=$(dpkg-architecture -q DEB_BUILD_ARCH) - -debuild_options=() -dpkg_buildpackage_options=(-us -uc) - -run mkdir -p /build -run cd /build -find . -not -path ./ccache -a -not -path "./ccache/*" -delete -if which ccache > /dev/null 2>&1; then - export CCACHE_COMPILERCHECK=content - export CCACHE_COMPRESS=1 - export CCACHE_COMPRESSLEVEL=6 - export CCACHE_DIR="${PWD}/ccache" - export CCACHE_MAXSIZE=500M - ccache --show-stats - debuild_options+=(-eCCACHE_COMPILERCHECK) - debuild_options+=(-eCCACHE_COMPRESS) - debuild_options+=(-eCCACHE_COMPRESSLEVEL) - debuild_options+=(-eCCACHE_DIR) - debuild_options+=(-eCCACHE_MAXSIZE) - if [ -d /usr/lib/ccache ] ;then - debuild_options+=(--prepend-path=/usr/lib/ccache) - fi -fi -run cp /host/tmp/${PACKAGE}-${VERSION}.tar.gz \ - ${PACKAGE}_${VERSION}.orig.tar.gz -run tar xfz ${PACKAGE}_${VERSION}.orig.tar.gz -case "${VERSION}" in - *~dev*) - run mv ${PACKAGE}-$(echo $VERSION | sed -e 's/~dev/-dev/') \ - ${PACKAGE}-${VERSION} - ;; - *~rc*) - run mv ${PACKAGE}-$(echo $VERSION | sed -r -e 's/~rc[0-9]+//') \ - ${PACKAGE}-${VERSION} - ;; -esac -run cd ${PACKAGE}-${VERSION}/ -platform="${distribution}-${code_name}" -if [ -d "/host/tmp/debian.${platform}-${architecture}" ]; then - run cp -rp "/host/tmp/debian.${platform}-${architecture}" debian -elif [ -d "/host/tmp/debian.${platform}" ]; then - run cp -rp "/host/tmp/debian.${platform}" debian -else - run cp -rp "/host/tmp/debian" debian -fi -: ${DEB_BUILD_OPTIONS:="parallel=$(nproc)"} -# DEB_BUILD_OPTIONS="${DEB_BUILD_OPTIONS} noopt" -export DEB_BUILD_OPTIONS -if [ "${DEBUG:-no}" = "yes" ]; then - run debuild "${debuild_options[@]}" "${dpkg_buildpackage_options[@]}" -else - run debuild "${debuild_options[@]}" "${dpkg_buildpackage_options[@]}" > /dev/null -fi -if which ccache > /dev/null 2>&1; then - ccache --show-stats -fi -run cd - - -repositories="/host/repositories" -package_initial=$(echo "${PACKAGE}" | sed -e 's/\(.\).*/\1/') -pool_dir="${repositories}/${distribution}/pool/${code_name}/${component}/${package_initial}/${PACKAGE}" -run mkdir -p "${pool_dir}/" -run cp \ - *.*deb \ - *.dsc \ - *.tar.* \ - "${pool_dir}/" - -run chown -R "$(stat --format "%u:%g" "${repositories}")" "${repositories}" diff --git a/dev/tasks/linux-packages/github.linux.amd64.yml b/dev/tasks/linux-packages/github.linux.amd64.yml deleted file mode 100644 index 4fa056c18c70..000000000000 --- a/dev/tasks/linux-packages/github.linux.amd64.yml +++ /dev/null @@ -1,100 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -{% import 'macros.jinja' as macros with context %} - -{{ macros.github_header() }} - -jobs: - package: - name: Package - runs-on: ubuntu-18.04 - steps: - {{ macros.github_checkout_arrow()|indent }} - {{ macros.github_login_dockerhub()|indent }} - - - name: Set up Ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: '2.6' - - name: Free Up Disk Space - shell: bash - run: arrow/ci/scripts/util_cleanup.sh - - name: Cache ccache - uses: actions/cache@v2 - with: - path: arrow/dev/tasks/linux-packages/apache-arrow/{{ task_namespace }}/build/{{ target }}/ccache - key: linux-{{ task_namespace }}-ccache-{{ target }}-{{ "${{ hashFiles('arrow/cpp/**') }}" }} - restore-keys: linux-{{ task_namespace }}-ccache-{{ target }}- - - name: Build - run: | - set -e - pushd arrow/dev/tasks/linux-packages - rake version:update - rake docker:pull || : - rake --trace {{ task_namespace }}:build BUILD_DIR=build - sudo rm -rf */*/build - popd - env: - APT_TARGETS: {{ target }} - ARROW_VERSION: {{ arrow.version }} - REPO: {{ '${{ secrets.REPO }}' }} - YUM_TARGETS: {{ target }} - - name: Docker Push - continue-on-error: true - shell: bash - run: | - pushd arrow/dev/tasks/linux-packages - rake docker:push - popd - env: - APT_TARGETS: {{ target }} - REPO: {{ '${{ secrets.REPO }}' }} - YUM_TARGETS: {{ target }} - - name: Set up test - run: | - set -e - sudo apt update - sudo apt install -y \ - apt-utils \ - createrepo \ - devscripts \ - gpg \ - rpm - (echo "Key-Type: RSA"; \ - echo "Key-Length: 4096"; \ - echo "Name-Real: Test"; \ - echo "Name-Email: test@example.com"; \ - echo "%no-protection") | \ - gpg --full-generate-key --batch - GPG_KEY_ID=$(gpg --list-keys --with-colon test@example.com | grep fpr | cut -d: -f10) - echo "GPG_KEY_ID=${GPG_KEY_ID}" >> ${GITHUB_ENV} - gpg --export --armor test@example.com > arrow/dev/tasks/linux-packages/KEYS - - name: Test - run: | - set -e - pushd arrow/dev/tasks/linux-packages - rake --trace {{ task_namespace }}:test - rm -rf {{ task_namespace }}/repositories - popd - env: - APT_TARGETS: {{ target }} - ARROW_VERSION: {{ arrow.version }} - YUM_TARGETS: {{ target }} - - {% set patterns = upload_extensions | format_all("arrow/dev/tasks/linux-packages/**/*{}") %} - {{ macros.github_upload_releases(patterns)|indent }} diff --git a/dev/tasks/linux-packages/helper.rb b/dev/tasks/linux-packages/helper.rb deleted file mode 100644 index 30ac3b8982fd..000000000000 --- a/dev/tasks/linux-packages/helper.rb +++ /dev/null @@ -1,70 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -module Helper - module ApacheArrow - private - def detect_release_time - release_time_env = ENV["ARROW_RELEASE_TIME"] - if release_time_env - Time.parse(release_time_env).utc - else - latest_commit_time(arrow_source_dir) || Time.now.utc - end - end - - def arrow_source_dir - File.join(__dir__, "..", "..", "..") - end - - def detect_version(release_time) - version_env = ENV["ARROW_VERSION"] - return version_env if version_env - - pom_xml_path = File.join(arrow_source_dir, "java", "pom.xml") - pom_xml_content = File.read(pom_xml_path) - version = pom_xml_content[/^ (.+?)<\/version>/, 1] - formatted_release_time = release_time.strftime("%Y%m%d") - version.gsub(/-SNAPSHOT\z/) {"-dev#{formatted_release_time}"} - end - - def detect_env(name) - value = ENV[name] - return value if value and not value.empty? - - dot_env_path = File.join(arrow_source_dir, ".env") - File.open(dot_env_path) do |dot_env| - dot_env.each_line do |line| - case line.chomp - when /\A#{Regexp.escape(name)}=(.*)/ - return $1 - end - end - end - raise "Failed to detect #{name} environment variable" - end - - def detect_repo - detect_env("REPO") - end - - def docker_image(os, architecture) - architecture ||= "amd64" - "#{detect_repo}:#{architecture}-#{os}-package-#{@package}" - end - end -end diff --git a/dev/tasks/linux-packages/package-task.rb b/dev/tasks/linux-packages/package-task.rb deleted file mode 100644 index 59f345935019..000000000000 --- a/dev/tasks/linux-packages/package-task.rb +++ /dev/null @@ -1,622 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -require "English" -require "open-uri" -require "time" - -class PackageTask - include Rake::DSL - - def initialize(package, version, release_time, options={}) - @package = package - @version = version - @release_time = release_time - - @archive_base_name = "#{@package}-#{@version}" - @archive_name = "#{@archive_base_name}.tar.gz" - @full_archive_name = File.expand_path(@archive_name) - - @rpm_package = @package - case @version - when /-((dev|rc)\d+)\z/ - base_version = $PREMATCH - sub_version = $1 - type = $2 - if type == "rc" and options[:rc_build_type] == :release - @deb_upstream_version = base_version - @deb_archive_base_name_version = base_version - @rpm_version = base_version - @rpm_release = "1" - else - @deb_upstream_version = "#{base_version}~#{sub_version}" - @deb_archive_base_name_version = @version - @rpm_version = base_version - @rpm_release = "0.#{sub_version}" - end - else - @deb_upstream_version = @version - @deb_archive_base_name_version = @version - @rpm_version = @version - @rpm_release = "1" - end - @deb_release = ENV["DEB_RELEASE"] || "1" - end - - def define - define_dist_task - define_apt_task - define_yum_task - define_version_task - define_docker_tasks - end - - private - def env_value(name) - value = ENV[name] - raise "Specify #{name} environment variable" if value.nil? - value - end - - def debug_build? - ENV["DEBUG"] != "no" - end - - def git_directory?(directory) - candidate_paths = [".git", "HEAD"] - candidate_paths.any? do |candidate_path| - File.exist?(File.join(directory, candidate_path)) - end - end - - def latest_commit_time(git_directory) - return nil unless git_directory?(git_directory) - cd(git_directory) do - return Time.iso8601(`git log -n 1 --format=%aI`.chomp).utc - end - end - - def download(url, output_path) - if File.directory?(output_path) - base_name = url.split("/").last - output_path = File.join(output_path, base_name) - end - absolute_output_path = File.expand_path(output_path) - - unless File.exist?(absolute_output_path) - mkdir_p(File.dirname(absolute_output_path)) - rake_output_message "Downloading... #{url}" - URI(url).open do |downloaded_file| - File.open(absolute_output_path, "wb") do |output_file| - output_file.print(downloaded_file.read) - end - end - end - - absolute_output_path - end - - def substitute_content(content) - content.gsub(/@(.+?)@/) do |matched| - yield($1, matched) - end - end - - def docker_image(os, architecture) - image = "#{@package}-#{os}" - image << "-#{architecture}" if architecture - image - end - - def docker_run(os, architecture, console: false) - id = os - id = "#{id}-#{architecture}" if architecture - image = docker_image(os, architecture) - build_command_line = [ - "docker", - "build", - "--cache-from", image, - "--tag", image, - ] - run_command_line = [ - "docker", - "run", - "--rm", - "--log-driver", "none", - "--volume", "#{Dir.pwd}:/host:rw", - ] - if $stdin.tty? - run_command_line << "--interactive" - run_command_line << "--tty" - else - run_command_line.concat(["--attach", "STDOUT"]) - run_command_line.concat(["--attach", "STDERR"]) - end - build_dir = ENV["BUILD_DIR"] - if build_dir - build_dir = "#{File.expand_path(build_dir)}/#{id}" - mkdir_p(build_dir) - run_command_line.concat(["--volume", "#{build_dir}:/build:rw"]) - end - if debug_build? - build_command_line.concat(["--build-arg", "DEBUG=yes"]) - run_command_line.concat(["--env", "DEBUG=yes"]) - end - pass_through_env_names = [ - "DEB_BUILD_OPTIONS", - "RPM_BUILD_NCPUS", - ] - pass_through_env_names.each do |name| - value = ENV[name] - next unless value - run_command_line.concat(["--env", "#{name}=#{value}"]) - end - if File.exist?(File.join(id, "Dockerfile")) - docker_context = id - else - from = File.readlines(File.join(id, "from")).find do |line| - /^[a-z]/i =~ line - end - build_command_line.concat(["--build-arg", "FROM=#{from.chomp}"]) - docker_context = os - end - build_command_line.concat(docker_build_options(os, architecture)) - run_command_line.concat(docker_run_options(os, architecture)) - build_command_line << docker_context - run_command_line << image - run_command_line << "/host/build.sh" unless console - - sh(*build_command_line) - sh(*run_command_line) - end - - def docker_build_options(os, architecture) - [] - end - - def docker_run_options(os, architecture) - [] - end - - def docker_pull(os, architecture) - image = docker_image(os, architecture) - command_line = [ - "docker", - "pull", - image, - ] - command_line.concat(docker_pull_options(os, architecture)) - sh(*command_line) - end - - def docker_pull_options(os, architecture) - [] - end - - def docker_push(os, architecture) - image = docker_image(os, architecture) - command_line = [ - "docker", - "push", - image, - ] - command_line.concat(docker_push_options(os, architecture)) - sh(*command_line) - end - - def docker_push_options(os, architecture) - [] - end - - def define_dist_task - define_archive_task - desc "Create release package" - task :dist => [@archive_name] - end - - def enable_apt? - true - end - - def apt_targets - return [] unless enable_apt? - - targets = (ENV["APT_TARGETS"] || "").split(",") - targets = apt_targets_default if targets.empty? - - targets.find_all do |target| - Dir.exist?(File.join(apt_dir, target)) - end - end - - def apt_targets_default - # Disable arm64 targets by default for now - # because they require some setups on host. - [ - "debian-buster", - # "debian-buster-arm64", - "debian-bullseye", - # "debian-bullseye-arm64", - "ubuntu-bionic", - # "ubuntu-bionic-arm64", - "ubuntu-focal", - # "ubuntu-focal-arm64", - "ubuntu-groovy", - # "ubuntu-groovy-arm64", - ] - end - - def deb_archive_base_name - "#{@package}-#{@deb_archive_base_name_version}" - end - - def deb_archive_name - "#{@package}-#{@deb_upstream_version}.tar.gz" - end - - def apt_dir - "apt" - end - - def apt_prepare_debian_dir(tmp_dir, target) - source_debian_dir = nil - specific_debian_dir = "debian.#{target}" - distribution, code_name, _architecture = target.split("-", 3) - platform = [distribution, code_name].join("-") - platform_debian_dir = "debian.#{platform}" - if File.exist?(specific_debian_dir) - source_debian_dir = specific_debian_dir - elsif File.exist?(platform_debian_dir) - source_debian_dir = platform_debian_dir - else - source_debian_dir = "debian" - end - - prepared_debian_dir = "#{tmp_dir}/debian.#{target}" - cp_r(source_debian_dir, prepared_debian_dir) - control_in_path = "#{prepared_debian_dir}/control.in" - if File.exist?(control_in_path) - control_in = File.read(control_in_path) - rm_f(control_in_path) - File.open("#{prepared_debian_dir}/control", "w") do |control| - prepared_control = apt_prepare_debian_control(control_in, target) - control.print(prepared_control) - end - end - end - - def apt_prepare_debian_control(control_in, target) - message = "#{__method__} must be defined to use debian/control.in" - raise NotImplementedError, message - end - - def apt_build(console: false) - tmp_dir = "#{apt_dir}/tmp" - rm_rf(tmp_dir) - mkdir_p(tmp_dir) - cp(deb_archive_name, - File.join(tmp_dir, deb_archive_name)) - apt_targets.each do |target| - apt_prepare_debian_dir(tmp_dir, target) - end - - env_sh = "#{apt_dir}/env.sh" - File.open(env_sh, "w") do |file| - file.puts(<<-ENV) -PACKAGE=#{@package} -VERSION=#{@deb_upstream_version} - ENV - end - - apt_targets.each do |target| - cd(apt_dir) do - distribution, version, architecture = target.split("-", 3) - os = "#{distribution}-#{version}" - docker_run(os, architecture, console: console) - end - end - end - - def define_apt_task - namespace :apt do - source_build_sh = "#{__dir__}/apt/build.sh" - build_sh = "#{apt_dir}/build.sh" - repositories_dir = "#{apt_dir}/repositories" - - file build_sh => source_build_sh do - cp(source_build_sh, build_sh) - end - - directory repositories_dir - - desc "Build deb packages" - if enable_apt? - build_dependencies = [ - deb_archive_name, - build_sh, - repositories_dir, - ] - else - build_dependencies = [] - end - task :build => build_dependencies do - apt_build if enable_apt? - end - - namespace :build do - desc "Open console" - task :console => build_dependencies do - apt_build(console: true) if enable_apt? - end - end - end - - desc "Release APT repositories" - apt_tasks = [ - "apt:build", - ] - task :apt => apt_tasks - end - - def enable_yum? - true - end - - def yum_targets - return [] unless enable_yum? - - targets = (ENV["YUM_TARGETS"] || "").split(",") - targets = yum_targets_default if targets.empty? - - targets.find_all do |target| - Dir.exist?(File.join(yum_dir, target)) - end - end - - def yum_targets_default - # Disable aarch64 targets by default for now - # because they require some setups on host. - [ - "centos-7", - # "centos-7-aarch64", - "centos-8", - # "centos-8-aarch64", - ] - end - - def rpm_archive_base_name - "#{@package}-#{@rpm_version}" - end - - def rpm_archive_name - "#{rpm_archive_base_name}.tar.gz" - end - - def yum_dir - "yum" - end - - def yum_build_sh - "#{yum_dir}/build.sh" - end - - def yum_expand_variable(key) - case key - when "PACKAGE" - @rpm_package - when "VERSION" - @rpm_version - when "RELEASE" - @rpm_release - else - nil - end - end - - def yum_spec_in_path - "#{yum_dir}/#{@rpm_package}.spec.in" - end - - def yum_build(console: false) - tmp_dir = "#{yum_dir}/tmp" - rm_rf(tmp_dir) - mkdir_p(tmp_dir) - cp(rpm_archive_name, - File.join(tmp_dir, rpm_archive_name)) - - env_sh = "#{yum_dir}/env.sh" - File.open(env_sh, "w") do |file| - file.puts(<<-ENV) -SOURCE_ARCHIVE=#{rpm_archive_name} -PACKAGE=#{@rpm_package} -VERSION=#{@rpm_version} -RELEASE=#{@rpm_release} - ENV - end - - spec = "#{tmp_dir}/#{@rpm_package}.spec" - spec_in_data = File.read(yum_spec_in_path) - spec_data = substitute_content(spec_in_data) do |key, matched| - yum_expand_variable(key) || matched - end - File.open(spec, "w") do |spec_file| - spec_file.print(spec_data) - end - - yum_targets.each do |target| - cd(yum_dir) do - distribution, version, architecture = target.split("-", 3) - os = "#{distribution}-#{version}" - docker_run(os, architecture, console: console) - end - end - end - - def define_yum_task - namespace :yum do - source_build_sh = "#{__dir__}/yum/build.sh" - file yum_build_sh => source_build_sh do - cp(source_build_sh, yum_build_sh) - end - - repositories_dir = "#{yum_dir}/repositories" - directory repositories_dir - - desc "Build RPM packages" - if enable_yum? - build_dependencies = [ - repositories_dir, - rpm_archive_name, - yum_build_sh, - yum_spec_in_path, - ] - else - build_dependencies = [] - end - task :build => build_dependencies do - yum_build if enable_yum? - end - - namespace :build do - desc "Open console" - task :console => build_dependencies do - yum_build(console: true) if enable_yum? - end - end - end - - desc "Release Yum repositories" - yum_tasks = [ - "yum:build", - ] - task :yum => yum_tasks - end - - def define_version_task - namespace :version do - desc "Update versions" - task :update do - update_debian_changelog - update_spec - end - end - end - - def package_changelog_message - "New upstream release." - end - - def packager_name - ENV["DEBFULLNAME"] || ENV["NAME"] || guess_packager_name_from_git - end - - def guess_packager_name_from_git - name = `git config --get user.name`.chomp - return name unless name.empty? - `git log -n 1 --format=%aN`.chomp - end - - def packager_email - ENV["DEBEMAIL"] || ENV["EMAIL"] || guess_packager_email_from_git - end - - def guess_packager_email_from_git - email = `git config --get user.email`.chomp - return email unless email.empty? - `git log -n 1 --format=%aE`.chomp - end - - def update_content(path) - if File.exist?(path) - content = File.read(path) - else - content = "" - end - content = yield(content) - File.open(path, "w") do |file| - file.puts(content) - end - end - - def update_debian_changelog - return unless enable_apt? - - Dir.glob("debian*") do |debian_dir| - update_content("#{debian_dir}/changelog") do |content| - <<-CHANGELOG.rstrip -#{@package} (#{@deb_upstream_version}-#{@deb_release}) unstable; urgency=low - - * New upstream release. - - -- #{packager_name} <#{packager_email}> #{@release_time.rfc2822} - -#{content} - CHANGELOG - end - end - end - - def update_spec - return unless enable_yum? - - release_time = @release_time.strftime("%a %b %d %Y") - update_content(yum_spec_in_path) do |content| - content = content.sub(/^(%changelog\n)/, <<-CHANGELOG) -%changelog -* #{release_time} #{packager_name} <#{packager_email}> - #{@rpm_version}-#{@rpm_release} -- #{package_changelog_message} - - CHANGELOG - content = content.sub(/^(Release:\s+)\d+/, "\\11") - content.rstrip - end - end - - def define_docker_tasks - namespace :docker do - pull_tasks = [] - push_tasks = [] - - (apt_targets + yum_targets).each do |target| - distribution, version, architecture = target.split("-", 3) - os = "#{distribution}-#{version}" - - namespace :pull do - desc "Pull built image for #{target}" - task target do - docker_pull(os, architecture) - end - pull_tasks << "docker:pull:#{target}" - end - - namespace :push do - desc "Push built image for #{target}" - task target do - docker_push(os, architecture) - end - push_tasks << "docker:push:#{target}" - end - end - - desc "Pull built images" - task :pull => pull_tasks - - desc "Push built images" - task :push => push_tasks - end - end -end diff --git a/dev/tasks/linux-packages/travis.linux.arm64.yml b/dev/tasks/linux-packages/travis.linux.arm64.yml deleted file mode 100644 index 6078942e737f..000000000000 --- a/dev/tasks/linux-packages/travis.linux.arm64.yml +++ /dev/null @@ -1,149 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -{% import 'macros.jinja' as macros with context %} - -arch: arm64-graviton2 -virt: vm -os: linux -dist: focal -group: edge -language: minimal - -addons: - apt: - packages: - - apt-utils - # We need Ubuntu 20.10 or later - # - createrepo_c - - devscripts - - gpg - - libgit2-dev - - python3-pip - - rake - - rpm - - # To build createrepo_c from source. - # We can remove them when we can install createrepo_c package - - cmake - - libbz2-dev - - libcurl4-openssl-dev - - libglib2.0-dev - - liblzma-dev - - libmagic-dev - - librpm-dev - - libsqlite3-dev - - libssl-dev - - libxml2-dev - - libzstd-dev - - pkg-config - - zlib1g-dev - -services: - - docker - -# don't build twice -if: tag IS blank - -env: - global: - - APT_TARGETS={{ target }} - - ARROW_VERSION={{ arrow.version }} - - BUILD_REF={{ arrow.head }} - - TRAVIS_TAG={{ task.tag }} - - YUM_TARGETS={{ target }} - -before_script: - - set -e - {{ macros.travis_checkout_arrow() }} - {{ macros.travis_docker_login() }} - - # Build createrepo_c from source. - # We can remove them when we can install createrepo_c package - - git clone --depth 1 https://github.com/rpm-software-management/createrepo_c.git - - pushd createrepo_c - - | - /usr/bin/cmake \ - -DCMAKE_INSTALL_PREFIX=/usr \ - -DENABLE_BASHCOMP=OFF \ - -DENABLE_DRPM=OFF \ - -DENABLE_PYTHON=OFF \ - -DWITH_LIBMODULEMD=OFF \ - -DWITH_ZCHUNK=OFF \ - . - - make -j$(nproc) - - sudo make install - - popd - - rm -rf createrepo_c - -script: - # Build packages - - pushd arrow/dev/tasks/linux-packages - - rake version:update - - | - rake docker:pull || : - - pushd apache-arrow-apt-source/apt - - | - for target in debian-* ubuntu-*; do - cp -a ${target} ${target}-arm64 - done - - popd - - pushd apache-arrow-release/yum - - | - for target in centos-*; do - cp -a ${target} ${target}-aarch64 - done - - popd - - | - rake \ - --trace \ - {{ task_namespace }}:build \ - BUILD_DIR=build \ - DEB_BUILD_OPTIONS=parallel=2 \ - RPM_BUILD_NCPUS=2 - - sudo rm -rf */*/build - - popd - # Push Docker image - - pushd arrow/dev/tasks/linux-packages - - | - docker login -u "${DOCKERHUB_USER}" \ - -p "${DOCKERHUB_TOKEN}" || : - - | - rake docker:push || : - - popd - # Test built packages - - | - (echo "Key-Type: RSA"; \ - echo "Key-Length: 4096"; \ - echo "Name-Real: Test"; \ - echo "Name-Email: test@example.com"; \ - echo "%no-protection") | \ - gpg --full-generate-key --batch - - | - GPG_KEY_ID=$(gpg --list-keys --with-colon test@example.com | grep fpr | cut -d: -f10) - - gpg --export --armor test@example.com > arrow/dev/tasks/linux-packages/KEYS - - pushd arrow/dev/tasks/linux-packages - - | - rake --trace {{ task_namespace }}:test \ - CREATEREPO=createrepo_c \ - GPG_KEY_ID=${GPG_KEY_ID} - - rm -rf {{ task_namespace }}/repositories - - popd - -after_success: - {% set patterns = upload_extensions | format_all("arrow/dev/tasks/linux-packages/**/*{}") %} - {{ macros.travis_upload_releases(patterns) }} diff --git a/dev/tasks/linux-packages/yum/build.sh b/dev/tasks/linux-packages/yum/build.sh deleted file mode 100755 index 01746803adf6..000000000000 --- a/dev/tasks/linux-packages/yum/build.sh +++ /dev/null @@ -1,157 +0,0 @@ -#!/bin/bash -# -*- sh-indentation: 2; sh-basic-offset: 2 -*- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -u - -run() -{ - "$@" - if test $? -ne 0; then - echo "Failed $@" - exit 1 - fi -} - -rpmbuild_options= - -. /host/env.sh - -distribution=$(cut -d " " -f 1 /etc/redhat-release | tr "A-Z" "a-z") -if grep -q Linux /etc/redhat-release; then - distribution_version=$(cut -d " " -f 4 /etc/redhat-release) -else - distribution_version=$(cut -d " " -f 3 /etc/redhat-release) -fi -distribution_version=$(echo ${distribution_version} | sed -e 's/\..*$//g') - -architecture="$(arch)" -lib_directory=/usr/lib64 -case "${architecture}" in - i*86) - architecture=i386 - lib_directory=/usr/lib - ;; -esac - -run mkdir -p /build -run cd /build -find . -not -path ./ccache -a -not -path "./ccache/*" -delete -if which ccache > /dev/null 2>&1; then - export CCACHE_COMPILERCHECK=content - export CCACHE_COMPRESS=1 - export CCACHE_COMPRESSLEVEL=6 - export CCACHE_MAXSIZE=500M - export CCACHE_DIR="${PWD}/ccache" - ccache --show-stats - if [ -d "${lib_directory}/ccache" ]; then - PATH="${lib_directory}/ccache:$PATH" - fi -fi - -run mkdir -p rpmbuild -run cd -rm -rf rpmbuild -run ln -fs /build/rpmbuild ./ -if [ -x /usr/bin/rpmdev-setuptree ]; then - rm -rf .rpmmacros - run rpmdev-setuptree -else - run cat < ~/.rpmmacros -%_topdir ${HOME}/rpmbuild -RPMMACROS - run mkdir -p rpmbuild/SOURCES - run mkdir -p rpmbuild/SPECS - run mkdir -p rpmbuild/BUILD - run mkdir -p rpmbuild/RPMS - run mkdir -p rpmbuild/SRPMS -fi - -repositories="/host/repositories" -repository="${repositories}/${distribution}/${distribution_version}" -rpm_dir="${repository}/${architecture}/Packages" -srpm_dir="${repository}/source/SRPMS" -run mkdir -p "${rpm_dir}" "${srpm_dir}" - -# for debug -# rpmbuild_options="$rpmbuild_options --define 'optflags -O0 -g3'" - -if [ -n "${SOURCE_ARCHIVE}" ]; then - case "${RELEASE}" in - 0.dev*) - source_archive_base_name=$( \ - echo ${SOURCE_ARCHIVE} | sed -e 's/\.tar\.gz$//') - run tar xf /host/tmp/${SOURCE_ARCHIVE} \ - --transform="s,^[^/]*,${PACKAGE}," - run mv \ - ${PACKAGE} \ - ${source_archive_base_name} - run tar czf \ - rpmbuild/SOURCES/${SOURCE_ARCHIVE} \ - ${source_archive_base_name} - run rm -rf ${source_archive_base_name} - ;; - *) - run cp /host/tmp/${SOURCE_ARCHIVE} rpmbuild/SOURCES/ - ;; - esac -else - run cp /host/tmp/${PACKAGE}-${VERSION}.* rpmbuild/SOURCES/ -fi -run cp \ - /host/tmp/${PACKAGE}.spec \ - rpmbuild/SPECS/ - -run cat < build.sh -#!/bin/bash - -rpmbuild -ba ${rpmbuild_options} rpmbuild/SPECS/${PACKAGE}.spec -BUILD -run chmod +x build.sh -if [ -n "${DEVTOOLSET_VERSION:-}" ]; then - run cat < which-strip.sh -#!/bin/bash - -which strip -WHICH_STRIP - run chmod +x which-strip.sh - run cat <> ~/.rpmmacros -%__strip $(run scl enable devtoolset-${DEVTOOLSET_VERSION} ./which-strip.sh) -USE_DEVTOOLSET_STRIP - if [ "${DEBUG:-no}" = "yes" ]; then - run scl enable devtoolset-${DEVTOOLSET_VERSION} ./build.sh - else - run scl enable devtoolset-${DEVTOOLSET_VERSION} ./build.sh > /dev/null - fi -else - if [ "${DEBUG:-no}" = "yes" ]; then - run ./build.sh - else - run ./build.sh > /dev/null - fi -fi - -if which ccache > /dev/null 2>&1; then - ccache --show-stats -fi - -run mv rpmbuild/RPMS/*/* "${rpm_dir}/" -run mv rpmbuild/SRPMS/* "${srpm_dir}/" - -run chown -R "$(stat --format "%u:%g" "${repositories}")" "${repositories}" diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja deleted file mode 100644 index bfbd6ec25884..000000000000 --- a/dev/tasks/macros.jinja +++ /dev/null @@ -1,198 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -{%- macro github_header() -%} -# NOTE: must set "Crossbow" as name to have the badge links working in the -# github comment reports! -name: Crossbow -on: - push: - branches: - - "*-github-*" -{% endmacro %} - -{%- macro github_checkout_arrow() -%} - - name: Checkout Arrow - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow config core.symlinks true - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive - - name: Fetch Submodules and Tags - shell: bash - run: cd arrow && ci/scripts/util_checkout.sh -{% endmacro %} - -{%- macro github_login_dockerhub() -%} - - name: Login to Dockerhub - uses: docker/login-action@v1 - with: - username: {{ '${{ secrets.DOCKERHUB_USER }}' }} - password: {{ '${{ secrets.DOCKERHUB_TOKEN }}' }} -{% endmacro %} - -{%- macro github_login_ghcr() -%} - - name: Login to GitHub Container Registry - shell: bash - run: docker login ghcr.io -u {{ '${{ github.repository_owner }}' }} -p {{ '${{ secrets.CROSSBOW_GHCR_TOKEN }}' }} -{% endmacro %} - -{%- macro github_install_archery() -%} - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: Install Archery - shell: bash - run: pip install -e arrow/dev/archery[all] -{% endmacro %} - -{%- macro github_upload_releases(pattern) -%} - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: Setup Crossbow - shell: bash - run: pip install -e arrow/dev/archery[crossbow] - - name: Upload artifacts - shell: bash - run: | - archery crossbow \ - --queue-path $(pwd) \ - --queue-remote {{ queue_remote_url }} \ - upload-artifacts \ - --sha {{ task.branch }} \ - --tag {{ task.tag }} \ - {% if pattern is string %} - --pattern "{{ pattern }}" - {% elif pattern is iterable %} - {% for p in pattern %} - --pattern "{{ p }}" {{ "\\" if not loop.last else "" }} - {% endfor %} - {% endif %} - env: - CROSSBOW_GITHUB_TOKEN: {{ '${{ secrets.CROSSBOW_GITHUB_TOKEN }}' }} -{% endmacro %} - -{%- macro github_upload_gemfury(pattern) -%} - {%- if arrow.branch == 'master' -%} - - name: Upload package to Gemfury - shell: bash - run: | - path=$(ls {{ pattern }}) - curl -F "package=@${path}" https://${CROSSBOW_GEMFURY_TOKEN}@push.fury.io/${CROSSBOW_GEMFURY_ORG}/ - env: - CROSSBOW_GEMFURY_TOKEN: {{ '${{ secrets.CROSSBOW_GEMFURY_TOKEN }}' }} - CROSSBOW_GEMFURY_ORG: {{ '${{ secrets.CROSSBOW_GEMFURY_ORG }}' }} - {% endif %} -{% endmacro %} - -{%- macro azure_checkout_arrow() -%} - - script: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive - displayName: Clone arrow -{% endmacro %} - -{%- macro azure_upload_releases(pattern) -%} - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.8' - - script: pip install -e arrow/dev/archery[crossbow] - displayName: Install Crossbow - - bash: | - archery crossbow \ - --queue-path $(pwd) \ - --queue-remote {{ queue_remote_url }} \ - upload-artifacts \ - --sha {{ task.branch }} \ - --tag {{ task.tag }} \ - {% if pattern is string %} - --pattern "{{ pattern }}" - {% elif pattern is iterable %} - {% for p in pattern %} - --pattern "{{ p }}" {{ "" if not loop.last else "" }} - {% endfor %} - {% endif %} - env: - CROSSBOW_GITHUB_TOKEN: $(CROSSBOW_GITHUB_TOKEN) - displayName: Upload packages as a GitHub release -{% endmacro %} - -{%- macro azure_upload_anaconda(pattern) -%} - {%- if arrow.branch == 'master' -%} - - task: CondaEnvironment@1 - inputs: - packageSpecs: 'anaconda-client' - installOptions: '-c conda-forge' - updateConda: no - - script: | - conda install -y anaconda-client - anaconda -t $(CROSSBOW_ANACONDA_TOKEN) upload --force {{ pattern }} - displayName: Upload packages to Anaconda - {% endif %} -{% endmacro %} - -{%- macro travis_checkout_arrow() -%} - - git clone --no-checkout {{ arrow.remote }} arrow - - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - - git -C arrow checkout FETCH_HEAD - - git -C arrow submodule update --init --recursive -{% endmacro %} - -{%- macro travis_install_archery() -%} - - sudo -H pip3 install --upgrade pip - - sudo -H pip3 install docker-compose - - sudo -H pip3 install -e arrow/dev/archery[docker] -{% endmacro %} - -{%- macro travis_docker_login() -%} - - echo "${DOCKERHUB_TOKEN}" | docker login --username "${DOCKERHUB_USER}" --password-stdin -{% endmacro %} - -{%- macro travis_upload_releases(pattern) -%} - - sudo -H pip3 install pygit2==1.0 - - sudo -H pip3 install -e arrow/dev/archery[crossbow] - - | - archery crossbow \ - --queue-path $(pwd) \ - --queue-remote {{ queue_remote_url }} \ - upload-artifacts \ - --sha {{ task.branch }} \ - --tag {{ task.tag }} \ - {% if pattern is string %} - --pattern "{{ pattern }}" - {% elif pattern is iterable %} - {% for p in pattern %} - --pattern "{{ p }}" {{ "\\" if not loop.last else "" }} - {% endfor %} - {% endif %} -{% endmacro %} - -{%- macro travis_upload_gemfury(pattern) -%} - {%- if arrow.branch == 'master' -%} - - | - WHEEL_PATH=$(echo arrow/python/repaired_wheels/*.whl) - curl \ - -F "package=@${WHEEL_PATH}" \ - "https://${CROSSBOW_GEMFURY_TOKEN}@push.fury.io/${CROSSBOW_GEMFURY_ORG}/" - {% endif %} -{% endmacro %} diff --git a/dev/tasks/nightlies.sample.yml b/dev/tasks/nightlies.sample.yml deleted file mode 100644 index 710f7c0ad377..000000000000 --- a/dev/tasks/nightlies.sample.yml +++ /dev/null @@ -1,68 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# this travis configuration can be used to submit cron scheduled tasks -# 1. copy this file to one of crossbow's branch (master for example) with -# filename .travis.yml -# 2. setup daily cron jobs for that particular branch, see travis' -# documentation https://docs.travis-ci.com/user/cron-jobs/ - -branches: - # don't attempt to build branches intended for windows builds - except: - - /.*win.*/ - -os: linux -dist: trusty -language: generic - -before_install: - # Install Miniconda. - - echo `pwd` - - | - echo "" - echo "Installing a fresh version of Miniconda." - MINICONDA_URL="https://repo.continuum.io/miniconda" - MINICONDA_FILE="Miniconda3-latest-Linux-x86_64.sh" - curl -L -O "${MINICONDA_URL}/${MINICONDA_FILE}" - bash $MINICONDA_FILE -b - - # Configure conda. - - | - echo "" - echo "Configuring conda." - source /home/travis/miniconda3/bin/activate root - conda config --remove channels defaults - conda config --add channels defaults - conda config --add channels conda-forge - conda config --set show_channel_urls true - -install: - - pushd .. - # to build against a specific branch of a fork - # git clone -b https://github.com//arrow - - git clone https://github.com/apache/arrow - - pip install dev/archery[crossbow] - -script: - # submit packaging tasks - - | - if [ $TRAVIS_EVENT_TYPE = "cron" ]; then - archery crossbow submit -g conda -g wheel -g linux - else - archery crossbow submit --dry-run -g conda -g wheel -g linux - fi diff --git a/dev/tasks/nuget-packages/github.linux.yml b/dev/tasks/nuget-packages/github.linux.yml deleted file mode 100644 index cd03a7bfeea1..000000000000 --- a/dev/tasks/nuget-packages/github.linux.yml +++ /dev/null @@ -1,43 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -{% import 'macros.jinja' as macros with context %} - -{{ macros.github_header() }} - -jobs: - package: - name: Package - runs-on: ubuntu-latest - steps: - {{ macros.github_checkout_arrow()|indent }} - {{ macros.github_install_archery()|indent }} - - - name: Prepare version - run: | - sed -i'' -E -e \ - "s/^ .+<\/Version>/ {{ arrow.no_rc_semver_version }}<\/Version>/" \ - arrow/csharp/Directory.Build.props - - name: Build package - run: | - pushd arrow - archery docker run {{ run }} - popd - - {% set patterns = ["arrow/csharp/artifacts/**/*.nupkg", - "arrow/csharp/artifacts/**/*.snupkg"] %} - {{ macros.github_upload_releases(patterns)|indent }} diff --git a/dev/tasks/python-sdist/github.yml b/dev/tasks/python-sdist/github.yml deleted file mode 100644 index 68371876ab80..000000000000 --- a/dev/tasks/python-sdist/github.yml +++ /dev/null @@ -1,45 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -{% import 'macros.jinja' as macros with context %} - -{{ macros.github_header() }} - -jobs: - build: - name: "Build sdist" - runs-on: ubuntu-20.04 - steps: - {{ macros.github_checkout_arrow()|indent }} - {{ macros.github_install_archery()|indent }} - - - name: Build sdist - run: | - archery docker run python-sdist - {% if arrow.branch == 'master' %} - archery docker push python-sdist || : - {% endif %} - env: - PYARROW_VERSION: {{ arrow.no_rc_version }} - - - name: Test sdist - run: archery docker run ubuntu-python-sdist-test - env: - PYARROW_VERSION: {{ arrow.no_rc_version }} - - {{ macros.github_upload_releases("arrow/python/dist/*.tar.gz")|indent }} - {{ macros.github_upload_gemfury("arrow/python/dist/*.tar.gz")|indent }} diff --git a/dev/tasks/python-wheels/github.linux.amd64.yml b/dev/tasks/python-wheels/github.linux.amd64.yml deleted file mode 100644 index a62640787155..000000000000 --- a/dev/tasks/python-wheels/github.linux.amd64.yml +++ /dev/null @@ -1,48 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -{% import 'macros.jinja' as macros with context %} - -{{ macros.github_header() }} - -jobs: - build: - name: "Build wheel for Manylinux {{ manylinux_version }}" - runs-on: ubuntu-latest - env: - # archery uses these environment variables - ARCH: amd64 - PYTHON: {{ python_version }} - - steps: - {{ macros.github_checkout_arrow()|indent }} - {{ macros.github_install_archery()|indent }} - {{ macros.github_login_dockerhub()|indent }} - - - name: Build wheel - shell: bash - run: archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-manylinux-{{ manylinux_version }} - - # TODO(kszucs): auditwheel show - - name: Test wheel - shell: bash - run: | - archery docker run python-wheel-manylinux-test-imports - archery docker run python-wheel-manylinux-test-unittests - - {{ macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }} - {{ macros.github_upload_gemfury("arrow/python/repaired_wheels/*.whl")|indent }} diff --git a/dev/tasks/python-wheels/github.osx.yml b/dev/tasks/python-wheels/github.osx.yml deleted file mode 100644 index af0cc44ef091..000000000000 --- a/dev/tasks/python-wheels/github.osx.yml +++ /dev/null @@ -1,133 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{% import 'macros.jinja' as macros with context %} - -{{ macros.github_header() }} - -env: - ARROW_S3: {{ arrow_s3 }} - MACOSX_DEPLOYMENT_TARGET: {{ macos_deployment_target }} - MB_PYTHON_VERSION: {{ python_version }} - PLAT: x86_64 - PYARROW_BUILD_VERBOSE: 1 - PYARROW_VERSION: {{ arrow.no_rc_version }} - PYTHON_VERSION: {{ python_version }} - SETUPTOOLS_SCM_PRETEND_VERSION: {{ arrow.no_rc_version }} - VCPKG_DEFAULT_TRIPLET: x64-osx-static-release - VCPKG_FEATURE_FLAGS: "-manifests" - VCPKG_OVERLAY_TRIPLETS: {{ "${{ github.workspace }}/arrow/ci/vcpkg" }} - -jobs: - build: - name: Build wheel for OS X - runs-on: macos-latest - steps: - {{ macros.github_checkout_arrow()|indent }} - - - name: Install System Dependencies - run: brew install bison ninja - - # Restore from cache the previously built ports. - # If cache-miss, download and build vcpkg (aka "bootstrap vcpkg"). - - name: Restore from Cache and Install Vcpkg - # Download and build vcpkg, without installing any port. - # If content is cached already, it is a no-op. - uses: kszucs/run-vcpkg@main - with: - # Required to prevent cache eviction on crossbow's main branch - # where we build pre-build the vcpkg packages - setupOnly: true - doNotSaveCache: true - appendedCacheKey: "-macos-{{ macos_deployment_target }}" - vcpkgDirectory: {{ "${{ github.workspace }}/vcpkg" }} - vcpkgGitCommitId: fced4bef1606260f110d74de1ae1975c2b9ac549 - - - name: Patch Vcpkg Ports - run: | - vcpkg_patch_file="../arrow/ci/vcpkg/ports.patch" - cd $VCPKG_ROOT - if ! git apply --reverse --check --ignore-whitespace ${vcpkg_patch_file}; then - git apply --ignore-whitespace ${vcpkg_patch_file} - echo "Patch successfully applied!" - fi - - # Now that vcpkg is installed, it is being used to run with the desired arguments. - - name: Install Vcpkg Dependencies - run: | - $VCPKG_ROOT/vcpkg install \ - abseil \ - boost-filesystem \ - brotli \ - bzip2 \ - c-ares \ - curl \ - flatbuffers \ - gflags \ - glog \ - grpc \ - lz4 \ - openssl \ - orc \ - protobuf \ - rapidjson \ - re2 \ - snappy \ - thrift \ - utf8proc \ - zlib \ - zstd - - {% if arrow_s3 == "ON" %} - - name: Install AWS SDK C++ - run: | - $VCPKG_ROOT/vcpkg install \ - aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] - {% endif %} - - - name: Setup Multibuild - run: | - git clone https://github.com/matthew-brett/multibuild - git -C multibuild checkout 03950c9a7feb09d215f82d6563c4ffd91274a1e1 - - - name: Build Wheel - env: - CONFIG_PATH: /dev/null - run: | - # configure environment and install python - source multibuild/common_utils.sh - source multibuild/travis_osx_steps.sh - before_install - - # install python dependencies - pip install -r arrow/python/requirements-wheel-build.txt delocate - - # build the wheel - arrow/ci/scripts/python_wheel_macos_build.sh $(pwd)/arrow $(pwd)/build - - - name: Setup Python for Testing - uses: actions/setup-python@v2 - with: - python-version: "{{ python_version }}" - - - name: Test the Wheel - run: | - # TODO(kszucs): temporarily remove homebrew libs - unset MACOSX_DEPLOYMENT_TARGET - arrow/ci/scripts/python_wheel_macos_test.sh $(pwd)/arrow - - {{ macros.github_upload_releases("arrow/python/dist/*.whl")|indent }} - {{ macros.github_upload_gemfury("arrow/python/dist/*.whl")|indent }} diff --git a/dev/tasks/python-wheels/github.windows.yml b/dev/tasks/python-wheels/github.windows.yml deleted file mode 100644 index 922533b0f827..000000000000 --- a/dev/tasks/python-wheels/github.windows.yml +++ /dev/null @@ -1,53 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -{% import 'macros.jinja' as macros with context %} - -{{ macros.github_header() }} - -jobs: - build: - name: "Build wheel for Windows" - runs-on: windows-2019 - env: - # archery uses this environment variable - PYTHON: {{ python_version }} - # this is a private repository at the moment (mostly because of licensing - # consideration of windows images with visual studio), but anyone can - # recreate the image by manually building it via: - # `archery build python-wheel-windows-vs2017` - # note that we don't run docker build since there wouldn't be a cache hit - # and rebuilding the dependencies takes a fair amount of time - REPO: ghcr.io/ursacomputing/arrow - # prefer the docker cli over docker-compose - ARCHERY_USE_DOCKER_CLI: 1 - - steps: - {{ macros.github_checkout_arrow()|indent }} - {{ macros.github_login_ghcr()|indent }} - {{ macros.github_install_archery()|indent }} - - - name: Build wheel - shell: cmd - run: archery docker run --no-build -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2017 - - - name: Test wheel - shell: cmd - run: archery docker run python-wheel-windows-test - - {{ macros.github_upload_releases("arrow/python/dist/*.whl")|indent }} - {{ macros.github_upload_gemfury("arrow/python/dist/*.whl")|indent }} diff --git a/dev/tasks/python-wheels/travis.linux.arm64.yml b/dev/tasks/python-wheels/travis.linux.arm64.yml deleted file mode 100644 index a5c0f7408d22..000000000000 --- a/dev/tasks/python-wheels/travis.linux.arm64.yml +++ /dev/null @@ -1,73 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -{% import 'macros.jinja' as macros with context %} - -arch: arm64-graviton2 -virt: vm -os: linux -dist: focal -group: edge -language: minimal - -addons: - apt: - packages: - - libgit2-dev - - python3-pip - -services: - - docker - -# don't build twice -if: tag IS blank - -env: - global: - - BUILD_REF={{ arrow.head }} - - TRAVIS_TAG={{ task.tag }} - # archery uses these environment variables - - ARCH=arm64v8 - - PYTHON={{ python_version }} - -before_script: - - set -e - {{ macros.travis_checkout_arrow() }} - {{ macros.travis_docker_login() }} - -script: - # Install Archery and Crossbow dependencies - {{ macros.travis_install_archery() }} - - # Build and Test packages - # output something every minutes to prevent travis from killing the build - - while sleep 1m; do echo "=====[ $SECONDS seconds still running ]====="; done & - - archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-manylinux-{{ manylinux_version }} - - archery docker run python-wheel-manylinux-test-imports - - archery docker run python-wheel-manylinux-test-unittests - - kill %1 - -after_success: - # Upload wheel as github artifact - {{ macros.travis_upload_releases("arrow/python/repaired_wheels/*.whl") }} - {{ macros.travis_upload_gemfury("arrow/python/repaired_wheels/*.whl") }} - - {% if arrow.branch == 'master' %} - # Push the docker image to dockerhub - - archery docker push python-wheel-manylinux-{{ manylinux_version }} - - archery docker push python-wheel-manylinux-test-unittests - {% endif %} diff --git a/dev/tasks/r/azure.linux.yml b/dev/tasks/r/azure.linux.yml deleted file mode 100644 index 6e022ec04361..000000000000 --- a/dev/tasks/r/azure.linux.yml +++ /dev/null @@ -1,74 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -jobs: - - job: linux - pool: - vmImage: ubuntu-latest - timeoutInMinutes: 360 - steps: - - script: | - set -ex - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive - displayName: Clone arrow - - - script: | - set -ex - docker -v - docker-compose -v - cd arrow - export R_ORG={{ r_org }} - export R_IMAGE={{ r_image }} - export R_TAG={{ r_tag }} - export DEVTOOLSET_VERSION={{ devtoolset_version|default("-1") }} - docker-compose pull --ignore-pull-failures r - docker-compose build r - displayName: Docker build - - - script: | - set -ex - cd arrow - export R_ORG={{ r_org }} - export R_IMAGE={{ r_image }} - export R_TAG={{ r_tag }} - export ARROW_R_DEV={{ not_cran|default("TRUE") }} - # Note that ci/scripts/r_test.sh sets NOT_CRAN=true if ARROW_R_DEV=TRUE - docker-compose run \ - -e ARROW_DATASET={{ arrow_dataset|default("") }} \ - -e ARROW_PARQUET={{ arrow_parquet|default("") }} \ - -e ARROW_S3={{ arrow_s3|default("") }} \ - -e ARROW_WITH_RE2={{ arrow_with_re2|default("") }} \ - -e ARROW_WITH_UTF8PROC={{ arrow_with_utf8proc|default("") }} \ - -e LIBARROW_MINIMAL={{ libarrow_minimal|default("") }} \ - -e LIBARROW_DOWNLOAD={{ libarrow_download|default("") }} \ - -e LIBARROW_BUILD={{ libarrow_build|default("") }} \ - -e TEST_R_WITH_ARROW={{ with_arrow|default("TRUE") }} \ - r - displayName: Docker run - - - script: | - set -ex - cat arrow/r/check/arrow.Rcheck/00install.out - displayName: Dump install logs - condition: succeededOrFailed() - - script: | - set -ex - cat arrow/r/check/arrow.Rcheck/tests/testthat.Rout* - displayName: Dump test logs - condition: succeededOrFailed() diff --git a/dev/tasks/r/github.devdocs.yml b/dev/tasks/r/github.devdocs.yml deleted file mode 100644 index 1224a2555c8a..000000000000 --- a/dev/tasks/r/github.devdocs.yml +++ /dev/null @@ -1,92 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# NOTE: must set "Crossbow" as name to have the badge links working in the -# github comment reports! -name: Crossbow - -on: - push: - branches: - - "*-github-*" - -jobs: - devdocs: - name: 'R devdocs {{ "${{ matrix.os }}" }} system install: {{ "${{ matrix.system-install }}" }}' - runs-on: {{ "${{ matrix.os }}" }} - strategy: - fail-fast: false - matrix: - os: [macOS-latest, ubuntu-20.04] - # should the install method install libarrow into a system directory - # or a temporary directory. old is the same as a temporary - # directory, but an old version of libarrow will be installed - # into a system directory first (to make sure we can link correctly when building) - system-install: [true, false] - - steps: - - name: Checkout Arrow - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive - - uses: r-lib/actions/setup-r@v1 - - uses: r-lib/actions/setup-pandoc@v1 - - name: Install knitr, rmarkdown - run: | - install.packages(c("rmarkdown", "knitr", "sessioninfo")) - shell: Rscript {0} - - name: Session info - run: | - options(width = 100) - pkgs <- installed.packages()[, "Package"] - sessioninfo::session_info(pkgs, include_base = TRUE) - shell: Rscript {0} - - name: Write the install script - env: - RUN_DEVDOCS: TRUE - DEVDOCS_MACOS: {{ "${{contains(matrix.os, 'macOS')}}" }} - DEVDOCS_UBUNTU: {{ "${{contains(matrix.os, 'ubuntu')}}" }} - DEVDOCS_SYSTEM_INSTALL: {{ "${{contains(matrix.system-install, 'true')}}" }} - DEVDOCS_PRIOR_SYSTEM_INSTALL: {{ "${{contains(matrix.system-install, 'old')}}" }} - run: | - # This isn't actually rendering the docs, but will save arrow/r/vignettes/script.sh - # which can be sourced to install arrow. - rmarkdown::render("arrow/r/vignettes/developing.Rmd") - shell: Rscript {0} - - name: Install from the devdocs - env: - LIBARROW_BINARY: FALSE - ARROW_R_DEV: TRUE - run: bash arrow/r/vignettes/script.sh - shell: bash - - name: Ensure that the Arrow package is loadable and we have the correct one - run: | - echo $LD_LIBRARY_PATH - R --no-save <= 3.3 - - "3.3" - - "3.4" - - "3.5" - env: - R_ORG: "rstudio" - R_IMAGE: "r-base" - R_TAG: "{{ MATRIX }}-bionic" - ARROW_R_DEV: "TRUE" - steps: - - name: Checkout Arrow - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive - - name: Free Up Disk Space - shell: bash - run: arrow/ci/scripts/util_cleanup.sh - - name: Fetch Submodules and Tags - shell: bash - run: cd arrow && ci/scripts/util_checkout.sh - - name: Docker Pull - shell: bash - run: cd arrow && docker-compose pull --ignore-pull-failures r - - name: Docker Build - shell: bash - run: cd arrow && docker-compose build r - - name: Docker Run - shell: bash - run: cd arrow && docker-compose run r - - name: Dump install logs - run: cat arrow/r/check/arrow.Rcheck/00install.out - if: always() - - name: Dump test logs - run: cat arrow/r/check/arrow.Rcheck/tests/testthat.Rout* - if: always() - - name: Save the test output - if: always() - uses: actions/upload-artifact@v2 - with: - name: test-output - path: arrow/r/check/arrow.Rcheck/tests/testthat.Rout* diff --git a/dev/tasks/r/github.macos-linux.local.yml b/dev/tasks/r/github.macos-linux.local.yml deleted file mode 100644 index d9a8ed94206d..000000000000 --- a/dev/tasks/r/github.macos-linux.local.yml +++ /dev/null @@ -1,88 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# NOTE: must set "Crossbow" as name to have the badge links working in the -# github comment reports! -name: Crossbow - -on: - push: - branches: - - "*-github-*" - -jobs: - autobrew: - name: "install from local source" - runs-on: {{ "${{ matrix.os }}" }} - strategy: - fail-fast: false - matrix: - os: [macOS-latest, ubuntu-20.04] - - steps: - - name: Checkout Arrow - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive - - name: Configure non-autobrew dependencies (macos) - run: | - cd arrow/r - brew install openssl - if: contains(matrix.os, 'macOS') - - name: Configure non-autobrew dependencies (linux) - run: | - cd arrow/r - sudo apt-get update - sudo apt install libcurl4-openssl-dev libssl-dev - if: contains(matrix.os, 'ubuntu') - - uses: r-lib/actions/setup-r@v1 - - name: Install dependencies - run: | - install.packages("remotes") - remotes::install_deps("arrow/r", dependencies = TRUE) - remotes::install_cran(c("rcmdcheck", "sys", "sessioninfo")) - shell: Rscript {0} - - name: Session info - run: | - options(width = 100) - pkgs <- installed.packages()[, "Package"] - sessioninfo::session_info(pkgs, include_base = TRUE) - shell: Rscript {0} - - name: Install - env: - _R_CHECK_CRAN_INCOMING_: false - ARROW_USE_PKG_CONFIG: false - FORCE_BUNDLED_BUILD: true - LIBARROW_MINIMAL: false - TEST_R_WITH_ARROW: TRUE - ARROW_R_DEV: TRUE - run: | - cd arrow/r - R CMD INSTALL . --install-tests - - name: Run the tests - run: R -e 'if(tools::testInstalledPackage("arrow") != 0L) stop("There was a test failure.")' - - name: Dump test logs - run: cat arrow-tests/testthat.Rout* - if: failure() - - name: Save the test output - uses: actions/upload-artifact@v2 - with: - name: test-output - path: arrow-tests/testthat.Rout* - if: always() diff --git a/dev/tasks/r/github.macos.autobrew.yml b/dev/tasks/r/github.macos.autobrew.yml deleted file mode 100644 index 1b8500f64b31..000000000000 --- a/dev/tasks/r/github.macos.autobrew.yml +++ /dev/null @@ -1,78 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# NOTE: must set "Crossbow" as name to have the badge links working in the -# github comment reports! -name: Crossbow - -on: - push: - branches: - - "*-github-*" - -jobs: - autobrew: - name: "Autobrew" - runs-on: macOS-latest - steps: - - name: Checkout Arrow - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive - - name: Configure autobrew script - run: | - cd arrow/r - # Put the formula inside r/ so that it's included in the package build - cp ../dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb tools/apache-arrow.rb - # Pin the current commit in the formula to test so that we're not always pulling from master - sed -i.bak -E -e 's@https://github.com/apache/arrow.git"$@{{ arrow.remote }}.git", :revision => "{{ arrow.head }}"@' tools/apache-arrow.rb && rm -f tools/apache-arrow.rb.bak - # Sometimes crossbow gives a remote URL with .git and sometimes not. Make sure there's only one - sed -i.bak -E -e 's@.git.git@.git@' tools/apache-arrow.rb && rm -f tools/apache-arrow.rb.bak - # Get minio for S3 testing - brew install minio - - uses: r-lib/actions/setup-r@v1 - - name: Install dependencies - run: | - install.packages("remotes") - remotes::install_deps("arrow/r", dependencies = TRUE) - remotes::install_cran(c("rcmdcheck", "sys", "sessioninfo")) - shell: Rscript {0} - - name: Session info - run: | - options(width = 100) - pkgs <- installed.packages()[, "Package"] - sessioninfo::session_info(pkgs, include_base = TRUE) - shell: Rscript {0} - - name: Check - env: - _R_CHECK_CRAN_INCOMING_: false - ARROW_USE_PKG_CONFIG: false - run: arrow/ci/scripts/r_test.sh arrow - - name: Dump install logs - run: cat arrow/r/check/arrow.Rcheck/00install.out - if: always() - - name: Dump test logs - run: cat arrow/r/check/arrow.Rcheck/tests/testthat.Rout* - if: always() - - name: Save the test output - if: always() - uses: actions/upload-artifact@v2 - with: - name: test-output - path: arrow/r/check/arrow.Rcheck/tests/testthat.Rout* diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml deleted file mode 100644 index eab3e15ce925..000000000000 --- a/dev/tasks/tasks.yml +++ /dev/null @@ -1,1703 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -groups: - # these groups are just for convenience - # makes it easier to submit related tasks - - ############################# Packaging tasks ############################### - - conda: - - conda-* - - wheel: - - wheel-* - - linux: - - debian-* - - ubuntu-* - - centos-* - - linux-amd64: - - debian-*-amd64 - - ubuntu-*-amd64 - - centos-*-amd64 - - linux-arm64: - - debian-*-arm64 - - ubuntu-*-arm64 - - centos-*-arm64 - - gandiva: - - gandiva-* - - homebrew: - - homebrew-* - - packaging: - - conda-* - - wheel-* - - debian-* - - ubuntu-* - - centos-* - - python-sdist - - nuget - - ############################# Testing tasks ################################# - - test: - - test-* - - cpp: - - test-*cpp* - - c-glib: - - test-*c-glib* - - python: - - test-*python* - - r: - - test*-r-* - - homebrew-r-autobrew - # r-conda tasks - - conda-linux-gcc-py*-cpu-r* - - conda-osx-clang-py*-r* - - conda-win-vs2017-py*-r* - - ruby: - - test-*ruby* - - vcpkg: - - test-*vcpkg* - - integration: - - test-*dask* - - test-*hdfs* - - test-*jpype* - - test-*kartothek* - - test-*pandas* - - test-*spark* - - test-*turbodbc* - - example: - - example-* - - example-cpp: - - example-*cpp* - - verify-rc: - - verify-rc-* - - verify-rc-binaries: - - verify-rc-binaries-* - - verify-rc-wheels: - - verify-rc-wheels-* - - verify-rc-source: - - verify-rc-source-* - - verify-rc-source-macos: - - verify-rc-source-macos-* - - verify-rc-source-linux: - - verify-rc-source-linux-* - - ######################## Tasks to run regularly ############################# - - nightly: - - debian-* - - ubuntu-* - - centos-* - - conda-* - - gandiva-* - # List the homebrews explicitly because we don't care about running homebrew-cpp-autobrew - - homebrew-cpp - - homebrew-r-autobrew - - nuget - - test-* - - example-* - - wheel-* - - python-sdist - -tasks: - # arbitrary_task_name: - # template: path of jinja2 templated yml - # params: optional extra parameters - # artifacts: list of regex patterns, each needs to match a single github - # release asset, version variable is replaced in the pattern - # e.g.: - # - pyarrow-{no_rc_version}-py36(h[a-z0-9]+)_0-linux-64.tar.bz2 - - ############################## Conda Linux ################################## - - conda-clean: - ci: azure - template: conda-recipes/azure.clean.yml - - # Important notes on the conda setup here: - # - # * On conda-forge the `pyarrow` and `arrow-cpp` packages are built in - # the same feedstock as the dependency matrix is the same for them as - # Python and the OS are the main dimension. The R package `r-arrow` is - # an independent feedstock as it doesn't have the Python but the - # R dimension. To limit the number of CI jobs, we are building `r-arrow` - # for R 3.6 with the Python 3.6 jobs and for R 4.0 with the Python 3.7 jobs. - # * The files in `dev/tasks/conda-recipes/.ci_support/` are automatically - # generated and to be synced regularly from the feedstock. We have no way - # yet to generate them inside the arrow repository automatically. - - conda-linux-gcc-py36-cpu-r36: - ci: azure - template: conda-recipes/azure.linux.yml - params: - config: linux_64_cuda_compiler_versionNonenumpy1.17python3.6.____cpython - r_config: linux_64_r_base3.6 - artifacts: - - arrow-cpp-{no_rc_version}-py36(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py36(h[a-z0-9]+)_0_cpu.tar.bz2 - - conda-linux-gcc-py37-cpu-r40: - ci: azure - template: conda-recipes/azure.linux.yml - params: - config: linux_64_cuda_compiler_versionNonenumpy1.17python3.7.____cpython - r_config: linux_64_r_base4.0 - artifacts: - - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 - - conda-linux-gcc-py38-cpu: - ci: azure - template: conda-recipes/azure.linux.yml - params: - config: linux_64_cuda_compiler_versionNonenumpy1.17python3.8.____cpython - artifacts: - - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - - conda-linux-gcc-py39-cpu: - ci: azure - template: conda-recipes/azure.linux.yml - params: - config: linux_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython - artifacts: - - arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 - - conda-linux-gcc-py36-cuda: - ci: azure - template: conda-recipes/azure.linux.yml - params: - config: linux_64_cuda_compiler_version10.2numpy1.17python3.6.____cpython - artifacts: - - arrow-cpp-{no_rc_version}-py36(h[a-z0-9]+)_0_cuda.tar.bz2 - - pyarrow-{no_rc_version}-py36(h[a-z0-9]+)_0_cuda.tar.bz2 - - conda-linux-gcc-py37-cuda: - ci: azure - template: conda-recipes/azure.linux.yml - params: - config: linux_64_cuda_compiler_version10.2numpy1.17python3.7.____cpython - artifacts: - - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cuda.tar.bz2 - - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cuda.tar.bz2 - - conda-linux-gcc-py38-cuda: - ci: azure - template: conda-recipes/azure.linux.yml - params: - config: linux_64_cuda_compiler_version10.2numpy1.17python3.8.____cpython - artifacts: - - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cuda.tar.bz2 - - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cuda.tar.bz2 - - conda-linux-gcc-py39-cuda: - ci: azure - template: conda-recipes/azure.linux.yml - params: - config: linux_64_cuda_compiler_version10.2numpy1.19python3.9.____cpython - artifacts: - - arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cuda.tar.bz2 - - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cuda.tar.bz2 - - conda-linux-gcc-py36-arm64: - ci: drone - template: conda-recipes/drone.yml - params: - config: linux_aarch64_python3.6.____cpython - artifacts: - - arrow-cpp-{no_rc_version}-py36(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py36(h[a-z0-9]+)_0_cpu.tar.bz2 - - conda-linux-gcc-py37-arm64: - ci: drone - template: conda-recipes/drone.yml - params: - config: linux_aarch64_python3.7.____cpython - artifacts: - - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 - - conda-linux-gcc-py38-arm64: - ci: drone - template: conda-recipes/drone.yml - params: - config: linux_aarch64_python3.8.____cpython - artifacts: - - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - - conda-linux-gcc-py39-arm64: - ci: drone - template: conda-recipes/drone.yml - params: - config: linux_aarch64_python3.9.____cpython - artifacts: - - arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 - - ############################## Conda OSX #################################### - - conda-osx-clang-py36-r36: - ci: azure - template: conda-recipes/azure.osx.yml - params: - config: osx_64_numpy1.17python3.6.____cpython - r_config: osx_64_r_base3.6 - artifacts: - - arrow-cpp-{no_rc_version}-py36(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py36(h[a-z0-9]+)_0_cpu.tar.bz2 - - conda-osx-clang-py37-r40: - ci: azure - template: conda-recipes/azure.osx.yml - params: - config: osx_64_numpy1.17python3.7.____cpython - r_config: osx_64_r_base4.0 - artifacts: - - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 - - conda-osx-clang-py38: - ci: azure - template: conda-recipes/azure.osx.yml - params: - config: osx_64_numpy1.17python3.8.____cpython - artifacts: - - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - - conda-osx-clang-py39: - ci: azure - template: conda-recipes/azure.osx.yml - params: - config: osx_64_numpy1.19python3.9.____cpython - artifacts: - - arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 - - conda-osx-arm64-clang-py38: - ci: azure - template: conda-recipes/azure.osx.yml - params: - config: osx_arm64_python3.8.____cpython - artifacts: - - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - - conda-osx-arm64-clang-py39: - ci: azure - template: conda-recipes/azure.osx.yml - params: - config: osx_arm64_python3.9.____cpython - artifacts: - - arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 - - ############################## Conda Windows ################################ - - conda-win-vs2017-py36-r36: - ci: azure - template: conda-recipes/azure.win.yml - params: - config: win_64_cuda_compiler_versionNonenumpy1.17python3.6.____cpython - r_config: win_64_r_base3.6 - artifacts: - - arrow-cpp-{no_rc_version}-py36(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py36(h[a-z0-9]+)_0_cpu.tar.bz2 - - conda-win-vs2017-py37-r40: - ci: azure - template: conda-recipes/azure.win.yml - params: - config: win_64_cuda_compiler_versionNonenumpy1.17python3.7.____cpython - r_config: win_64_r_base4.0 - artifacts: - - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 - - conda-win-vs2017-py38: - ci: azure - template: conda-recipes/azure.win.yml - params: - config: win_64_cuda_compiler_versionNonenumpy1.17python3.8.____cpython - artifacts: - - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - - conda-win-vs2017-py39: - ci: azure - template: conda-recipes/azure.win.yml - params: - config: win_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython - artifacts: - - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - - -{% for python_version, python_tag, abi_tag in [("3.6", "cp36", "cp36m"), - ("3.7", "cp37", "cp37m"), - ("3.8", "cp38", "cp38"), - ("3.9", "cp39", "cp39")] %} - -{############################## Wheel Linux ##################################} - -{% for ci, arch, arch_alias, manylinux in [("github", "amd64", "x86_64", "2010"), - ("github", "amd64", "x86_64", "2014"), - ("travis", "arm64", "aarch64", "2014")] %} - {% set platform_tag = "manylinux{}_{}".format(manylinux, arch_alias) %} - - wheel-manylinux{{ manylinux }}-{{ python_tag }}-{{ arch }}: - ci: {{ ci }} - template: python-wheels/{{ ci }}.linux.{{ arch }}.yml - params: - python_version: {{ python_version }} - manylinux_version: {{ manylinux }} - artifacts: - - pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-{{ platform_tag }}.whl - -{% endfor %} - -{############################## Wheel OSX ####################################} - -# enable S3 support from macOS 10.13 so we don't need to bundle curl, crypt and ssl -{% for macos_version, macos_codename, arrow_s3 in [("10.9", "mavericks", "OFF"), - ("10.13", "high-sierra", "ON")] %} - {% set platform_tag = "macosx_{}_x86_64".format(macos_version.replace('.', '_')) %} - - wheel-osx-{{ macos_codename }}-{{ python_tag }}: - ci: github - template: python-wheels/github.osx.yml - params: - python_version: {{ python_version }} - macos_deployment_target: {{ macos_version }} - arrow_s3: {{ arrow_s3 }} - artifacts: - - pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-{{ platform_tag }}.whl - -{% endfor %} - -{############################## Wheel Windows ################################} - - wheel-windows-{{ python_tag }}: - ci: github - template: python-wheels/github.windows.yml - params: - python_version: {{ python_version }} - artifacts: - - pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-win_amd64.whl - -{% endfor %} - -{############################ Python sdist ####################################} - - python-sdist: - ci: github - template: python-sdist/github.yml - artifacts: - - pyarrow-{no_rc_version}.tar.gz - - ############################## Linux PKGS #################################### - - debian-buster-amd64: - ci: github - template: linux-packages/github.linux.amd64.yml - params: - target: "debian-buster" - task_namespace: "apt" - upload_extensions: - - .ddeb - - .deb - - .debian.tar.xz - - .dsc - - .orig.tar.gz - artifacts: - - apache-arrow-apt-source_{no_rc_version}-1.debian.tar.xz - - apache-arrow-apt-source_{no_rc_version}-1.dsc - - apache-arrow-apt-source_{no_rc_version}-1_all.deb - - apache-arrow-apt-source_{no_rc_version}.orig.tar.gz - - apache-arrow_{no_rc_version}-1.debian.tar.xz - - apache-arrow_{no_rc_version}-1.dsc - - apache-arrow_{no_rc_version}.orig.tar.gz - - gir1.2-arrow-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-arrow-cuda-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-gandiva-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-parquet-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-plasma-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-cuda-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-cuda-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-cuda-glib400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-cuda-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-cuda400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-cuda400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-dataset-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-dataset400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-flight-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-flight400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-flight400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-python-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-python400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-python400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow400_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-glib400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libgandiva-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libgandiva400_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libparquet-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libparquet400_{no_rc_version}-1_[a-z0-9]+.deb - - libplasma-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libplasma-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libplasma-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libplasma-glib400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libplasma-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libplasma400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libplasma400_{no_rc_version}-1_[a-z0-9]+.deb - - plasma-store-server-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - plasma-store-server_{no_rc_version}-1_[a-z0-9]+.deb - - debian-buster-arm64: - ci: travis - template: linux-packages/travis.linux.arm64.yml - params: - target: "debian-buster-arm64" - task_namespace: "apt" - upload_extensions: - - .ddeb - - .deb - - .debian.tar.xz - - .dsc - - .orig.tar.gz - artifacts: - - apache-arrow_{no_rc_version}-1.debian.tar.xz - - apache-arrow_{no_rc_version}-1.dsc - - apache-arrow_{no_rc_version}.orig.tar.gz - - gir1.2-arrow-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-arrow-dataset-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-gandiva-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-parquet-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-dataset-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-dataset400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-flight-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-flight400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-flight400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-python-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-python400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-python400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow400_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-glib400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libgandiva-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libgandiva400_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libparquet-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libparquet400_{no_rc_version}-1_[a-z0-9]+.deb - - debian-bullseye-amd64: - ci: github - template: linux-packages/github.linux.amd64.yml - params: - target: "debian-bullseye" - task_namespace: "apt" - upload_extensions: - - .ddeb - - .deb - - .debian.tar.xz - - .dsc - - .orig.tar.gz - artifacts: - - apache-arrow-apt-source_{no_rc_version}-1.debian.tar.xz - - apache-arrow-apt-source_{no_rc_version}-1.dsc - - apache-arrow-apt-source_{no_rc_version}-1_all.deb - - apache-arrow-apt-source_{no_rc_version}.orig.tar.gz - - apache-arrow_{no_rc_version}-1.debian.tar.xz - - apache-arrow_{no_rc_version}-1.dsc - - apache-arrow_{no_rc_version}.orig.tar.gz - - gir1.2-arrow-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-arrow-cuda-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-gandiva-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-parquet-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-plasma-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-cuda-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-cuda-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-cuda-glib400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-cuda-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-cuda400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-cuda400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-dataset-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-dataset400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-flight-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-flight400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-flight400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-python-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-python400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-python400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow400_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-glib400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libgandiva-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libgandiva400_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libparquet-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libparquet400_{no_rc_version}-1_[a-z0-9]+.deb - - libplasma-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libplasma-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libplasma-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libplasma-glib400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libplasma-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libplasma400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libplasma400_{no_rc_version}-1_[a-z0-9]+.deb - - plasma-store-server-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - plasma-store-server_{no_rc_version}-1_[a-z0-9]+.deb - - debian-bullseye-arm64: - ci: travis - template: linux-packages/travis.linux.arm64.yml - params: - target: "debian-bullseye-arm64" - task_namespace: "apt" - upload_extensions: - - .ddeb - - .deb - - .debian.tar.xz - - .dsc - - .orig.tar.gz - artifacts: - - apache-arrow_{no_rc_version}-1.debian.tar.xz - - apache-arrow_{no_rc_version}-1.dsc - - apache-arrow_{no_rc_version}.orig.tar.gz - - gir1.2-arrow-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-arrow-dataset-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-gandiva-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-parquet-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-dataset-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-dataset400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-flight-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-flight400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-flight400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-python-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-python400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-python400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow400_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-glib400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libgandiva-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libgandiva400_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libparquet-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet400-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libparquet400_{no_rc_version}-1_[a-z0-9]+.deb - - ubuntu-bionic-amd64: - ci: github - template: linux-packages/github.linux.amd64.yml - params: - target: "ubuntu-bionic" - task_namespace: "apt" - env: - UBUNTU: 18.04 - upload_extensions: - - .ddeb - - .deb - - .debian.tar.xz - - .dsc - - .orig.tar.gz - artifacts: - - apache-arrow-apt-source_{no_rc_version}-1.debian.tar.xz - - apache-arrow-apt-source_{no_rc_version}-1.dsc - - apache-arrow-apt-source_{no_rc_version}-1_all.deb - - apache-arrow-apt-source_{no_rc_version}.orig.tar.gz - - apache-arrow_{no_rc_version}-1.debian.tar.xz - - apache-arrow_{no_rc_version}-1.dsc - - apache-arrow_{no_rc_version}.orig.tar.gz - - gir1.2-arrow-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-arrow-cuda-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-gandiva-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-parquet-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-plasma-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-cuda-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-cuda-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-cuda-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-cuda400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-flight-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-flight400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-python-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-python400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow400_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva400_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet400_{no_rc_version}-1_[a-z0-9]+.deb - - libplasma-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libplasma-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libplasma-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libplasma-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libplasma400_{no_rc_version}-1_[a-z0-9]+.deb - - plasma-store-server-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - plasma-store-server_{no_rc_version}-1_[a-z0-9]+.deb - - ubuntu-bionic-arm64: - ci: travis - template: linux-packages/travis.linux.arm64.yml - params: - target: "ubuntu-bionic-arm64" - task_namespace: "apt" - upload_extensions: - - .ddeb - - .deb - - .debian.tar.xz - - .dsc - - .orig.tar.gz - artifacts: - - apache-arrow_{no_rc_version}-1.debian.tar.xz - - apache-arrow_{no_rc_version}-1.dsc - - apache-arrow_{no_rc_version}.orig.tar.gz - - gir1.2-arrow-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-arrow-dataset-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-gandiva-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-parquet-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-flight-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-flight400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-python-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-python400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow400_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva400_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet400_{no_rc_version}-1_[a-z0-9]+.deb - - ubuntu-focal-amd64: - ci: github - template: linux-packages/github.linux.amd64.yml - params: - target: "ubuntu-focal" - task_namespace: "apt" - upload_extensions: - - .ddeb - - .deb - - .debian.tar.xz - - .dsc - - .orig.tar.gz - artifacts: - - apache-arrow-apt-source_{no_rc_version}-1.debian.tar.xz - - apache-arrow-apt-source_{no_rc_version}-1.dsc - - apache-arrow-apt-source_{no_rc_version}-1_all.deb - - apache-arrow-apt-source_{no_rc_version}.orig.tar.gz - - apache-arrow_{no_rc_version}-1.debian.tar.xz - - apache-arrow_{no_rc_version}-1.dsc - - apache-arrow_{no_rc_version}.orig.tar.gz - - gir1.2-arrow-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-arrow-cuda-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-gandiva-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-parquet-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-plasma-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-cuda-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-cuda-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-cuda-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-cuda400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-flight-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-flight400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-python-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-python400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow400_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva400_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet400_{no_rc_version}-1_[a-z0-9]+.deb - - libplasma-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libplasma-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libplasma-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libplasma-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libplasma400_{no_rc_version}-1_[a-z0-9]+.deb - - plasma-store-server-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - plasma-store-server_{no_rc_version}-1_[a-z0-9]+.deb - - ubuntu-focal-arm64: - ci: travis - template: linux-packages/travis.linux.arm64.yml - params: - target: "ubuntu-focal-arm64" - task_namespace: "apt" - upload_extensions: - - .ddeb - - .deb - - .debian.tar.xz - - .dsc - - .orig.tar.gz - artifacts: - - apache-arrow_{no_rc_version}-1.debian.tar.xz - - apache-arrow_{no_rc_version}-1.dsc - - apache-arrow_{no_rc_version}.orig.tar.gz - - gir1.2-arrow-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-arrow-dataset-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-gandiva-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-parquet-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-flight-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-flight400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-python-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-python400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow400_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva400_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet400_{no_rc_version}-1_[a-z0-9]+.deb - - ubuntu-groovy-amd64: - ci: github - template: linux-packages/github.linux.amd64.yml - params: - target: "ubuntu-groovy" - task_namespace: "apt" - upload_extensions: - - .ddeb - - .deb - - .debian.tar.xz - - .dsc - - .orig.tar.gz - artifacts: - - apache-arrow-apt-source_{no_rc_version}-1.debian.tar.xz - - apache-arrow-apt-source_{no_rc_version}-1.dsc - - apache-arrow-apt-source_{no_rc_version}-1_all.deb - - apache-arrow-apt-source_{no_rc_version}.orig.tar.gz - - apache-arrow_{no_rc_version}-1.debian.tar.xz - - apache-arrow_{no_rc_version}-1.dsc - - apache-arrow_{no_rc_version}.orig.tar.gz - - gir1.2-arrow-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-arrow-cuda-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-gandiva-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-parquet-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-plasma-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-cuda-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-cuda-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-cuda-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-cuda400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-flight-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-flight400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-python-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-python400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow400_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva400_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet400_{no_rc_version}-1_[a-z0-9]+.deb - - libplasma-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libplasma-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libplasma-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libplasma-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libplasma400_{no_rc_version}-1_[a-z0-9]+.deb - - plasma-store-server-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - plasma-store-server_{no_rc_version}-1_[a-z0-9]+.deb - - ubuntu-groovy-arm64: - ci: travis - template: linux-packages/travis.linux.arm64.yml - params: - target: "ubuntu-groovy-arm64" - task_namespace: "apt" - upload_extensions: - - .ddeb - - .deb - - .debian.tar.xz - - .dsc - - .orig.tar.gz - artifacts: - - apache-arrow_{no_rc_version}-1.debian.tar.xz - - apache-arrow_{no_rc_version}-1.dsc - - apache-arrow_{no_rc_version}.orig.tar.gz - - gir1.2-arrow-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-arrow-dataset-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-gandiva-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - gir1.2-parquet-1.0_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-flight-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-flight400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-python-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-python400_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow400_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva400_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib400_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet400_{no_rc_version}-1_[a-z0-9]+.deb - - centos-7-amd64: - ci: github - template: linux-packages/github.linux.amd64.yml - params: - target: "centos-7" - task_namespace: yum - upload_extensions: - - .rpm - artifacts: - - apache-arrow-release-{no_rc_version}-1.el7.noarch.rpm - - apache-arrow-release-{no_rc_version}-1.el7.src.rpm - - arrow-{no_rc_version}-1.el7.src.rpm - - arrow-debuginfo-{no_rc_version}-1.el7.[a-z0-9_]+.rpm - - arrow-devel-{no_rc_version}-1.el7.[a-z0-9_]+.rpm - - arrow-glib-devel-{no_rc_version}-1.el7.[a-z0-9_]+.rpm - - arrow-glib-doc-{no_rc_version}-1.el7.[a-z0-9_]+.rpm - - arrow-glib-libs-{no_rc_version}-1.el7.[a-z0-9_]+.rpm - - arrow-libs-{no_rc_version}-1.el7.[a-z0-9_]+.rpm - - arrow-dataset-devel-{no_rc_version}-1.el7.[a-z0-9_]+.rpm - - arrow-dataset-glib-devel-{no_rc_version}-1.el7.[a-z0-9_]+.rpm - - arrow-dataset-glib-doc-{no_rc_version}-1.el7.[a-z0-9_]+.rpm - - arrow-dataset-glib-libs-{no_rc_version}-1.el7.[a-z0-9_]+.rpm - - arrow-dataset-libs-{no_rc_version}-1.el7.[a-z0-9_]+.rpm - - arrow-python-devel-{no_rc_version}-1.el7.[a-z0-9_]+.rpm - - arrow-python-libs-{no_rc_version}-1.el7.[a-z0-9_]+.rpm - - parquet-devel-{no_rc_version}-1.el7.[a-z0-9_]+.rpm - - parquet-glib-devel-{no_rc_version}-1.el7.[a-z0-9_]+.rpm - - parquet-glib-doc-{no_rc_version}-1.el7.[a-z0-9_]+.rpm - - parquet-glib-libs-{no_rc_version}-1.el7.[a-z0-9_]+.rpm - - parquet-libs-{no_rc_version}-1.el7.[a-z0-9_]+.rpm - - plasma-devel-{no_rc_version}-1.el7.[a-z0-9_]+.rpm - - plasma-glib-devel-{no_rc_version}-1.el7.[a-z0-9_]+.rpm - - plasma-glib-doc-{no_rc_version}-1.el7.[a-z0-9_]+.rpm - - plasma-glib-libs-{no_rc_version}-1.el7.[a-z0-9_]+.rpm - - plasma-libs-{no_rc_version}-1.el7.[a-z0-9_]+.rpm - - plasma-store-server-{no_rc_version}-1.el7.[a-z0-9_]+.rpm - - centos-8-amd64: - ci: github - template: linux-packages/github.linux.amd64.yml - params: - target: "centos-8" - task_namespace: yum - upload_extensions: - - .rpm - artifacts: - - apache-arrow-release-{no_rc_version}-1.el8.noarch.rpm - - apache-arrow-release-{no_rc_version}-1.el8.src.rpm - - arrow-{no_rc_version}-1.el8.src.rpm - - arrow-dataset-devel-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-dataset-glib-doc-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-dataset-glib-libs-debuginfo-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-dataset-glib-libs-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-dataset-libs-debuginfo-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-dataset-libs-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-debuginfo-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-devel-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-flight-devel-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-flight-libs-debuginfo-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-flight-libs-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-glib-devel-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-glib-doc-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-glib-libs-debuginfo-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-glib-libs-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-libs-debuginfo-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-libs-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-python-devel-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-python-flight-devel-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-python-flight-libs-debuginfo-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-python-flight-libs-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-python-libs-debuginfo-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-python-libs-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - gandiva-devel-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - gandiva-glib-devel-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - gandiva-glib-doc-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - gandiva-glib-libs-debuginfo-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - gandiva-glib-libs-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - gandiva-libs-debuginfo-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - gandiva-libs-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - parquet-devel-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - parquet-glib-devel-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - parquet-glib-doc-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - parquet-glib-libs-debuginfo-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - parquet-glib-libs-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - parquet-libs-debuginfo-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - parquet-libs-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - plasma-devel-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - plasma-glib-devel-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - plasma-glib-doc-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - plasma-glib-libs-debuginfo-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - plasma-glib-libs-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - plasma-libs-debuginfo-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - plasma-libs-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - plasma-store-server-debuginfo-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - plasma-store-server-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - centos-8-arm64: - ci: travis - template: linux-packages/travis.linux.arm64.yml - params: - target: "centos-8-aarch64" - task_namespace: yum - upload_extensions: - - .rpm - artifacts: - - arrow-{no_rc_version}-1.el8.src.rpm - - arrow-dataset-devel-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-dataset-glib-devel-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-dataset-glib-doc-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-dataset-glib-libs-debuginfo-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-dataset-glib-libs-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-dataset-libs-debuginfo-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-dataset-libs-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-debuginfo-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-devel-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-flight-devel-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-flight-libs-debuginfo-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-flight-libs-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-glib-devel-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-glib-doc-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-glib-libs-debuginfo-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-glib-libs-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-libs-debuginfo-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-libs-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-python-devel-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-python-flight-devel-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-python-flight-libs-debuginfo-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-python-flight-libs-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-python-libs-debuginfo-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - arrow-python-libs-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - parquet-devel-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - parquet-glib-devel-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - parquet-glib-doc-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - parquet-glib-libs-debuginfo-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - parquet-glib-libs-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - parquet-libs-debuginfo-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - parquet-libs-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - plasma-devel-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - plasma-glib-devel-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - plasma-glib-doc-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - plasma-glib-libs-debuginfo-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - plasma-glib-libs-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - plasma-libs-debuginfo-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - plasma-libs-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - plasma-store-server-debuginfo-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - plasma-store-server-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - - ############################## Homebrew Tasks ################################ - - homebrew-cpp: - ci: github - template: homebrew-formulae/github.macos.yml - params: - formula: apache-arrow.rb - - homebrew-cpp-autobrew: - ci: github - template: homebrew-formulae/github.macos.yml - params: - formula: autobrew/apache-arrow.rb - - homebrew-r-autobrew: - # This tests that the autobrew formula + script work in practice - ci: github - template: r/github.macos.autobrew.yml - - ############################## Gandiva Tasks ################################ - - gandiva-jar-ubuntu: - ci: github - template: gandiva-jars/github.linux.yml - artifacts: - - arrow-gandiva-{no_rc_version}-SNAPSHOT.jar - - gandiva-jar-osx: - ci: github - template: gandiva-jars/github.osx.yml - artifacts: - - arrow-gandiva-{no_rc_version}-SNAPSHOT.jar - - ############################## NuGet packages ############################### - - nuget: - ci: github - template: nuget-packages/github.linux.yml - params: - run: ubuntu-csharp - artifacts: - - Apache.Arrow.Flight.AspNetCore.{no_rc_version}.nupkg - - Apache.Arrow.Flight.AspNetCore.{no_rc_version}.snupkg - - Apache.Arrow.Flight.{no_rc_version}.nupkg - - Apache.Arrow.Flight.{no_rc_version}.snupkg - - Apache.Arrow.{no_rc_version}.nupkg - - Apache.Arrow.{no_rc_version}.snupkg - - ########################### Release verification ############################ - -{% for target in ["binary", "yum", "apt"] %} - verify-rc-binaries-{{ target }}: - ci: github - template: verify-rc/github.linux.yml - params: - env: - TEST_DEFAULT: 0 - TEST_{{ target|upper }}: 1 - artifact: "binaries" -{% endfor %} - -{% for platform in ["linux", "macos"] %} - - verify-rc-wheels-{{ platform }}: - ci: github - template: verify-rc/github.{{ platform }}.yml - params: - env: - TEST_DEFAULT: 0 - artifact: "wheels" - -{% for target in ["csharp", - "go", - "integration" - "java", - "js", - "python", - "ruby", - "rust"] %} - - verify-rc-source-{{ platform }}-{{ target }}: - ci: github - template: verify-rc/github.{{ platform }}.yml - params: - env: - INSTALL_NODE: 0 - TEST_DEFAULT: 0 - TEST_{{ target|upper }}: 1 - artifact: "source" - -{% endfor %} - -{% endfor %} - - verify-rc-source-windows: - ci: github - template: verify-rc/github.win.yml - params: - script: "verify-release-candidate.bat" - - verify-rc-wheels-windows: - ci: github - template: verify-rc/github.win.yml - params: - script: "verify-release-candidate-wheels.bat" - -{############################## Docker tests #################################} - -{% for image in ["conda-cpp", - "conda-cpp-valgrind", - "debian-c-glib", - "ubuntu-c-glib", - "debian-ruby", - "ubuntu-ruby"] %} - test-{{ image }}: - ci: github - template: docker-tests/github.linux.yml - params: - run: {{ image }} -{% endfor %} - - test-debian-10-cpp: - ci: github - template: docker-tests/github.linux.yml - params: - env: - DEBIAN: 10 - run: debian-cpp - - test-ubuntu-18.04-cpp: - ci: github - template: docker-tests/github.linux.yml - params: - env: - UBUNTU: 18.04 - run: ubuntu-cpp - - test-fedora-33-cpp: - ci: github - template: docker-tests/github.linux.yml - params: - env: - FEDORA: 33 - run: fedora-cpp - - test-ubuntu-18.04-cpp-release: - ci: github - template: docker-tests/github.linux.yml - params: - env: - UBUNTU: 18.04 - run: "-e ARROW_BUILD_TYPE=release ubuntu-cpp" - - test-ubuntu-18.04-cpp-static: - ci: github - template: docker-tests/github.linux.yml - params: - env: - UBUNTU: 18.04 - run: "-e ARROW_BUILD_SHARED=OFF -e ARROW_BUILD_STATIC=ON -e ARROW_TEST_LINKAGE=static ubuntu-cpp" - - test-ubuntu-20.04-cpp: - ci: github - template: docker-tests/github.linux.yml - params: - env: - UBUNTU: 20.04 - run: ubuntu-cpp - -{% for cpp_standard in [14, 17] %} - test-ubuntu-20.04-cpp-{{ cpp_standard }}: - ci: github - template: docker-tests/github.linux.yml - params: - env: - UBUNTU: 20.04 - run: "-e CMAKE_ARGS=-DCMAKE_CXX_STANDARD={{ cpp_standard }} ubuntu-cpp" -{% endfor %} - - test-ubuntu-20.04-cpp-thread-sanitizer: - ci: github - template: docker-tests/github.linux.yml - params: - env: - CLANG_TOOLS: 11 - UBUNTU: 20.04 - run: ubuntu-cpp-thread-sanitizer - -{% for python_version in ["3.6", "3.7", "3.8", "3.9"] %} - test-conda-python-{{ python_version }}: - ci: github - template: docker-tests/github.linux.yml - params: - env: - PYTHON: {{ python_version }} - run: conda-python -{% endfor %} - - test-conda-python-3.8-hypothesis: - ci: github - template: docker-tests/github.linux.yml - params: - env: - HYPOTHESIS_PROFILE: ci - PYARROW_TEST_HYPOTHESIS: ON - PYTHON: 3.8 - # limit to execute hypothesis tests only - PYTEST_ARGS: "-m hypothesis" - run: conda-python-pandas - - test-debian-10-python-3: - ci: azure - template: docker-tests/azure.linux.yml - params: - env: - DEBIAN: 10 - run: debian-python - - test-ubuntu-18.04-python-3: - ci: azure - template: docker-tests/azure.linux.yml - params: - env: - UBUNTU: 18.04 - run: ubuntu-python - - test-fedora-33-python-3: - ci: azure - template: docker-tests/azure.linux.yml - params: - env: - FEDORA: 33 - run: fedora-python - - test-r-linux-as-cran: - ci: github - template: r/github.linux.cran.yml - params: - MATRIX: {{ "${{ matrix.r_image }}" }} - - test-r-version-compatibility: - ci: github - template: r/github.linux.version.compatibility.yml - - test-r-versions: - ci: github - template: r/github.linux.versions.yml - params: - MATRIX: {{ "${{ matrix.r_version }}" }} - - test-r-install-local: - ci: github - template: r/github.macos-linux.local.yml - - test-r-devdocs: - ci: github - template: r/github.devdocs.yml - - test-r-rhub-ubuntu-gcc-release: - ci: azure - template: r/azure.linux.yml - params: - r_org: rhub - r_image: ubuntu-gcc-release - r_tag: latest - - test-r-rocker-r-base-latest: - ci: azure - template: r/azure.linux.yml - params: - r_org: rocker - r_image: r-base - r_tag: latest - - test-r-rstudio-r-base-3.6-bionic: - ci: azure - template: r/azure.linux.yml - params: - r_org: rstudio - r_image: r-base - r_tag: 3.6-bionic - - test-r-rstudio-r-base-3.6-centos8: - ci: azure - template: r/azure.linux.yml - params: - r_org: rstudio - r_image: r-base - r_tag: 3.6-centos8 - - test-r-rstudio-r-base-3.6-centos7-devtoolset-8: - ci: azure - template: r/azure.linux.yml - params: - r_org: rstudio - r_image: r-base - r_tag: 3.6-centos7 - devtoolset_version: 8 - - test-r-rstudio-r-base-3.6-opensuse15: - ci: azure - template: r/azure.linux.yml - params: - r_org: rstudio - r_image: r-base - r_tag: 3.6-opensuse15 - - test-r-rstudio-r-base-3.6-opensuse42: - ci: azure - template: r/azure.linux.yml - params: - r_org: rstudio - r_image: r-base - r_tag: 3.6-opensuse42 - - test-r-minimal-build: - ci: azure - template: r/azure.linux.yml - params: - r_org: rocker - r_image: r-base - r_tag: latest - arrow_dataset: "OFF" - arrow_parquet: "OFF" - arrow_s3: "OFF" - arrow_with_re2: "OFF" - arrow_with_utf8proc: "OFF" - libarrow_minimal: "TRUE" - - test-r-without-arrow: - ci: azure - template: r/azure.linux.yml - params: - r_org: rhub - r_image: ubuntu-gcc-release - r_tag: latest - libarrow_download: "FALSE" - libarrow_build: "FALSE" - with_arrow: "FALSE" - not_cran: "FALSE" - - test-ubuntu-18.04-r-sanitizer: - ci: azure - template: docker-tests/azure.linux.yml - params: - env: - UBUNTU: 18.04 - run: ubuntu-r-sanitizer - - test-debian-10-go-1.15: - ci: azure - template: docker-tests/azure.linux.yml - params: - env: - GO: 1.15 - run: debian-go - - test-ubuntu-20.10-docs: - ci: azure - template: docker-tests/azure.linux.yml - params: - env: - UBUNTU: "20.10" - run: ubuntu-docs - - ############################## vcpkg tests ################################## - - test-build-vcpkg-win: - ci: github - template: vcpkg-tests/github.windows.yml - - ############################## Integration tests ############################ - - test-conda-python-3.7-pandas-latest: - ci: github - template: docker-tests/github.linux.yml - params: - env: - PYTHON: 3.7 - PANDAS: latest - # use the latest pandas release, so prevent reusing any cached layers - run: --no-leaf-cache conda-python-pandas - - test-conda-python-3.8-pandas-latest: - ci: github - template: docker-tests/github.linux.yml - params: - env: - PYTHON: 3.8 - PANDAS: latest - # use the latest pandas release, so prevent reusing any cached layers - run: --no-leaf-cache conda-python-pandas - - test-conda-python-3.8-pandas-nightly: - ci: github - template: docker-tests/github.linux.yml - params: - env: - PYTHON: 3.8 - PANDAS: nightly - NUMPY: nightly - run: --no-leaf-cache conda-python-pandas - - test-conda-python-3.7-pandas-master: - ci: github - template: docker-tests/github.linux.yml - params: - env: - PYTHON: 3.7 - PANDAS: master - # use the master branch of pandas, so prevent reusing any cached layers - run: --no-leaf-cache conda-python-pandas - - test-conda-python-3.6-pandas-0.23: - ci: github - template: docker-tests/github.linux.yml - params: - env: - PYTHON: 3.6 - PANDAS: 0.23 - run: conda-python-pandas - - test-conda-python-3.7-dask-latest: - ci: github - template: docker-tests/github.linux.yml - params: - env: - PYTHON: 3.7 - DASK: latest - # use the latest dask release, so prevent reusing any cached layers - run: --no-leaf-cache conda-python-dask - - test-conda-python-3.8-dask-master: - ci: github - template: docker-tests/github.linux.yml - params: - env: - PYTHON: 3.8 - DASK: master - # use the master branch of dask, so prevent reusing any cached layers - run: --no-leaf-cache conda-python-dask - - test-conda-python-3.8-jpype: - ci: github - template: docker-tests/github.linux.yml - params: - env: - PYTHON: 3.8 - run: conda-python-jpype - - test-conda-python-3.7-turbodbc-latest: - ci: github - template: docker-tests/github.linux.yml - params: - env: - PYTHON: 3.7 - TURBODBC: latest - # use the latest turbodbc release, so prevent reusing any cached layers - run: --no-leaf-cache conda-python-turbodbc - - test-conda-python-3.7-turbodbc-master: - ci: github - template: docker-tests/github.linux.yml - params: - env: - PYTHON: 3.7 - TURBODBC: master - # use the master branch of dask, so prevent reusing any cached layers - run: --no-leaf-cache conda-python-turbodbc - - test-conda-python-3.7-kartothek-latest: - ci: github - template: docker-tests/github.linux.yml - params: - env: - PYTHON: 3.7 - KARTOTHEK: latest - run: --no-leaf-cache conda-python-kartothek - - test-conda-python-3.7-kartothek-master: - ci: github - template: docker-tests/github.linux.yml - params: - env: - PYTHON: 3.7 - KARTOTHEK: master - # use the master branch of kartothek, so prevent reusing any layers - run: --no-leaf-cache conda-python-kartothek - - test-conda-python-3.7-hdfs-3.2: - ci: github - template: docker-tests/github.linux.yml - params: - env: - PYTHON: 3.7 - HDFS: 3.2.1 - run: conda-python-hdfs - - test-conda-python-3.7-spark-branch-3.0: - ci: github - template: docker-tests/github.linux.yml - params: - env: - PYTHON: 3.7 - SPARK: "branch-3.0" - TEST_PYARROW_ONLY: "true" - # use the branch-3.0 of spark, so prevent reusing any layers - run: --no-leaf-cache conda-python-spark - - test-conda-python-3.8-spark-master: - ci: github - template: docker-tests/github.linux.yml - params: - env: - PYTHON: 3.8 - SPARK: master - # use the master branch of spark, so prevent reusing any layers - run: --no-leaf-cache conda-python-spark - - # Remove the "skipped-" prefix in ARROW-8475 - skipped-test-conda-cpp-hiveserver2: - ci: github - template: docker-tests/github.linux.yml - params: - run: conda-cpp-hiveserver2 - - example-cpp-minimal-build-static: - ci: github - template: cpp-examples/github.linux.yml - params: - type: minimal_build - run: static - - example-cpp-minimal-build-static-system-dependency: - ci: github - template: cpp-examples/github.linux.yml - params: - type: minimal_build - run: static-system-dependency diff --git a/dev/tasks/vcpkg-tests/cpp-build-vcpkg.bat b/dev/tasks/vcpkg-tests/cpp-build-vcpkg.bat deleted file mode 100644 index f748f92f3bcf..000000000000 --- a/dev/tasks/vcpkg-tests/cpp-build-vcpkg.bat +++ /dev/null @@ -1,90 +0,0 @@ -@rem Licensed to the Apache Software Foundation (ASF) under one -@rem or more contributor license agreements. See the NOTICE file -@rem distributed with this work for additional information -@rem regarding copyright ownership. The ASF licenses this file -@rem to you under the Apache License, Version 2.0 (the -@rem "License"); you may not use this file except in compliance -@rem with the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, -@rem software distributed under the License is distributed on an -@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -@rem KIND, either express or implied. See the License for the -@rem specific language governing permissions and limitations -@rem under the License. - -@rem Run VsDevCmd.bat to set Visual Studio environment variables for building -@rem on the command line. This is the path for Visual Studio Enterprise 2019 - -call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=amd64 - - -@rem Install build dependencies with vcpkg - -@rem TODO(ianmcook): change --x-manifest-root to --manifest-root after it -@rem changes in vcpkg - -vcpkg install ^ - --triplet x64-windows ^ - --x-manifest-root cpp ^ - --clean-after-build ^ - || exit /B 1 - - -@rem Set environment variables - -set ARROW_TEST_DATA=%cd%\testing\data -set PARQUET_TEST_DATA=%cd%\cpp\submodules\parquet-testing\data - - -@rem Build Arrow C++ library - -mkdir cpp\build -pushd cpp\build - -@rem TODO(ianmcook): test using --parallel %NUMBER_OF_PROCESSORS% with -@rem cmake --build instead of specifying -DARROW_CXXFLAGS="/MP" here -@rem (see https://gitlab.kitware.com/cmake/cmake/-/issues/20564) - -@rem TODO(ianmcook): Add -DARROW_BUILD_BENCHMARKS=ON after the issue described -@rem at https://github.com/google/benchmark/issues/1046 is resolved - -cmake -G "Visual Studio 16 2019" -A x64 ^ - -DARROW_BOOST_USE_SHARED=ON ^ - -DARROW_BUILD_SHARED=ON ^ - -DARROW_BUILD_STATIC=OFF ^ - -DARROW_BUILD_TESTS=ON ^ - -DARROW_CXXFLAGS="/MP" ^ - -DARROW_DATASET=ON ^ - -DARROW_DEPENDENCY_SOURCE=VCPKG ^ - -DARROW_FLIGHT=ON ^ - -DARROW_MIMALLOC=ON ^ - -DARROW_PARQUET=ON ^ - -DARROW_PYTHON=OFF ^ - -DARROW_WITH_BROTLI=ON ^ - -DARROW_WITH_BZ2=ON ^ - -DARROW_WITH_LZ4=ON ^ - -DARROW_WITH_SNAPPY=ON ^ - -DARROW_WITH_ZLIB=ON ^ - -DARROW_WITH_ZSTD=ON ^ - -DCMAKE_BUILD_TYPE=release ^ - -DCMAKE_UNITY_BUILD=ON ^ - .. || exit /B 1 - -cmake --build . --target INSTALL --config Release || exit /B 1 - - -@rem Test Arrow C++ library - -@rem TODO(ARROW-11675): Uncomment the below -@rem and troubleshoot two test failures: -@rem - TestStatisticsSortOrder/0.MinMax -@rem - TestStatistic.Int32Extremums - -@rem ctest --output-on-failure ^ -@rem --parallel %NUMBER_OF_PROCESSORS% ^ -@rem --timeout 300 || exit /B 1 - -popd diff --git a/dev/tasks/vcpkg-tests/github.windows.yml b/dev/tasks/vcpkg-tests/github.windows.yml deleted file mode 100644 index eacb6317c303..000000000000 --- a/dev/tasks/vcpkg-tests/github.windows.yml +++ /dev/null @@ -1,63 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# NOTE: must set "Crossbow" as name to have the badge links working in the -# github comment reports! -name: Crossbow - -on: - push: - branches: - - "*-github-*" - -jobs: - test-vcpkg-win: - name: Install build deps with vcpkg and build Arrow C++ - runs-on: windows-2019 - steps: - - name: Checkout Arrow - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive - - name: Remove and Reinstall vcpkg - # As of January 2021, the version of vcpkg that is preinstalled on the - # Github Actions windows-2019 image is 2020.11.12, as noted at - # https://github.com/actions/virtual-environments/blob/main/images/win/Windows2019-Readme.md - # This version of vcpkg has a bug that causes the installation of - # aws-cpp-sdk to fail. See details at - # https://github.com/awslabs/aws-c-common/issues/734 - # and https://github.com/microsoft/vcpkg/pull/14716. - # When running vcpkg in Github Actions on Windows, remove the - # preinstalled vcpkg and install the newest version from source. - shell: cmd - run: | - CALL vcpkg integrate remove 2>NUL - CALL C: - CALL cd \ - CALL rmdir /s /q vcpkg 2>NUL - CALL git clone https://github.com/microsoft/vcpkg.git vcpkg - CALL cd vcpkg - CALL bootstrap-vcpkg.bat -win64 -disableMetrics - CALL vcpkg integrate install - CALL setx PATH "%PATH%;C:\vcpkg" - - name: Install Dependencies with vcpkg and Build Arrow C++ - shell: cmd - run: | - CALL cd arrow - CALL dev\tasks\vcpkg-tests\cpp-build-vcpkg.bat diff --git a/dev/tasks/verify-rc/github.linux.yml b/dev/tasks/verify-rc/github.linux.yml deleted file mode 100644 index 8729426fd134..000000000000 --- a/dev/tasks/verify-rc/github.linux.yml +++ /dev/null @@ -1,75 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -{% import 'macros.jinja' as macros with context %} - -{{ macros.github_header() }} - -jobs: - verify: - name: "Verify release candidate Ubuntu {{ artifact }}" - runs-on: ubuntu-20.04 - {% if env is defined %} - env: - {% for key, value in env.items() %} - {{ key }}: {{ value }} - {% endfor %} - {% endif %} - - steps: - {{ macros.github_checkout_arrow()|indent }} - - - name: Install System Dependencies - run: | - # TODO: don't require removing newer llvms - sudo apt-get --purge remove -y llvm-9 clang-9 - sudo apt-get install -y \ - autoconf-archive \ - binfmt-support \ - bison \ - curl \ - flex \ - gtk-doc-tools \ - jq \ - libboost-all-dev \ - libgirepository1.0-dev \ - qemu-user-static \ - wget - - if [ "$TEST_JAVA" = "1" ]; then - # Maven - MAVEN_VERSION=3.6.3 - wget https://downloads.apache.org/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.zip - unzip apache-maven-$MAVEN_VERSION-bin.zip - mkdir -p $HOME/java - mv apache-maven-$MAVEN_VERSION $HOME/java - export PATH=$HOME/java/apache-maven-$MAVEN_VERSION/bin:$PATH - fi - - if [ "$TEST_RUBY" = "1" ]; then - ruby --version - sudo gem install bundler - fi - - uses: actions/setup-node@v2-beta - with: - node-version: '14' - - name: Run verification - shell: bash - run: | - arrow/dev/release/verify-release-candidate.sh \ - {{ artifact }} \ - {{ release|default("1.0.0") }} {{ rc|default("0") }} diff --git a/dev/tasks/verify-rc/github.macos.yml b/dev/tasks/verify-rc/github.macos.yml deleted file mode 100644 index ab0c6563bdc0..000000000000 --- a/dev/tasks/verify-rc/github.macos.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -{% import 'macros.jinja' as macros with context %} - -{{ macros.github_header() }} - -jobs: - verify: - name: "Verify release candidate macOS {{ artifact }}" - runs-on: macos-latest - {% if env is defined %} - env: - {% for key, value in env.items() %} - {{ key }}: {{ value }} - {% endfor %} - {% endif %} - - steps: - {{ macros.github_checkout_arrow()|indent }} - - - name: Install System Dependencies - shell: bash - run: | - brew update - brew bundle --file=arrow/cpp/Brewfile - brew bundle --file=arrow/c_glib/Brewfile - - uses: actions/setup-node@v2-beta - with: - node-version: '14' - - name: Run verification - shell: bash - run: | - arrow/dev/release/verify-release-candidate.sh \ - {{ artifact }} \ - {{ release|default("1.0.0") }} {{ rc|default("0") }} diff --git a/dev/tasks/verify-rc/github.win.yml b/dev/tasks/verify-rc/github.win.yml deleted file mode 100644 index 5406327e8741..000000000000 --- a/dev/tasks/verify-rc/github.win.yml +++ /dev/null @@ -1,45 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -{% import 'macros.jinja' as macros with context %} - -{{ macros.github_header() }} - -jobs: - verify: - name: "Verify release candidate Windows source" - runs-on: windows-2016 - {% if env is defined %} - env: - {% for key, value in env.items() %} - {{ key }}: {{ value }} - {% endfor %} - {% endif %} - - steps: - {{ macros.github_checkout_arrow()|indent }} - - - uses: conda-incubator/setup-miniconda@v2 - - name: Install System Dependencies - run: | - choco install boost-msvc-14.1 - choco install wget - - name: Run verification - shell: cmd - run: | - cd arrow - dev/release/{{ script }} {{ release|default("1.0.0") }} {{ rc|default("0") }} From f52763dc3705eced7f1762a102f2f09220dba542 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sat, 15 May 2021 07:47:26 -0400 Subject: [PATCH 2/2] Add rat files back in --- dev/release/rat_exclude_files.txt | 107 ++++++++++++++++++++++++++++++ dev/release/run-rat.sh | 43 ++++++++++++ 2 files changed, 150 insertions(+) create mode 100644 dev/release/rat_exclude_files.txt create mode 100755 dev/release/run-rat.sh diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt new file mode 100644 index 000000000000..f3eb273adff4 --- /dev/null +++ b/dev/release/rat_exclude_files.txt @@ -0,0 +1,107 @@ +*.npmrc +*.gitignore +.gitmodules +*_generated.h +*_generated.js +*_generated.ts +*.csv +*.json +*.snap +.github/ISSUE_TEMPLATE/*.md +.github/pull_request_template.md +ci/etc/rprofile +ci/etc/*.patch +ci/vcpkg/*.patch +CHANGELOG.md +dev/requirements*.txt +dev/archery/MANIFEST.in +dev/archery/requirements*.txt +dev/archery/archery/tests/fixtures/* +dev/archery/archery/crossbow/tests/fixtures/* +dev/release/rat_exclude_files.txt +dev/tasks/homebrew-formulae/apache-arrow.rb +dev/tasks/linux-packages/apache-arrow-apt-source/debian/apache-arrow-apt-source.install +dev/tasks/linux-packages/apache-arrow-apt-source/debian/compat +dev/tasks/linux-packages/apache-arrow-apt-source/debian/control +dev/tasks/linux-packages/apache-arrow-apt-source/debian/rules +dev/tasks/linux-packages/apache-arrow-apt-source/debian/source/format +dev/tasks/linux-packages/apache-arrow/debian/compat +dev/tasks/linux-packages/apache-arrow/debian/control.in +dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-1.0.install +dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-cuda-1.0.install +dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-dataset-1.0.install +dev/tasks/linux-packages/apache-arrow/debian/gir1.2-gandiva-1.0.install +dev/tasks/linux-packages/apache-arrow/debian/gir1.2-parquet-1.0.install +dev/tasks/linux-packages/apache-arrow/debian/gir1.2-plasma-1.0.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-dev.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.doc-base +dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.links +dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib400.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-dev.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib-dev.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib400.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda400.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-dev.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-dev.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.doc-base +dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.links +dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib400.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset400.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-dev.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight400.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-dev.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight-dev.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight400.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-python400.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow400.install +dev/tasks/linux-packages/apache-arrow/debian/libgandiva-dev.install +dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-dev.install +dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.doc-base +dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.install +dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.links +dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib400.install +dev/tasks/linux-packages/apache-arrow/debian/libgandiva400.install +dev/tasks/linux-packages/apache-arrow/debian/libparquet-dev.install +dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-dev.install +dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.doc-base +dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.install +dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.links +dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib400.install +dev/tasks/linux-packages/apache-arrow/debian/libparquet400.install +dev/tasks/linux-packages/apache-arrow/debian/libplasma-dev.install +dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-dev.install +dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.doc-base +dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.install +dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.links +dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib400.install +dev/tasks/linux-packages/apache-arrow/debian/libplasma400.install +dev/tasks/linux-packages/apache-arrow/debian/patches/series +dev/tasks/linux-packages/apache-arrow/debian/plasma-store-server.install +dev/tasks/linux-packages/apache-arrow/debian/rules +dev/tasks/linux-packages/apache-arrow/debian/source/format +dev/tasks/linux-packages/apache-arrow/debian/watch +dev/tasks/requirements*.txt +dev/tasks/conda-recipes/* +pax_global_header +MANIFEST.in +__init__.pxd +__init__.py +requirements.txt +*.html +*.sgml +*.css +*.png +*.ico +*.svg +*.devhelp2 +*.scss +.gitattributes +arrow/test/data/*.csv +rust-toolchain +arrow-flight/src/arrow.flight.protocol.rs +ballista/rust/benchmarks/tpch/queries/q*.sql +ballista/rust/scheduler/testdata/* +ballista/ui/scheduler/yarn.lock diff --git a/dev/release/run-rat.sh b/dev/release/run-rat.sh new file mode 100755 index 000000000000..94fa55fbe097 --- /dev/null +++ b/dev/release/run-rat.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +RAT_VERSION=0.13 + +# download apache rat +if [ ! -f apache-rat-${RAT_VERSION}.jar ]; then + curl -s https://repo1.maven.org/maven2/org/apache/rat/apache-rat/${RAT_VERSION}/apache-rat-${RAT_VERSION}.jar > apache-rat-${RAT_VERSION}.jar +fi + +RAT="java -jar apache-rat-${RAT_VERSION}.jar -x " + +RELEASE_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd) + +# generate the rat report +$RAT $1 > rat.txt +python $RELEASE_DIR/check-rat-report.py $RELEASE_DIR/rat_exclude_files.txt rat.txt > filtered_rat.txt +cat filtered_rat.txt +UNAPPROVED=`cat filtered_rat.txt | grep "NOT APPROVED" | wc -l` + +if [ "0" -eq "${UNAPPROVED}" ]; then + echo "No unapproved licenses" +else + echo "${UNAPPROVED} unapproved licences. Check rat report: rat.txt" + exit 1 +fi