From 30ad26e705f50442f05dd579990372196323fc86 Mon Sep 17 00:00:00 2001 From: Andrey Anshin Date: Fri, 27 Jan 2023 16:52:53 +0400 Subject: [PATCH] Fix integration with `codecov` (#29145) --- .coveragerc | 27 +++++++++++++------ .github/actions/codecov-action | 1 - .github/actions/post_tests/action.yml | 20 ++++++++++----- .github/workflows/ci.yml | 37 --------------------------- .gitmodules | 3 --- Dockerfile.ci | 5 ++-- codecov.yml | 15 ++++++----- scripts/docker/entrypoint_ci.sh | 5 ++-- scripts/in_container/run_ci_tests.sh | 1 - 9 files changed, 46 insertions(+), 68 deletions(-) delete mode 160000 .github/actions/codecov-action diff --git a/.coveragerc b/.coveragerc index cdcc55d446971..a26f16a53bcb1 100644 --- a/.coveragerc +++ b/.coveragerc @@ -16,14 +16,25 @@ # specific language governing permissions and limitations # under the License. -[report] +[run] +branch = True +relative_files = True +source = airflow omit = - tests/* - scripts/* - dev/* - airflow/migrations/* - airflow/www/node_modules/** airflow/_vendor/** + airflow/contrib/** + airflow/example_dags/** + airflow/migrations/** + airflow/providers/**/example_dags/** + airflow/www/node_modules/** -[run] -relative_files = True +[report] +skip_empty = True +exclude_lines = + pragma: no cover + @abstractmethod + @abstractproperty + def __repr__ + raise NotImplementedError + if __name__ == .__main__.: + if TYPE_CHECKING: diff --git a/.github/actions/codecov-action b/.github/actions/codecov-action deleted file mode 160000 index 1fc7722ded470..0000000000000 --- a/.github/actions/codecov-action +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 1fc7722ded4708880a5aea49f2bfafb9336f0c8d diff --git a/.github/actions/post_tests/action.yml b/.github/actions/post_tests/action.yml index 96bed9211bb5b..b7c60aef3b630 100644 --- a/.github/actions/post_tests/action.yml +++ b/.github/actions/post_tests/action.yml @@ -35,19 +35,25 @@ runs: name: container-logs-${{env.JOB_ID}} path: "./files/container_logs*" retention-days: 7 - - name: "Upload artifact for coverage" - uses: actions/upload-artifact@v3 - if: env.COVERAGE == 'true' - with: - name: coverage-${{env.JOB_ID}} - path: ./files/coverage*.xml - retention-days: 7 - name: "Upload artifact for warnings" uses: actions/upload-artifact@v3 with: name: test-warnings-${{env.JOB_ID}} path: ./files/warnings-*.txt retention-days: 7 + - name: "Move coverage artifacts in separate directory" + if: env.COVERAGE == 'true' && env.TEST_TYPES != 'Helm' + shell: bash + run: | + mkdir ./files/coverage-reposts + mv ./files/coverage*.xml ./files/coverage-reposts/ || true + - name: "Upload all coverage reports to codecov" + uses: codecov/codecov-action@v3 + if: env.COVERAGE == 'true' && env.TEST_TYPES != 'Helm' + with: + name: coverage-${{env.JOB_ID}} + flags: python-${{env.PYTHON_MAJOR_MINOR_VERSION}},${{env.BACKEND}}-${{env.BACKEND_VERSION}} + directory: "./files/coverage-reposts/" - name: "Fix ownership" shell: bash run: breeze ci fix-ownership diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fe4c3cb9fa54d..26a5562fa24ae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1138,43 +1138,6 @@ jobs: Post Tests: ${{needs.build-info.outputs.default-python-version}}:Quarantined" uses: ./.github/actions/post_tests - upload-coverage: - timeout-minutes: 15 - name: "Upload coverage" - runs-on: "${{needs.build-info.outputs.runs-on}}" - continue-on-error: true - needs: - - build-info - - tests-postgres - - tests-sqlite - - tests-mysql - - tests-mssql - - tests-quarantined - - tests-integration-postgres - - tests-integration-mysql - env: - RUNS_ON: "${{needs.build-info.outputs.runs-on}}" - # Only upload coverage on merges to main - if: needs.build-info.outputs.run-coverage == 'true' - steps: - - name: Cleanup repo - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v3 - with: - persist-credentials: false - submodules: recursive - - name: "Download all artifacts from the current build" - uses: actions/download-artifact@v3 - with: - path: ./coverage-files - - name: "Removes unnecessary artifacts" - run: ls ./coverage-files | grep -v coverage | xargs rm -rfv - - name: "Upload all coverage reports to codecov" - uses: ./.github/actions/codecov-action - with: - directory: "./coverage-files" - summarize-warnings: timeout-minutes: 15 name: "Summarize warnings" diff --git a/.gitmodules b/.gitmodules index aa1358f88496d..4195e904ce24b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,9 +4,6 @@ [submodule ".github/actions/configure-aws-credentials"] path = .github/actions/configure-aws-credentials url = https://github.com/aws-actions/configure-aws-credentials -[submodule ".github/actions/codecov-action"] - path = .github/actions/codecov-action - url = https://github.com/codecov/codecov-action [submodule ".github/actions/github-push-action"] path = .github/actions/github-push-action url = https://github.com/ad-m/github-push-action diff --git a/Dockerfile.ci b/Dockerfile.ci index 6511d8f776df9..76afec03213b5 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -866,9 +866,10 @@ EXTRA_PYTEST_ARGS=( if [[ "${TEST_TYPE}" == "Helm" ]]; then _cpus="$(grep -c 'cpu[0-9]' /proc/stat)" echo "Running tests with ${_cpus} CPUs in parallel" - # Enable parallelism + # Enable parallelism and disable coverage EXTRA_PYTEST_ARGS+=( "-n" "${_cpus}" + "--no-cov" ) else EXTRA_PYTEST_ARGS+=( @@ -878,7 +879,7 @@ fi if [[ ${ENABLE_TEST_COVERAGE:="false"} == "true" ]]; then EXTRA_PYTEST_ARGS+=( - "--cov=airflow/" + "--cov=airflow" "--cov-config=.coveragerc" "--cov-report=xml:/files/coverage-${TEST_TYPE/\[*\]/}-${BACKEND}.xml" ) diff --git a/codecov.yml b/codecov.yml index a16f547cb4206..3fb24d7099110 100644 --- a/codecov.yml +++ b/codecov.yml @@ -81,12 +81,13 @@ coverage: only_pulls: false paths: - "airflow" -parsers: - gcov: - branch_detection: - conditional: true - loop: true - method: false - macro: false + +ignore: + - "airflow/_vendor" + - "airflow/contrib" + - "airflow/example_dags" + - "airflow/migrations" + - "airflow/providers/**/example_dags" + - "airflow/www/node_modules" comment: false diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh index 2e06d1d4ef55d..85bf24e009e2c 100755 --- a/scripts/docker/entrypoint_ci.sh +++ b/scripts/docker/entrypoint_ci.sh @@ -316,9 +316,10 @@ EXTRA_PYTEST_ARGS=( if [[ "${TEST_TYPE}" == "Helm" ]]; then _cpus="$(grep -c 'cpu[0-9]' /proc/stat)" echo "Running tests with ${_cpus} CPUs in parallel" - # Enable parallelism + # Enable parallelism and disable coverage EXTRA_PYTEST_ARGS+=( "-n" "${_cpus}" + "--no-cov" ) else EXTRA_PYTEST_ARGS+=( @@ -328,7 +329,7 @@ fi if [[ ${ENABLE_TEST_COVERAGE:="false"} == "true" ]]; then EXTRA_PYTEST_ARGS+=( - "--cov=airflow/" + "--cov=airflow" "--cov-config=.coveragerc" "--cov-report=xml:/files/coverage-${TEST_TYPE/\[*\]/}-${BACKEND}.xml" ) diff --git a/scripts/in_container/run_ci_tests.sh b/scripts/in_container/run_ci_tests.sh index 963375d77645d..c8eb4c05c5a5f 100755 --- a/scripts/in_container/run_ci_tests.sh +++ b/scripts/in_container/run_ci_tests.sh @@ -45,7 +45,6 @@ fi set +x if [[ "${RES}" == "0" && ( ${CI:="false"} == "true" || ${CI} == "True" ) ]]; then echo "All tests successful" - cp .coverage /files fi MAIN_GITHUB_REPOSITORY="apache/airflow"