diff --git a/.github/workflows/add_dependabot_pr_to_mq.yml b/.github/workflows/add_dependabot_pr_to_mq.yml new file mode 100644 index 0000000000000..14e0bda85fc67 --- /dev/null +++ b/.github/workflows/add_dependabot_pr_to_mq.yml @@ -0,0 +1,71 @@ +--- +name: Add dependabot PR to the Merge Queue +on: + pull_request_review: + types: + - submitted + - edited + +permissions: {} +jobs: + add_to_merge_queue: + if: github.event.pull_request.user.login == 'dependabot[bot]' + runs-on: ubuntu-latest + + steps: + # Use a token as only the github App can push to the merge queue + - uses: actions/create-github-app-token@c1a285145b9d317df6ced56c09f525b5c2b6f755 # v1.11.1 + id: app-token + with: + app-id: ${{ vars.DD_GITHUB_TOKEN_GENERATOR_APP_ID }} + private-key: ${{ secrets.DD_GITHUB_TOKEN_GENERATOR_PRIVATE_KEY }} + - name: Check if the PR is mergeable + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + id: check-mergeable + with: + github-token: ${{ steps.app-token.outputs.token }} + script: | + const pullRequestNumber = context.payload.pull_request.number; + const { data: pullRequest } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: pullRequestNumber + }); + const { data: reviews } = await github.rest.pulls.listReviews({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: pullRequestNumber + }); + + // Users can have several reviews, which are listed in chronological order: we use a map to keep the last review state. + let reviewers = new Map(); + for (const review of reviews) { + reviewers.set(review.user.login, review.state); + } + let allApproved = true; + for (const [reviewer, state] of reviewers) { + if (state === 'CHANGES_REQUESTED') { + allApproved = false; + break; + } + } + // When a required reviewer approves, the team is removed from the requested_teams list. + // As such, a mergeable PR has no more requested teams and no changes requested in its reviews. + return `${allAproved && pullRequest.requested_teams.length === 0}`; + result-encoding: string + - name: Add Merge Comment to Pull Request + if: ${{ steps.check-mergeable.outputs.result == 'true' }} + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + github-token: ${{ steps.app-token.outputs.token }} + script: | + const pullRequestNumber = context.payload.pull_request.number; + const commentBody = "/merge"; + + // Add a comment to the pull request + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pullRequestNumber, + body: commentBody + }); diff --git a/.github/workflows/ask_dependabot_pr_review.yml b/.github/workflows/ask_dependabot_pr_review.yml new file mode 100644 index 0000000000000..8d78e81d0bec4 --- /dev/null +++ b/.github/workflows/ask_dependabot_pr_review.yml @@ -0,0 +1,37 @@ +--- +name: "Add reviewers and ask review for dependabot PR" + +on: + pull_request: + types: [opened, synchronize, reopened] + branches: + - main + +permissions: {} +jobs: + add_reviewers: + if: github.event.pull_request.user.login == 'dependabot[bot]' + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + + - name: Setup python + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + python-version-file: .python-version + cache: 'pip' + cache-dependency-path: '**/requirements*.txt' + + - name: Install dependencies + run: pip install -r requirements.txt -r tasks/requirements.txt + + - name: Add reviewers and ask review + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: inv -e issue.add-reviewers -p $PR_NUMBER diff --git a/.github/workflows/warn_failed_dependabot_pr.yml b/.github/workflows/warn_failed_dependabot_pr.yml new file mode 100644 index 0000000000000..45d3e08940ea5 --- /dev/null +++ b/.github/workflows/warn_failed_dependabot_pr.yml @@ -0,0 +1,29 @@ +--- +name: Warn Failed Dependabot PR + +on: + issue_comment: + types: + - created + - edited + +jobs: + check_comment: + if: github.actor == 'dd-devflow[bot]' + runs-on: ubuntu-latest + steps: + - name: Check for error in comment + id: check-comment + env: + PR_BODY: ${{ github.event.comment.body }} + run: | + if echo "$PR_BODY" | grep -iE "(blocked|cancelled|conflicts|draft|error|failed|failing|timeout|unqueued|updated)"; then + echo "not_merged=true" >> $GITHUB_OUTPUT + else + echo "not_merged=false" >> $GITHUB_OUTPUT + fi + - name: Contact agent-devx + if: ${{ steps.check-comment.outputs.not_merged == 'true' }} + run: | + message="Hi!\nThis dependabot PR ${{github.event.issue.html_url}} was not merged.\nPlease have a look." + curl -X POST -H "Content-Type: application/json" --data '{"message": "$message"}' ${{ secrets.SLACK_AGENT_DEVX_WEBHOOK }} diff --git a/tasks/issue.py b/tasks/issue.py index c9f3dea146f7f..98770d73251c5 100644 --- a/tasks/issue.py +++ b/tasks/issue.py @@ -1,11 +1,13 @@ import os import random +import re from invoke import task from tasks.libs.ciproviders.github_api import GithubAPI, ask_review_actor from tasks.libs.issue.assign import assign_with_model, assign_with_rules from tasks.libs.issue.model.actions import fetch_data_and_train_model +from tasks.libs.owners.parsing import search_owners from tasks.libs.pipeline.notifications import GITHUB_SLACK_MAP, GITHUB_SLACK_REVIEW_MAP, HELP_SLACK_CHANNEL @@ -69,3 +71,51 @@ def ask_reviews(_, pr_id): except Exception as e: message = f"An error occurred while sending a review message from {actor} for PR <{pr.html_url}/s|{pr.title}> to channel {channel}. Error: {e}" client.chat_postMessage(channel='#agent-devx-ops', text=message) + + +@task +def add_reviewers(ctx, pr_id): + """ + Add team labels and reviewers to a dependabot bump PR based on the changed dependencies + """ + + gh = GithubAPI() + pr = gh.repo.get_pull(int(pr_id)) + + if pr.user.login != "dependabot[bot]": + print("This is not a (dependabot) bump PR, this action should not be run on it.") + return + + folder = "" + if pr.title.startswith("Bump the "): + match = re.match(r"^Bump the (\S+) group (.*$)", pr.title) + if match.group(2).startswith("in"): + match_folder = re.match(r"^in (\S+).*$", match.group(2)) + folder = match_folder.group(1).removeprefix("/") + else: + match = re.match(r"^Bump (\S+) from (\S+) to (\S+)$", pr.title) + dependency = match.group(1) + + # Find the responsible person for each file + owners = set() + git_files = ctx.run("git ls-files | grep -e \"^.*.go$\"", hide=True).stdout + for file in git_files.splitlines(): + if not file.startswith(folder): + continue + in_import = False + with open(file) as f: + for line in f: + # Look for the import block + if "import (" in line: + in_import = True + if in_import: + # Early exit at the end of the import block + if ")" in line: + break + else: + if dependency in line: + owners.update(set(search_owners(file, ".github/CODEOWNERS"))) + break + # Teams are added by slug, so we need to remove the @DataDog/ prefix + pr.create_review_request(team_reviewers=[owner.casefold().removeprefix("@datadog/") for owner in owners]) + pr.add_to_labels("ask-review") diff --git a/tasks/libs/owners/parsing.py b/tasks/libs/owners/parsing.py index d4b0127bf83f6..214fbd3e773d9 100644 --- a/tasks/libs/owners/parsing.py +++ b/tasks/libs/owners/parsing.py @@ -1,9 +1,11 @@ from __future__ import annotations from collections import Counter +from functools import lru_cache from typing import Any +@lru_cache def read_owners(owners_file: str, remove_default_pattern=False) -> Any: """ - remove_default_pattern: If True, will remove the '*' entry diff --git a/tasks/unit_tests/issue_tests.py b/tasks/unit_tests/issue_tests.py index 1e5cf2203d3db..e35eaf2b7b17a 100644 --- a/tasks/unit_tests/issue_tests.py +++ b/tasks/unit_tests/issue_tests.py @@ -1,6 +1,9 @@ import unittest -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch +from invoke.context import MockContext, Result + +from tasks.issue import add_reviewers from tasks.libs.issue.assign import guess_from_keywords, guess_from_labels @@ -34,3 +37,167 @@ def test_with_a_file(self): def test_no_match(self): issue = MagicMock(title="fix bug", body="It comes from the file... hm I don't know.") self.assertEqual(guess_from_keywords(issue), "triage") + + +class TestAddReviewers(unittest.TestCase): + @patch('builtins.print') + @patch('tasks.issue.GithubAPI') + def test_dependabot_only(self, gh_mock, print_mock): + pr_mock = MagicMock() + pr_mock.user.login = "InvisibleMan" + gh_mock.repo.get_pull.return_value = pr_mock + c = MockContext() + add_reviewers(c, 1234) + print_mock.assert_called_once_with("This is not a (dependabot) bump PR, this action should not be run on it.") + + @patch('builtins.print') + @patch('tasks.issue.GithubAPI') + def test_single_dependency_one_reviewer(self, gh_mock, print_mock): + pr_mock = MagicMock() + pr_mock.user.login = "dependabot[bot]" + pr_mock.title = "Bump github.com/redis/go-redis/v9 from 9.1.0 to 9.7.0" + gh_instance = MagicMock() + gh_instance.repo.get_pull.return_value = pr_mock + gh_mock.return_value = gh_instance + c = MockContext( + run={ + "git ls-files | grep -e \"^.*.go$\"": Result("""pkg/network/protocols/redis/client.go +pkg/network/usm/tests/tracer_usm_linux_test.go +""") + } + ) + add_reviewers(c, 1234) + print_mock.assert_not_called() + pr_mock.create_review_request.assert_called_once_with(team_reviewers=['universal-service-monitoring']) + + @patch('builtins.print') + @patch('tasks.issue.GithubAPI') + def test_single_dependency_several_reviewers(self, gh_mock, print_mock): + pr_mock = MagicMock() + pr_mock.user.login = "dependabot[bot]" + pr_mock.title = "Bump github.com/go-delve/delve from 1.6.0 to 1.7.0" + gh_instance = MagicMock() + gh_instance.repo.get_pull.return_value = pr_mock + gh_mock.return_value = gh_instance + c = MockContext( + run={ + "git ls-files | grep -e \"^.*.go$\"": Result("""generate_tools.go +pkg/dynamicinstrumentation/diconfig/dwarf.go +pkg/network/go/asmscan/scan.go +pkg/network/go/bininspect/dwarf.go +pkg/network/go/bininspect/newproc.go +pkg/network/go/bininspect/types.go +pkg/network/go/bininspect/utils.go +pkg/network/go/dwarfutils/compile_unit.go +pkg/network/go/dwarfutils/locexpr/exec.go +pkg/network/go/dwarfutils/type_finder.go +pkg/network/go/goid/goid_offset.go +pkg/network/go/goversion/version.go +pkg/network/go/lutgen/run.go +pkg/network/protocols/http/gotls/lookup/luts.go""") + } + ) + add_reviewers(c, 1234) + print_mock.assert_not_called() + self.assertCountEqual( + pr_mock.create_review_request.call_args[1]['team_reviewers'], + ['universal-service-monitoring', 'debugger', 'agent-devx-infra'], + ) + + @patch('builtins.print') + @patch('tasks.issue.GithubAPI') + def test_group_dependency(self, gh_mock, print_mock): + pr_mock = MagicMock() + pr_mock.user.login = "dependabot[bot]" + pr_mock.title = "Bump the aws-sdk-go-v2 group with 5 updates" + gh_instance = MagicMock() + gh_instance.repo.get_pull.return_value = pr_mock + gh_mock.return_value = gh_instance + c = MockContext( + run={ + "git ls-files | grep -e \"^.*.go$\"": Result("""pkg/databasemonitoring/aws/aurora.go +pkg/databasemonitoring/aws/aurora_test.go +pkg/databasemonitoring/aws/client.go +pkg/databasemonitoring/aws/rdsclient_mockgen.go +pkg/serverless/apikey/api_key.go +pkg/serverless/apikey/api_key_test.go +pkg/serverless/trace/inferredspan/propagation_test.go +pkg/serverless/trace/propagation/carriers_test.go +pkg/serverless/trace/propagation/extractor_test.go +pkg/serverless/trigger/extractor.go +pkg/util/ec2/ec2_tags.go +test/new-e2e/examples/ecs_test.go +test/new-e2e/go.mod +test/new-e2e/pkg/provisioners/aws/kubernetes/kubernetes_dump.go +test/new-e2e/pkg/runner/parameters/store_aws.go +test/new-e2e/pkg/utils/clients/aws.go +test/new-e2e/pkg/utils/e2e/client/ecs/ecs.go +test/new-e2e/pkg/utils/e2e/client/ecs/session-manager-plugin.go +test/new-e2e/tests/containers/ecs_test.go +test/new-e2e/tests/windows/common/pipeline/pipeline.go""") + } + ) + add_reviewers(c, 1234) + print_mock.assert_not_called() + self.assertCountEqual( + pr_mock.create_review_request.call_args[1]['team_reviewers'], + [ + 'windows-agent', + 'database-monitoring', + 'container-integrations', + 'agent-devx-loops', + 'serverless', + 'container-platform', + 'windows-kernel-integrations', + 'agent-runtimes', + 'agent-e2e-testing', + 'serverless-aws', + ], + ) + + @patch('builtins.print') + @patch('tasks.issue.GithubAPI') + def test_group_dependency_scoped(self, gh_mock, print_mock): + pr_mock = MagicMock() + pr_mock.user.login = "dependabot[bot]" + pr_mock.title = "Bump the aws-sdk-go-v2 group in /test/new-e2e with 5 updates" + gh_instance = MagicMock() + gh_instance.repo.get_pull.return_value = pr_mock + gh_mock.return_value = gh_instance + c = MockContext( + run={ + "git ls-files | grep -e \"^.*.go$\"": Result("""pkg/databasemonitoring/aws/aurora.go +pkg/databasemonitoring/aws/aurora_test.go +pkg/databasemonitoring/aws/client.go +pkg/databasemonitoring/aws/rdsclient_mockgen.go +pkg/serverless/apikey/api_key.go +pkg/serverless/apikey/api_key_test.go +pkg/serverless/trace/inferredspan/propagation_test.go +pkg/serverless/trace/propagation/carriers_test.go +pkg/serverless/trace/propagation/extractor_test.go +pkg/serverless/trigger/extractor.go +pkg/util/ec2/ec2_tags.go +test/new-e2e/examples/ecs_test.go +test/new-e2e/go.mod +test/new-e2e/pkg/provisioners/aws/kubernetes/kubernetes_dump.go +test/new-e2e/pkg/runner/parameters/store_aws.go +test/new-e2e/pkg/utils/clients/aws.go +test/new-e2e/pkg/utils/e2e/client/ecs/ecs.go +test/new-e2e/pkg/utils/e2e/client/ecs/session-manager-plugin.go +test/new-e2e/tests/containers/ecs_test.go +test/new-e2e/tests/windows/common/pipeline/pipeline.go""") + } + ) + add_reviewers(c, 1234) + print_mock.assert_not_called() + self.assertCountEqual( + pr_mock.create_review_request.call_args[1]['team_reviewers'], + [ + 'windows-agent', + 'container-integrations', + 'agent-devx-loops', + 'container-platform', + 'windows-kernel-integrations', + 'agent-e2e-testing', + ], + )