Add create-pipeline and job-regexp options

marge/app.py

@@ -203,6 +203,17 @@ def regexp(str_regex):
         default='.*',
         help='Only process MRs whose source branches match the given regular expression.\n',
     )
+    parser.add_argument(
+        '--job-regexp',
+        type=regexp,
+        default='',
+        help='Require pipelines to have jobs matching the given regular expression.\n',
+    )
+    parser.add_argument(
+        '--create-pipeline',
+        action='store_true',
+        help='Create new pipeline if not up to date or not matching job-regexp.\n',
+    )
     parser.add_argument(
         '--debug',
         action='store_true',
@@ -306,6 +317,8 @@ def main(args=None):
                 embargo=options.embargo,
                 ci_timeout=options.ci_timeout,
                 fusion=fusion,
+                job_regexp=options.job_regexp,
+                create_pipeline=options.create_pipeline,
             ),
             batch=options.batch,
         )

marge/batch_job.py

@@ -152,7 +152,7 @@ def accept_mr(
 
         sleep(2)
 
-        # At this point Gitlab should have recognised the MR as being accepted.
+        # At this point GitLab should have recognised the MR as being accepted.
         log.info('Successfully merged MR !%s', merge_request.iid)
 
         pipelines = Pipeline.pipelines_by_branch(

marge/gitlab.py

@@ -10,7 +10,7 @@ def __init__(self, gitlab_url, auth_token):
         self._auth_token = auth_token
         self._api_base_url = gitlab_url.rstrip('/') + '/api/v4'
 
-    def call(self, command, sudo=None):
+    def call(self, command, sudo=None, response_json=None):
         method = command.method
         url = self._api_base_url + command.endpoint
         headers = {'PRIVATE-TOKEN': self._auth_token}
@@ -28,6 +28,9 @@ def call(self, command, sudo=None):
         log.debug('RESPONSE CODE: %s', response.status_code)
         log.debug('RESPONSE BODY: %r', response.content)
 
+        if response_json is not None:
+            response_json.update(response.json())
+
         if response.status_code == 202:
             return True  # Accepted
 

marge/job.py

@@ -2,6 +2,7 @@
 import enum
 import logging as log
 import time
+import re
 from collections import namedtuple
 from datetime import datetime, timedelta
 
@@ -149,15 +150,62 @@ def get_mr_ci_status(self, merge_request, commit_sha=None):
                 self._api,
             )
         current_pipeline = next(iter(pipeline for pipeline in pipelines if pipeline.sha == commit_sha), None)
+        create_pipeline = self.opts.create_pipeline
+
+        trigger = False
 
         if current_pipeline:
             ci_status = current_pipeline.status
+            if self.opts.job_regexp.pattern:
+                jobs = current_pipeline.get_jobs()
+                if not any(self.opts.job_regexp.match(j['name']) for j in jobs):
+                    if create_pipeline:
+                        message = 'CI doesn\'t contain the required jobs.'
+                        log.warning(message)
+                        trigger = True
+                    else:
+                        raise CannotMerge('CI doesn\'t contain the required jobs.')
         else:
-            log.warning('No pipeline listed for %s on branch %s', commit_sha, merge_request.source_branch)
+            message = 'No pipeline listed for {sha} on branch {branch}.'.format(
+                sha=commit_sha, branch=merge_request.source_branch
+            )
+            log.warning(message)
+            ci_status = None
+            if create_pipeline:
+                trigger = True
+
+        if trigger:
+            self.trigger_pipeline(merge_request, message)
             ci_status = None
 
         return ci_status
 
+    def trigger_pipeline(self, merge_request, message=''):
+        if merge_request.triggered(self._user.id):
+            raise CannotMerge(
+                ('{message}\n\nI don\'t know what else I can do. ' +
+                 'You may need to manually trigger the pipeline or rename the branch.').format(
+                    message=message
+                )
+            )
+        new_pipeline = Pipeline.create(
+            merge_request.source_project_id,
+            merge_request.source_branch,
+            self._api,
+        )
+        if new_pipeline:
+            log.info('New pipeline created')
+            merge_request.comment(
+                ('{message}\n\nI created a new pipeline for [{sha:.8s}](/../commit/{sha}): ' +
+                 '[#{pipeline_id}](/../pipelines/{pipeline_id}).').format(
+                    message=message, sha=merge_request.sha, pipeline_id=new_pipeline.id
+                )
+            )
+        else:
+            raise CannotMerge(
+                '{message}\n\nI couldn\'t create a new pipeline.'.format(message=message)
+            )
+
     def wait_for_ci_to_pass(self, merge_request, commit_sha=None):
         time_0 = datetime.utcnow()
         waiting_time_in_secs = 10
@@ -280,7 +328,7 @@ def update_from_target_branch_and_push(
         target_branch = merge_request.target_branch
         assert source_repo_url != repo.remote_url
         if source_repo_url is None and source_branch == target_branch:
-            raise CannotMerge('source and target branch seem to coincide!')
+            raise CannotMerge('Source and target branch seem to coincide!')
 
         branch_update_done = commits_rewrite_done = False
         try:
@@ -295,16 +343,16 @@ def update_from_target_branch_and_push(
             # the sha from the remote target branch.
             target_sha = repo.get_commit_hash('origin/' + target_branch)
             if updated_sha == target_sha:
-                raise CannotMerge('these changes already exist in branch `{}`'.format(target_branch))
+                raise CannotMerge('These changes already exist in branch `{}`.'.format(target_branch))
             final_sha = self.add_trailers(merge_request) or updated_sha
             commits_rewrite_done = True
             branch_was_modified = final_sha != initial_mr_sha
             self.synchronize_mr_with_local_changes(merge_request, branch_was_modified, source_repo_url)
         except git.GitError:
             if not branch_update_done:
-                raise CannotMerge('got conflicts while rebasing, your problem now...')
+                raise CannotMerge('Got conflicts while rebasing, your problem now...')
             if not commits_rewrite_done:
-                raise CannotMerge('failed on filter-branch; check my logs!')
+                raise CannotMerge('Failed on filter-branch; check my logs!')
             raise
         else:
             return target_sha, updated_sha, final_sha
@@ -408,6 +456,8 @@ class Fusion(enum.Enum):
     'embargo',
     'ci_timeout',
     'fusion',
+    'job_regexp',
+    'create_pipeline',
 ]
 
 
@@ -423,6 +473,7 @@ def default(
             cls, *,
             add_tested=False, add_part_of=False, add_reviewers=False, reapprove=False,
             approval_timeout=None, embargo=None, ci_timeout=None, fusion=Fusion.rebase,
+            job_regexp=re.compile(''), create_pipeline=False
     ):
         approval_timeout = approval_timeout or timedelta(seconds=0)
         embargo = embargo or IntervalUnion.empty()
@@ -436,6 +487,8 @@ def default(
             embargo=embargo,
             ci_timeout=ci_timeout,
             fusion=fusion,
+            job_regexp=job_regexp,
+            create_pipeline=create_pipeline,
         )
 
 

marge/merge_request.py

@@ -8,6 +8,7 @@
 GET, POST, PUT, DELETE = gitlab.GET, gitlab.POST, gitlab.PUT, gitlab.DELETE
 
 
+# pylint: disable=R0904
 class MergeRequest(gitlab.Resource):
 
     @classmethod
@@ -189,6 +190,18 @@ def fetch_approvals(self):
     def fetch_commits(self):
         return self._api.call(GET('/projects/{0.project_id}/merge_requests/{0.iid}/commits'.format(self)))
 
+    def triggered(self, user_id):
+        if self._api.version().release >= (9, 2, 2):
+            notes_url = '/projects/{0.project_id}/merge_requests/{0.iid}/notes'.format(self)
+        else:
+            # GitLab botched the v4 api before 9.2.2
+            notes_url = '/projects/{0.project_id}/merge_requests/{0.id}/notes'.format(self)
+
+        comments = self._api.collect_all_pages(GET(notes_url))
+        message = 'I created a new pipeline for [{sha:.8s}]'.format(sha=self.sha)
+        my_comments = [c['body'] for c in comments if c['author']['id'] == user_id]
+        return any(message in c for c in my_comments)
+
 
 class MergeRequestRebaseFailed(Exception):
     pass

marge/pipeline.py

@@ -42,6 +42,18 @@ def pipelines_by_merge_request(cls, project_id, merge_request_iid, api):
         pipelines_info.sort(key=lambda pipeline_info: pipeline_info['id'], reverse=True)
         return [cls(api, pipeline_info, project_id) for pipeline_info in pipelines_info]
 
+    @classmethod
+    def create(cls, project_id, ref, api):
+        try:
+            pipeline_info = {}
+            api.call(POST(
+                '/projects/{project_id}/pipeline'.format(project_id=project_id), {'ref': ref}),
+                response_json=pipeline_info
+            )
+            return cls(api, pipeline_info, project_id)
+        except gitlab.ApiError:
+            return None
+
     @property
     def project_id(self):
         return self.info['project_id']
@@ -66,3 +78,10 @@ def cancel(self):
         return self._api.call(POST(
             '/projects/{0.project_id}/pipelines/{0.id}/cancel'.format(self),
         ))
+
+    def get_jobs(self):
+        jobs_info = self._api.call(GET(
+            '/projects/{0.project_id}/pipelines/{0.id}/jobs'.format(self),
+        ))
+
+        return jobs_info

marge/single_merge_job.py

@@ -57,7 +57,7 @@ def update_merge_request_and_accept(self, approvals):
                     source_repo_url=source_repo_url,
                 )
             except GitLabRebaseResultMismatch:
-                log.info("Gitlab rebase didn't give expected result")
+                log.info("GitLab rebase didn't give expected result")
                 merge_request.comment("Someone skipped the queue! Will have to try again...")
                 continue
 
@@ -130,14 +130,14 @@ def update_merge_request_and_accept(self, approvals):
                     updated_into_up_to_date_target_branch = True
                 else:
                     raise CannotMerge(
-                        "Gitlab refused to merge this request and I don't know why!" + (
+                        "GitLab refused to merge this request and I don't know why!" + (
                             " Maybe you have unresolved discussions?"
                             if self._project.only_allow_merge_if_all_discussions_are_resolved else ""
                         )
                     )
             except gitlab.ApiError:
                 log.exception('Unanticipated ApiError from GitLab on merge attempt')
-                raise CannotMerge('had some issue with GitLab, check my logs...')
+                raise CannotMerge('Had some issue with GitLab, check my logs...')
             else:
                 self.wait_for_branch_to_be_merged()
                 updated_into_up_to_date_target_branch = True
@@ -153,7 +153,7 @@ def wait_for_branch_to_be_merged(self):
             if merge_request.state == 'merged':
                 return  # success!
             if merge_request.state == 'closed':
-                raise CannotMerge('someone closed the merge request while merging!')
+                raise CannotMerge('Someone closed the merge request while merging!')
             assert merge_request.state in ('opened', 'reopened', 'locked'), merge_request.state
 
             log.info('Giving %s more secs for !%s to be merged...', waiting_time_in_secs, merge_request.iid)

tests/gitlab_api_mock.py

@@ -101,7 +101,7 @@ def __init__(self, gitlab_url, auth_token, initial_state):
         self.state = initial_state
         self.notes = []
 
-    def call(self, command, sudo=None):
+    def call(self, command, sudo=None, response_json=None):
         log.info(
             'CALL: %s%s @ %s',
             'sudo %s ' % sudo if sudo is not None else '',

tests/test_job.py

@@ -1,4 +1,5 @@
 # pylint: disable=protected-access
+import re
 from datetime import timedelta
 from unittest.mock import ANY, Mock, patch, create_autospec
 
@@ -200,6 +201,8 @@ def test_default(self):
             embargo=marge.interval.IntervalUnion.empty(),
             ci_timeout=timedelta(minutes=15),
             fusion=Fusion.rebase,
+            job_regexp=re.compile(''),
+            create_pipeline=False,
         )
 
     def test_default_ci_time(self):

tests/test_single_job.py

@@ -646,7 +646,7 @@ def test_assumes_unresolved_discussions_on_merge_refusal(self, mocks):
             from_state='unresolved_discussions',
         )
         message = (
-            "Gitlab refused to merge this request and I don't know why! "
+            "GitLab refused to merge this request and I don't know why! "
             "Maybe you have unresolved discussions?"
         )
         with mocklab.expected_failure(message):
@@ -687,7 +687,7 @@ def test_tells_explicitly_that_gitlab_refused_to_merge(self, mocks):
             Error(marge.gitlab.MethodNotAllowed(405, {'message': '405 Method Not Allowed'})),
             from_state='passed', to_state='rejected_for_mysterious_reasons',
         )
-        message = "Gitlab refused to merge this request and I don't know why!"
+        message = "GitLab refused to merge this request and I don't know why!"
         with mocklab.expected_failure(message):
             job.execute()
         assert api.state == 'rejected_for_mysterious_reasons'
@@ -748,7 +748,7 @@ def test_fails_if_changes_already_exist(self, mocks):
         target_branch = mocklab.merge_request_info['target_branch']
 
         remote_target_repo.set_ref(target_branch, remote_source_repo.get_ref(source_branch))
-        expected_message = 'these changes already exist in branch `%s`' % target_branch
+        expected_message = 'These changes already exist in branch `%s`.' % target_branch
 
         with mocklab.expected_failure(expected_message):
             job.execute()