Merge branch 'gitlab' into 'master'

Gitlab code review support

See merge request !12

dr_botzo/dr_botzo/settings.py

@@ -47,6 +47,7 @@ INSTALLED_APPS = (
     'countdown',
     'dispatch',
     'facts',
+    'gitlab_bot',
     'ircbot',
     'karma',
     'markov',

dr_botzo/gitlab_bot/admin.py

@@ -0,0 +1,8 @@
+"""Admin stuff for GitLab bot models."""
+
+from django.contrib import admin
+
+from gitlab_bot.models import GitlabConfig, GitlabProjectConfig
+
+admin.site.register(GitlabConfig)
+admin.site.register(GitlabProjectConfig)

dr_botzo/gitlab_bot/lib.py

@@ -0,0 +1,253 @@
+"""Wrapped client for the configured GitLab bot."""
+
+import logging
+import random
+import re
+
+import gitlab
+
+from gitlab_bot.models import GitlabConfig
+
+log = logging.getLogger(__name__)
+
+
+class GitlabBot(object):
+
+    """Bot for doing GitLab stuff. Might be used by the IRC bot, or daemons, or whatever."""
+
+    REVIEWS_START_FORMAT = "Starting review process."
+    REVIEWS_RESET_FORMAT = "Review process reset."
+    REVIEWS_REVIEW_REGISTERED_FORMAT = "Review identified."
+    REVIEWS_PROGRESS_FORMAT = "{0:d} reviews of {1:d} necessary to proceed."
+    REVIEWS_REVIEWS_COMPLETE = "Reviews complete."
+    NEW_REVIEWER_FORMAT = "Assigning to {0:s} to review merge request."
+    NEW_ACCEPTER_FORMAT = "Assigning to {0:s} to accept merge request."
+
+    NOTE_COMMENT = [
+        "The computer is your friend.",
+        "Trust the computer.",
+        "Quality is mandatory.",
+        "Errors show your disloyalty to the computer.",
+    ]
+
+    def __init__(self):
+        """Initialize the actual GitLab client."""
+        config = GitlabConfig.objects.first()
+        self.client = gitlab.Gitlab(config.url, private_token=config.token)
+        self.client.auth()
+
+    def random_reviews_start_message(self):
+        return "{0:s} {1:s} {2:s}".format(self.REVIEWS_START_FORMAT, self.REVIEWS_PROGRESS_FORMAT,
+                                          random.choice(self.NOTE_COMMENT))
+
+    def random_reviews_reset_message(self):
+        return "{0:s} {1:s} {2:s}".format(self.REVIEWS_RESET_FORMAT, self.REVIEWS_PROGRESS_FORMAT,
+                                          random.choice(self.NOTE_COMMENT))
+
+    def random_review_progress_message(self):
+        return "{0:s} {1:s} {2:s}".format(self.REVIEWS_REVIEW_REGISTERED_FORMAT, self.REVIEWS_PROGRESS_FORMAT,
+                                          random.choice(self.NOTE_COMMENT))
+
+    def random_reviews_complete_message(self):
+        return "{0:s} {1:s}".format(self.REVIEWS_REVIEWS_COMPLETE, random.choice(self.NOTE_COMMENT))
+
+    def random_new_reviewer_message(self):
+        return "{0:s} {1:s}".format(self.NEW_REVIEWER_FORMAT, random.choice(self.NOTE_COMMENT))
+
+    def random_reviews_done_message(self):
+        return "{0:s} {1:s}".format(self.NEW_ACCEPTER_FORMAT, random.choice(self.NOTE_COMMENT))
+
+    def scan_project_for_reviews(self, project, merge_request_ids=None):
+        project_obj = self.client.projects.get(project.project_id)
+        if not project_obj:
+            return
+
+        if merge_request_ids:
+            merge_requests = []
+            for merge_request_id in merge_request_ids:
+                merge_requests.append(project_obj.mergerequests.get(id=merge_request_id))
+        else:
+            merge_requests = project_obj.mergerequests.list(state='opened')
+
+        for merge_request in merge_requests:
+            log.debug("scanning merge request '%s'", merge_request.title)
+
+            if merge_request.state in ['merged', 'closed']:
+                log.info("merge request '%s' is already %s, doing nothing", merge_request.title,
+                         merge_request.state)
+                continue
+
+            request_state = _MergeRequestScanningState()
+            notes = sorted(self.client.project_mergerequest_notes.list(project_id=merge_request.project_id,
+                                                                       merge_request_id=merge_request.id,
+                                                                       all=True),
+                           key=lambda x: x.id)
+            for note in notes:
+                if not note.system:
+                    log.debug("merge request '%s', note '%s' is a normal message", merge_request.title, note.id)
+                    # note that we can't ignore note.system = True + merge_request.author, that might have been
+                    # a new push or something, so we only ignore normal notes from the author
+                    if note.author == merge_request.author:
+                        log.debug("skipping note from the merge request author")
+                    elif note.author.username == self.client.user.username:
+                        log.debug("saw a message from myself, i might have already sent a notice i'm sitting on")
+                        if note.body.find(self.REVIEWS_RESET_FORMAT) >= 0 and request_state.unlogged_approval_reset:
+                            log.debug("saw a reset message, unsetting the flag")
+                            request_state.unlogged_approval_reset = False
+                        elif note.body.find(self.REVIEWS_START_FORMAT) >= 0 and request_state.unlogged_review_start:
+                            log.debug("saw a start message, unsetting the flag")
+                            request_state.unlogged_review_start = False
+                        elif note.body.find(self.REVIEWS_REVIEW_REGISTERED_FORMAT) >= 0 and request_state.unlogged_approval:
+                            log.debug("saw a review log message, unsetting the flag")
+                            request_state.unlogged_approval = False
+                        elif note.body.find(self.REVIEWS_REVIEWS_COMPLETE) >= 0 and request_state.unlogged_review_complete:
+                            log.debug("saw a review complete message, unsetting the flag")
+                            request_state.unlogged_review_complete = False
+                        else:
+                            log.debug("nothing in particular relevant in '%s'", note.body)
+                    else:
+                        if note.body.find("LGTM") >= 0:
+                            log.debug("merge request '%s', note '%s' has a LGTM", merge_request.title, note.id)
+                            request_state.unlogged_approval = True
+                            request_state.approver_list.append(note.author.username)
+                            log.debug("approvers: %s", request_state.approver_list)
+                            if len(request_state.approver_list) < project.code_reviews_necessary:
+                                log.debug("not enough code reviews yet, setting needs_reviewer")
+                                request_state.needs_reviewer = True
+                                request_state.needs_accepter = False
+                            else:
+                                log.debug("enough code reviews, setting needs_accepter")
+                                request_state.needs_accepter = True
+                                request_state.needs_reviewer = False
+                                request_state.unlogged_review_complete = True
+                        else:
+                            log.debug("merge request '%s', note '%s' does not have a LGTM", merge_request.title,
+                                      note.id)
+                else:
+                    log.debug("merge request '%s', note '%s' is a system message", merge_request.title, note.id)
+                    if re.match(r'Added \d+ commit', note.body):
+                        log.debug("resetting approval list, '%s' looks like a push!", note.body)
+
+                        # only set the unlogged approval reset flag if there's some kind of progress
+                        if len(request_state.approver_list) > 0:
+                            request_state.unlogged_approval_reset = True
+                            request_state.needs_reviewer = True
+                        request_state.approver_list.clear()
+                    else:
+                        log.debug("leaving the approval list as it is, i don't think '%s' is a push", note.body)
+
+            # do some cleanup
+            excluded_review_candidates = request_state.approver_list + [merge_request.author.username]
+            review_candidates = [x for x in project.code_reviewers.split(',') if x not in excluded_review_candidates]
+            if merge_request.assignee:
+                if request_state.needs_reviewer and merge_request.assignee.username in review_candidates:
+                    log.debug("unsetting the needs_reviewer flag, the request is already assigned to one")
+                    request_state.needs_reviewer = False
+                elif request_state.needs_reviewer and merge_request.assignee.username == merge_request.author.username:
+                    log.info("unsetting the needs_reviewer flag, the request is assigned to the author")
+                    log.info("in this case we are assuming that the author has work to do, and will re/unassign")
+                    request_state.needs_reviewer = False
+
+            excluded_accept_candidates = [merge_request.author.username]
+            accept_candidates = [x for x in project.code_review_final_merge_assignees.split(',') if x not in excluded_accept_candidates]
+            if merge_request.assignee:
+                if request_state.needs_accepter and merge_request.assignee.username in accept_candidates:
+                    log.debug("unsetting the needs_accepter flag, the request is already assigned to one")
+                    request_state.needs_accepter = False
+                elif request_state.needs_accepter and merge_request.assignee.username == merge_request.author.username:
+                    log.info("unsetting the needs_accepter flag, the request is assigned to the author")
+                    log.info("in this case we are assuming that the author has work to do, and will re/unassign")
+                    request_state.needs_accepter = False
+
+            log.debug("%s", request_state.__dict__)
+
+            # status message stuff
+            if request_state.unlogged_review_start:
+                log.info("sending message for start of reviews")
+                msg = {'body': self.random_reviews_start_message().format(len(request_state.approver_list),
+                                                                          project.code_reviews_necessary)}
+                self.client.project_mergerequest_notes.create(msg, project_id=project_obj.id,
+                                                              merge_request_id=merge_request.id)
+            if request_state.unlogged_approval_reset:
+                log.info("sending message for review reset")
+                msg = {'body': self.random_reviews_reset_message().format(len(request_state.approver_list),
+                                                                          project.code_reviews_necessary)}
+                self.client.project_mergerequest_notes.create(msg, project_id=project_obj.id,
+                                                              merge_request_id=merge_request.id)
+            if request_state.unlogged_approval:
+                log.info("sending message for code review progress")
+                msg = {'body': self.random_review_progress_message().format(len(request_state.approver_list),
+                                                                            project.code_reviews_necessary)}
+                self.client.project_mergerequest_notes.create(msg, project_id=project_obj.id,
+                                                              merge_request_id=merge_request.id)
+            if request_state.unlogged_review_complete:
+                log.info("sending message for code review complete")
+                msg = {'body': self.random_reviews_complete_message()}
+                self.client.project_mergerequest_notes.create(msg, project_id=project_obj.id,
+                                                              merge_request_id=merge_request.id)
+
+            # if there's a reviewer necessary, assign the merge request
+            if len(request_state.approver_list) < project.code_reviews_necessary and request_state.needs_reviewer:
+                log.debug("%s needs a code review", merge_request.title)
+                if merge_request.assignee is not None:
+                    log.debug("%s currently assigned to %s", merge_request.title, merge_request.assignee.username)
+
+                if merge_request.assignee is None or merge_request.assignee.username not in review_candidates:
+                    if len(review_candidates) > 0:
+                        new_reviewer = review_candidates[merge_request.iid % len(review_candidates)]
+                        log.debug("%s is the new reviewer", new_reviewer)
+
+                        # get the user object for the new reviewer
+                        new_reviewer_obj = self.client.users.get_by_username(new_reviewer)
+
+                        # create note for the update
+                        msg = {'body': self.random_new_reviewer_message().format(new_reviewer_obj.name)}
+                        self.client.project_mergerequest_notes.create(msg, project_id=project_obj.id,
+                                                                      merge_request_id=merge_request.id)
+
+                        # assign the merge request to the new reviewer
+                        self.client.update(merge_request, assignee_id=new_reviewer_obj.id)
+                    else:
+                        log.warning("no reviewers left to review %s, doing nothing", merge_request.title)
+                else:
+                    log.debug("needs_reviewer set but the request is assigned to a reviewer, doing nothing")
+
+            # if there's an accepter necessary, assign the merge request
+            if len(request_state.approver_list) >= project.code_reviews_necessary and request_state.needs_accepter:
+                log.debug("%s needs an accepter", merge_request.title)
+                if merge_request.assignee is not None:
+                    log.debug("%s currently assigned to %s", merge_request.title, merge_request.assignee.username)
+
+                if merge_request.assignee is None or merge_request.assignee.username not in accept_candidates:
+                    if len(accept_candidates) > 0:
+                        new_accepter = accept_candidates[merge_request.iid % len(accept_candidates)]
+                        log.debug("%s is the new accepter", new_accepter)
+
+                        # get the user object for the new accepter
+                        new_accepter_obj = self.client.users.get_by_username(new_accepter)
+
+                        # create note for the update
+                        msg = {'body': self.random_reviews_done_message().format(new_accepter_obj.name)}
+                        self.client.project_mergerequest_notes.create(msg, project_id=project_obj.id,
+                                                                      merge_request_id=merge_request.id)
+
+                        # assign the merge request to the new reviewer
+                        self.client.update(merge_request, assignee_id=new_accepter_obj.id)
+                    else:
+                        log.warning("no accepters left to accept %s, doing nothing", merge_request.title)
+
+
+class _MergeRequestScanningState(object):
+
+    """Track the state of a merge request as it is scanned and appropriate action identified."""
+
+    def __init__(self):
+        """Set default flags/values."""
+        self.approver_list = []
+
+        self.unlogged_review_start = True
+        self.unlogged_approval_reset = False
+        self.unlogged_approval = False
+        self.unlogged_review_complete = False
+        self.needs_reviewer = True
+        self.needs_accepter = False

dr_botzo/gitlab_bot/management/commands/code_review_scan.py

@@ -0,0 +1,20 @@
+"""Find merge requests that need code reviewers."""
+
+import logging
+
+from django.core.management import BaseCommand
+
+from gitlab_bot.lib import GitlabBot
+from gitlab_bot.models import GitlabProjectConfig
+
+log = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    help = "Find merge requests needing code reviewers/accepters"
+
+    def handle(self, *args, **options):
+        bot = GitlabBot()
+        projects = GitlabProjectConfig.objects.filter(manage_merge_request_code_reviews=True)
+        for project in projects:
+            bot.scan_project_for_reviews(project)

dr_botzo/gitlab_bot/management/commands/code_review_specific.py

@@ -0,0 +1,25 @@
+"""Run the code review process on a specific merge request."""
+
+import logging
+
+from django.core.management import BaseCommand
+
+from gitlab_bot.lib import GitlabBot
+from gitlab_bot.models import GitlabProjectConfig
+
+log = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    help = "Assign code reviewers/accepters for a specific merge request"
+
+    def add_arguments(self, parser):
+        parser.add_argument('project_id', type=int)
+        parser.add_argument('merge_request_id', type=int)
+
+    def handle(self, *args, **options):
+        project = GitlabProjectConfig.objects.get(pk=options['project_id'])
+        merge_request_ids = [options['merge_request_id'], ]
+
+        bot = GitlabBot()
+        bot.scan_project_for_reviews(project, merge_request_ids=merge_request_ids)

dr_botzo/gitlab_bot/migrations/0001_initial.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='GitlabConfig',
+            fields=[
+                ('id', models.AutoField(auto_created=True, verbose_name='ID', primary_key=True, serialize=False)),
+                ('url', models.URLField()),
+                ('token', models.CharField(max_length=64)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='GitlabProjectConfig',
+            fields=[
+                ('id', models.AutoField(auto_created=True, verbose_name='ID', primary_key=True, serialize=False)),
+                ('project_id', models.CharField(max_length=64)),
+                ('manage_merge_request_code_reviews', models.BooleanField(default=False)),
+                ('code_reviews_necessary', models.PositiveSmallIntegerField(default=0)),
+                ('code_reviewers', models.TextField(blank=True, default='')),
+                ('code_review_final_merge_assignees', models.TextField(blank=True, default='')),
+                ('gitlab_config', models.ForeignKey(to='gitlab_bot.GitlabConfig', null=True)),
+            ],
+        ),
+    ]

dr_botzo/gitlab_bot/migrations/0002_auto_20160624_1354.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gitlab_bot', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='gitlabprojectconfig',
+            name='project_id',
+            field=models.CharField(unique=True, max_length=64),
+        ),
+    ]

dr_botzo/gitlab_bot/models.py

@@ -0,0 +1,36 @@
+"""Bot/daemons for doing stuff with GitLab."""
+
+import logging
+
+from django.db import models
+
+log = logging.getLogger(__name__)
+
+
+class GitlabConfig(models.Model):
+
+    """Maintain bot-wide settings (URL, auth key, etc.)."""
+
+    url = models.URLField()
+    token = models.CharField(max_length=64)
+
+    def __str__(self):
+        """String representation."""
+        return "bot @ {0:s}".format(self.url)
+
+
+class GitlabProjectConfig(models.Model):
+
+    """Maintain settings for a particular project in GitLab."""
+
+    gitlab_config = models.ForeignKey('GitlabConfig', null=True)
+    project_id = models.CharField(max_length=64, unique=True)
+
+    manage_merge_request_code_reviews = models.BooleanField(default=False)
+    code_reviews_necessary = models.PositiveSmallIntegerField(default=0)
+    code_reviewers = models.TextField(default='', blank=True)
+    code_review_final_merge_assignees = models.TextField(default='', blank=True)
+
+    def __str__(self):
+        """String representation."""
+        return "configuration for {0:s} @ {1:s}".format(self.project_id, self.gitlab_config.url)

requirements.txt

@@ -43,6 +43,7 @@ pylint-common==0.2.2
 pylint-django==0.7.1
 pylint-plugin-utils==0.2.3
 python-dateutil==2.4.2
+python-gitlab==0.13
 pytz==2015.7
 PyYAML==3.11
 requests==2.9.1
@@ -52,7 +53,6 @@ setoptconf==0.2.0
 six==1.10.0
 tempora==1.4
 twython==3.3.0
-wheel==0.26.0
 wrapt==1.10.6
 yg.lockfile==2.1
 zc.lockfile==1.1.0