diff --git a/dr_botzo/dr_botzo/settings.py b/dr_botzo/dr_botzo/settings.py index 5d0f678..770bd36 100644 --- a/dr_botzo/dr_botzo/settings.py +++ b/dr_botzo/dr_botzo/settings.py @@ -47,6 +47,7 @@ INSTALLED_APPS = ( 'countdown', 'dispatch', 'facts', + 'gitlab_bot', 'ircbot', 'karma', 'markov', diff --git a/dr_botzo/gitlab_bot/__init__.py b/dr_botzo/gitlab_bot/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dr_botzo/gitlab_bot/admin.py b/dr_botzo/gitlab_bot/admin.py new file mode 100644 index 0000000..e160534 --- /dev/null +++ b/dr_botzo/gitlab_bot/admin.py @@ -0,0 +1,8 @@ +"""Admin stuff for GitLab bot models.""" + +from django.contrib import admin + +from gitlab_bot.models import GitlabConfig, GitlabProjectConfig + +admin.site.register(GitlabConfig) +admin.site.register(GitlabProjectConfig) diff --git a/dr_botzo/gitlab_bot/lib.py b/dr_botzo/gitlab_bot/lib.py new file mode 100644 index 0000000..c4e621b --- /dev/null +++ b/dr_botzo/gitlab_bot/lib.py @@ -0,0 +1,253 @@ +"""Wrapped client for the configured GitLab bot.""" + +import logging +import random +import re + +import gitlab + +from gitlab_bot.models import GitlabConfig + +log = logging.getLogger(__name__) + + +class GitlabBot(object): + + """Bot for doing GitLab stuff. Might be used by the IRC bot, or daemons, or whatever.""" + + REVIEWS_START_FORMAT = "Starting review process." + REVIEWS_RESET_FORMAT = "Review process reset." + REVIEWS_REVIEW_REGISTERED_FORMAT = "Review identified." + REVIEWS_PROGRESS_FORMAT = "{0:d} reviews of {1:d} necessary to proceed." + REVIEWS_REVIEWS_COMPLETE = "Reviews complete." + NEW_REVIEWER_FORMAT = "Assigning to {0:s} to review merge request." + NEW_ACCEPTER_FORMAT = "Assigning to {0:s} to accept merge request." + + NOTE_COMMENT = [ + "The computer is your friend.", + "Trust the computer.", + "Quality is mandatory.", + "Errors show your disloyalty to the computer.", + ] + + def __init__(self): + """Initialize the actual GitLab client.""" + config = GitlabConfig.objects.first() + self.client = gitlab.Gitlab(config.url, private_token=config.token) + self.client.auth() + + def random_reviews_start_message(self): + return "{0:s} {1:s} {2:s}".format(self.REVIEWS_START_FORMAT, self.REVIEWS_PROGRESS_FORMAT, + random.choice(self.NOTE_COMMENT)) + + def random_reviews_reset_message(self): + return "{0:s} {1:s} {2:s}".format(self.REVIEWS_RESET_FORMAT, self.REVIEWS_PROGRESS_FORMAT, + random.choice(self.NOTE_COMMENT)) + + def random_review_progress_message(self): + return "{0:s} {1:s} {2:s}".format(self.REVIEWS_REVIEW_REGISTERED_FORMAT, self.REVIEWS_PROGRESS_FORMAT, + random.choice(self.NOTE_COMMENT)) + + def random_reviews_complete_message(self): + return "{0:s} {1:s}".format(self.REVIEWS_REVIEWS_COMPLETE, random.choice(self.NOTE_COMMENT)) + + def random_new_reviewer_message(self): + return "{0:s} {1:s}".format(self.NEW_REVIEWER_FORMAT, random.choice(self.NOTE_COMMENT)) + + def random_reviews_done_message(self): + return "{0:s} {1:s}".format(self.NEW_ACCEPTER_FORMAT, random.choice(self.NOTE_COMMENT)) + + def scan_project_for_reviews(self, project, merge_request_ids=None): + project_obj = self.client.projects.get(project.project_id) + if not project_obj: + return + + if merge_request_ids: + merge_requests = [] + for merge_request_id in merge_request_ids: + merge_requests.append(project_obj.mergerequests.get(id=merge_request_id)) + else: + merge_requests = project_obj.mergerequests.list(state='opened') + + for merge_request in merge_requests: + log.debug("scanning merge request '%s'", merge_request.title) + + if merge_request.state in ['merged', 'closed']: + log.info("merge request '%s' is already %s, doing nothing", merge_request.title, + merge_request.state) + continue + + request_state = _MergeRequestScanningState() + notes = sorted(self.client.project_mergerequest_notes.list(project_id=merge_request.project_id, + merge_request_id=merge_request.id, + all=True), + key=lambda x: x.id) + for note in notes: + if not note.system: + log.debug("merge request '%s', note '%s' is a normal message", merge_request.title, note.id) + # note that we can't ignore note.system = True + merge_request.author, that might have been + # a new push or something, so we only ignore normal notes from the author + if note.author == merge_request.author: + log.debug("skipping note from the merge request author") + elif note.author.username == self.client.user.username: + log.debug("saw a message from myself, i might have already sent a notice i'm sitting on") + if note.body.find(self.REVIEWS_RESET_FORMAT) >= 0 and request_state.unlogged_approval_reset: + log.debug("saw a reset message, unsetting the flag") + request_state.unlogged_approval_reset = False + elif note.body.find(self.REVIEWS_START_FORMAT) >= 0 and request_state.unlogged_review_start: + log.debug("saw a start message, unsetting the flag") + request_state.unlogged_review_start = False + elif note.body.find(self.REVIEWS_REVIEW_REGISTERED_FORMAT) >= 0 and request_state.unlogged_approval: + log.debug("saw a review log message, unsetting the flag") + request_state.unlogged_approval = False + elif note.body.find(self.REVIEWS_REVIEWS_COMPLETE) >= 0 and request_state.unlogged_review_complete: + log.debug("saw a review complete message, unsetting the flag") + request_state.unlogged_review_complete = False + else: + log.debug("nothing in particular relevant in '%s'", note.body) + else: + if note.body.find("LGTM") >= 0: + log.debug("merge request '%s', note '%s' has a LGTM", merge_request.title, note.id) + request_state.unlogged_approval = True + request_state.approver_list.append(note.author.username) + log.debug("approvers: %s", request_state.approver_list) + if len(request_state.approver_list) < project.code_reviews_necessary: + log.debug("not enough code reviews yet, setting needs_reviewer") + request_state.needs_reviewer = True + request_state.needs_accepter = False + else: + log.debug("enough code reviews, setting needs_accepter") + request_state.needs_accepter = True + request_state.needs_reviewer = False + request_state.unlogged_review_complete = True + else: + log.debug("merge request '%s', note '%s' does not have a LGTM", merge_request.title, + note.id) + else: + log.debug("merge request '%s', note '%s' is a system message", merge_request.title, note.id) + if re.match(r'Added \d+ commit', note.body): + log.debug("resetting approval list, '%s' looks like a push!", note.body) + + # only set the unlogged approval reset flag if there's some kind of progress + if len(request_state.approver_list) > 0: + request_state.unlogged_approval_reset = True + request_state.needs_reviewer = True + request_state.approver_list.clear() + else: + log.debug("leaving the approval list as it is, i don't think '%s' is a push", note.body) + + # do some cleanup + excluded_review_candidates = request_state.approver_list + [merge_request.author.username] + review_candidates = [x for x in project.code_reviewers.split(',') if x not in excluded_review_candidates] + if merge_request.assignee: + if request_state.needs_reviewer and merge_request.assignee.username in review_candidates: + log.debug("unsetting the needs_reviewer flag, the request is already assigned to one") + request_state.needs_reviewer = False + elif request_state.needs_reviewer and merge_request.assignee.username == merge_request.author.username: + log.info("unsetting the needs_reviewer flag, the request is assigned to the author") + log.info("in this case we are assuming that the author has work to do, and will re/unassign") + request_state.needs_reviewer = False + + excluded_accept_candidates = [merge_request.author.username] + accept_candidates = [x for x in project.code_review_final_merge_assignees.split(',') if x not in excluded_accept_candidates] + if merge_request.assignee: + if request_state.needs_accepter and merge_request.assignee.username in accept_candidates: + log.debug("unsetting the needs_accepter flag, the request is already assigned to one") + request_state.needs_accepter = False + elif request_state.needs_accepter and merge_request.assignee.username == merge_request.author.username: + log.info("unsetting the needs_accepter flag, the request is assigned to the author") + log.info("in this case we are assuming that the author has work to do, and will re/unassign") + request_state.needs_accepter = False + + log.debug("%s", request_state.__dict__) + + # status message stuff + if request_state.unlogged_review_start: + log.info("sending message for start of reviews") + msg = {'body': self.random_reviews_start_message().format(len(request_state.approver_list), + project.code_reviews_necessary)} + self.client.project_mergerequest_notes.create(msg, project_id=project_obj.id, + merge_request_id=merge_request.id) + if request_state.unlogged_approval_reset: + log.info("sending message for review reset") + msg = {'body': self.random_reviews_reset_message().format(len(request_state.approver_list), + project.code_reviews_necessary)} + self.client.project_mergerequest_notes.create(msg, project_id=project_obj.id, + merge_request_id=merge_request.id) + if request_state.unlogged_approval: + log.info("sending message for code review progress") + msg = {'body': self.random_review_progress_message().format(len(request_state.approver_list), + project.code_reviews_necessary)} + self.client.project_mergerequest_notes.create(msg, project_id=project_obj.id, + merge_request_id=merge_request.id) + if request_state.unlogged_review_complete: + log.info("sending message for code review complete") + msg = {'body': self.random_reviews_complete_message()} + self.client.project_mergerequest_notes.create(msg, project_id=project_obj.id, + merge_request_id=merge_request.id) + + # if there's a reviewer necessary, assign the merge request + if len(request_state.approver_list) < project.code_reviews_necessary and request_state.needs_reviewer: + log.debug("%s needs a code review", merge_request.title) + if merge_request.assignee is not None: + log.debug("%s currently assigned to %s", merge_request.title, merge_request.assignee.username) + + if merge_request.assignee is None or merge_request.assignee.username not in review_candidates: + if len(review_candidates) > 0: + new_reviewer = review_candidates[merge_request.iid % len(review_candidates)] + log.debug("%s is the new reviewer", new_reviewer) + + # get the user object for the new reviewer + new_reviewer_obj = self.client.users.get_by_username(new_reviewer) + + # create note for the update + msg = {'body': self.random_new_reviewer_message().format(new_reviewer_obj.name)} + self.client.project_mergerequest_notes.create(msg, project_id=project_obj.id, + merge_request_id=merge_request.id) + + # assign the merge request to the new reviewer + self.client.update(merge_request, assignee_id=new_reviewer_obj.id) + else: + log.warning("no reviewers left to review %s, doing nothing", merge_request.title) + else: + log.debug("needs_reviewer set but the request is assigned to a reviewer, doing nothing") + + # if there's an accepter necessary, assign the merge request + if len(request_state.approver_list) >= project.code_reviews_necessary and request_state.needs_accepter: + log.debug("%s needs an accepter", merge_request.title) + if merge_request.assignee is not None: + log.debug("%s currently assigned to %s", merge_request.title, merge_request.assignee.username) + + if merge_request.assignee is None or merge_request.assignee.username not in accept_candidates: + if len(accept_candidates) > 0: + new_accepter = accept_candidates[merge_request.iid % len(accept_candidates)] + log.debug("%s is the new accepter", new_accepter) + + # get the user object for the new accepter + new_accepter_obj = self.client.users.get_by_username(new_accepter) + + # create note for the update + msg = {'body': self.random_reviews_done_message().format(new_accepter_obj.name)} + self.client.project_mergerequest_notes.create(msg, project_id=project_obj.id, + merge_request_id=merge_request.id) + + # assign the merge request to the new reviewer + self.client.update(merge_request, assignee_id=new_accepter_obj.id) + else: + log.warning("no accepters left to accept %s, doing nothing", merge_request.title) + + +class _MergeRequestScanningState(object): + + """Track the state of a merge request as it is scanned and appropriate action identified.""" + + def __init__(self): + """Set default flags/values.""" + self.approver_list = [] + + self.unlogged_review_start = True + self.unlogged_approval_reset = False + self.unlogged_approval = False + self.unlogged_review_complete = False + self.needs_reviewer = True + self.needs_accepter = False diff --git a/dr_botzo/gitlab_bot/management/__init__.py b/dr_botzo/gitlab_bot/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dr_botzo/gitlab_bot/management/commands/__init__.py b/dr_botzo/gitlab_bot/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dr_botzo/gitlab_bot/management/commands/code_review_scan.py b/dr_botzo/gitlab_bot/management/commands/code_review_scan.py new file mode 100644 index 0000000..63fd9a8 --- /dev/null +++ b/dr_botzo/gitlab_bot/management/commands/code_review_scan.py @@ -0,0 +1,20 @@ +"""Find merge requests that need code reviewers.""" + +import logging + +from django.core.management import BaseCommand + +from gitlab_bot.lib import GitlabBot +from gitlab_bot.models import GitlabProjectConfig + +log = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = "Find merge requests needing code reviewers/accepters" + + def handle(self, *args, **options): + bot = GitlabBot() + projects = GitlabProjectConfig.objects.filter(manage_merge_request_code_reviews=True) + for project in projects: + bot.scan_project_for_reviews(project) diff --git a/dr_botzo/gitlab_bot/management/commands/code_review_specific.py b/dr_botzo/gitlab_bot/management/commands/code_review_specific.py new file mode 100644 index 0000000..447ed3f --- /dev/null +++ b/dr_botzo/gitlab_bot/management/commands/code_review_specific.py @@ -0,0 +1,25 @@ +"""Run the code review process on a specific merge request.""" + +import logging + +from django.core.management import BaseCommand + +from gitlab_bot.lib import GitlabBot +from gitlab_bot.models import GitlabProjectConfig + +log = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = "Assign code reviewers/accepters for a specific merge request" + + def add_arguments(self, parser): + parser.add_argument('project_id', type=int) + parser.add_argument('merge_request_id', type=int) + + def handle(self, *args, **options): + project = GitlabProjectConfig.objects.get(pk=options['project_id']) + merge_request_ids = [options['merge_request_id'], ] + + bot = GitlabBot() + bot.scan_project_for_reviews(project, merge_request_ids=merge_request_ids) diff --git a/dr_botzo/gitlab_bot/migrations/0001_initial.py b/dr_botzo/gitlab_bot/migrations/0001_initial.py new file mode 100644 index 0000000..4800d92 --- /dev/null +++ b/dr_botzo/gitlab_bot/migrations/0001_initial.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='GitlabConfig', + fields=[ + ('id', models.AutoField(auto_created=True, verbose_name='ID', primary_key=True, serialize=False)), + ('url', models.URLField()), + ('token', models.CharField(max_length=64)), + ], + ), + migrations.CreateModel( + name='GitlabProjectConfig', + fields=[ + ('id', models.AutoField(auto_created=True, verbose_name='ID', primary_key=True, serialize=False)), + ('project_id', models.CharField(max_length=64)), + ('manage_merge_request_code_reviews', models.BooleanField(default=False)), + ('code_reviews_necessary', models.PositiveSmallIntegerField(default=0)), + ('code_reviewers', models.TextField(blank=True, default='')), + ('code_review_final_merge_assignees', models.TextField(blank=True, default='')), + ('gitlab_config', models.ForeignKey(to='gitlab_bot.GitlabConfig', null=True)), + ], + ), + ] diff --git a/dr_botzo/gitlab_bot/migrations/0002_auto_20160624_1354.py b/dr_botzo/gitlab_bot/migrations/0002_auto_20160624_1354.py new file mode 100644 index 0000000..8e026bb --- /dev/null +++ b/dr_botzo/gitlab_bot/migrations/0002_auto_20160624_1354.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('gitlab_bot', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='gitlabprojectconfig', + name='project_id', + field=models.CharField(unique=True, max_length=64), + ), + ] diff --git a/dr_botzo/gitlab_bot/migrations/__init__.py b/dr_botzo/gitlab_bot/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dr_botzo/gitlab_bot/models.py b/dr_botzo/gitlab_bot/models.py new file mode 100644 index 0000000..b21c685 --- /dev/null +++ b/dr_botzo/gitlab_bot/models.py @@ -0,0 +1,36 @@ +"""Bot/daemons for doing stuff with GitLab.""" + +import logging + +from django.db import models + +log = logging.getLogger(__name__) + + +class GitlabConfig(models.Model): + + """Maintain bot-wide settings (URL, auth key, etc.).""" + + url = models.URLField() + token = models.CharField(max_length=64) + + def __str__(self): + """String representation.""" + return "bot @ {0:s}".format(self.url) + + +class GitlabProjectConfig(models.Model): + + """Maintain settings for a particular project in GitLab.""" + + gitlab_config = models.ForeignKey('GitlabConfig', null=True) + project_id = models.CharField(max_length=64, unique=True) + + manage_merge_request_code_reviews = models.BooleanField(default=False) + code_reviews_necessary = models.PositiveSmallIntegerField(default=0) + code_reviewers = models.TextField(default='', blank=True) + code_review_final_merge_assignees = models.TextField(default='', blank=True) + + def __str__(self): + """String representation.""" + return "configuration for {0:s} @ {1:s}".format(self.project_id, self.gitlab_config.url) diff --git a/requirements.txt b/requirements.txt index 7db8cd6..bb05e39 100644 --- a/requirements.txt +++ b/requirements.txt @@ -43,6 +43,7 @@ pylint-common==0.2.2 pylint-django==0.7.1 pylint-plugin-utils==0.2.3 python-dateutil==2.4.2 +python-gitlab==0.13 pytz==2015.7 PyYAML==3.11 requests==2.9.1 @@ -52,7 +53,6 @@ setoptconf==0.2.0 six==1.10.0 tempora==1.4 twython==3.3.0 -wheel==0.26.0 wrapt==1.10.6 yg.lockfile==2.1 zc.lockfile==1.1.0