Merge branch 'better-plugin-support' into 'master'

Better plugin (un)loading --- more logging, !reload plugin with option
of reloading entire package

In addition to a variety of logging and internal changes, this
also adds a !reload [plugin] command, which, where 'plugin' is
'package.ircplugin', just does a simple unload/load on the
plugin, but where 'plugin' is 'package.', reloads all modules
in the specified package (assuming it is still a plugin's
package), which allows for making more code changes and applying
them without having to restart the bot.

Like all things this internal to the bot, this probably needs
the tires kicked a bit more, but initial testing has succeeded.

See merge request !2

dr_botzo/ircbot/bot.py

@@ -3,6 +3,7 @@
 import bisect
 import collections
 import copy
+import importlib
 import logging
 import re
 from xmlrpc.server import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
@@ -356,6 +357,10 @@ class IRCBot(irc.client.SimpleIRCClient):
                                                          getattr(self, 'handle_unload'), -20)
         self.connection.reactor.add_global_regex_handler('privmsg', r'^!unload\s+([\S]+)$',
                                                          getattr(self, 'handle_unload'), -20)
+        self.connection.reactor.add_global_regex_handler('pubmsg', r'^!reload\s+([\S]+)$',
+                                                         getattr(self, 'handle_reload'), -20)
+        self.connection.reactor.add_global_regex_handler('privmsg', r'^!reload\s+([\S]+)$',
+                                                         getattr(self, 'handle_reload'), -20)
 
         # load XML-RPC server
         self.xmlrpc = SimpleXMLRPCServer((settings.IRCBOT_XMLRPC_HOST, settings.IRCBOT_XMLRPC_PORT),
@@ -576,7 +581,18 @@ class IRCBot(irc.client.SimpleIRCClient):
         pass
 
     def handle_load(self, connection, event, match):
-        """Handle IRC requests to load a plugin."""
+        """Handle IRC requests to load a plugin.
+
+        This loads a plugin, as requested, and the interpreter takes care of
+        loading dependencies.
+
+        :param connection: connection for this load request
+        :type connection: LenientServerConnection
+        :param event: associated irc event object
+        :type event: Event
+        :param match: the matched regex (via add_global_regex_handler)
+        :type match: Match
+        """
 
         has_perm = ircbotlib.has_permission(event.source, 'ircbot.manage_loaded_plugins')
         log.debug("has permission to load?: %s", str(has_perm))
@@ -588,16 +604,33 @@ class IRCBot(irc.client.SimpleIRCClient):
     def _load_plugin(self, connection, event, plugin_path, feedback=True):
         """Load an IRC plugin.
 
-        The general assumption here is that a plugin's init loads its hooks and handlers.
+        A plugin's init must initialize its hooks and handlers as references to this bot, this
+        is handed off but otherwise nothing is done with that here, this just loads it from a
+        Python sense.
+
+        :param connection: connection for this load request
+        :type connection: LenientServerConnection
+        :param event: associated irc event object
+        :type event: Event
+        :param plugin_path: path (likely a relative one) of the plugin to attempt to load
+        :type plugin_path: str
+        :param feedback: whether or not to send messages to IRC regarding the load attempt
+        :type feedback: bool
         """
 
-        log.debug("trying to load %s", plugin_path)
+        log.debug("trying to load plugin %s", plugin_path)
 
         dest = None
         if feedback:
             dest = ircbotlib.reply_destination_for_event(event)
 
+        # loop over the sys.modules entries, for debugging
+        log.debug("loaded modules:")
+        for module_name, module in sys.modules.items():
+            log.debug("  %s", module_name)
+
         for path, plugin in self.plugins:
+            log.debug("looping already loaded plugin: %s %s", plugin, path)
             if plugin_path == path:
                 if feedback:
                     self.privmsg(dest, "Plugin '{0:s}' is already loaded.".format(plugin_path))
@@ -629,35 +662,163 @@ class IRCBot(irc.client.SimpleIRCClient):
             if feedback:
                 self.privmsg(dest, "Plugin '{0:s}' could not be loaded: {1:s}".format(plugin_path, str(e)))
 
+        # loop over the sys.modules entries, for debugging
+        log.debug("new list of loaded modules:")
+        for module_name, module in sys.modules.items():
+            log.debug("  %s", module_name)
+
+    def handle_reload(self, connection, event, match):
+        """Handle IRC requests to reload a plugin (module or package).
+
+        This allows code for a plugin (and potentially also its sibling modules in the package)
+        to be reloaded on demand, in the event that changes were made.
+
+        :param connection: connection for this reload request
+        :type connection: LenientServerConnection
+        :param event: associated irc event object
+        :type event: Event
+        :param match: the matched regex (via add_global_regex_handler)
+        :type match: Match
+        """
+
+        has_perm = ircbotlib.has_permission(event.source, 'ircbot.manage_loaded_plugins')
+        log.debug("has permission to reload?: %s", str(has_perm))
+        if has_perm:
+            plugin_path = match.group(1)
+            log.debug("calling _reload_plugin on %s", plugin_path)
+            self._reload_plugin(connection, event, plugin_path)
+
+    def _reload_plugin(self, connection, event, plugin_path, feedback=True):
+        """Execute the built-in reload() on a plugin (module) or it and its siblings (package).
+
+        :param connection: connection for this load request
+        :type connection: LenientServerConnection
+        :param event: associated irc event object
+        :type event: Event
+        :param plugin_path: path (likely a relative one) of the plugin to attempt to reload
+        :type plugin_path: str
+        :param feedback: whether or not to send messages to IRC regarding the reload attempt
+        :type feedback: bool
+        """
+
+        log.debug("trying to unload plugin %s", plugin_path)
+
+        dest = ircbotlib.reply_destination_for_event(event)
+
+        # loop over the sys.modules entries, for debugging
+        log.debug("loaded modules:")
+        for module_name, module in sys.modules.items():
+            log.debug("  %s", module_name)
+
+        # make sure what was provided was a legitimate plugin
+        # (so that we don't go reloading totally arbitrary code)
+        search_path = None
+        for path, plugin in self.plugins:
+            # first check exact path
+            if path == plugin_path:
+                log.debug("requested %s is %s, so we will try to reload %s",
+                          plugin_path, path, plugin_path)
+                search_path = plugin_path
+            if plugin_path[-1] == '.' and path.startswith(plugin_path):
+                log.debug("requested %s is substring of %s, so we will try to reload all %s items",
+                          plugin_path, path, plugin_path)
+                search_path = plugin_path[:-1]
+
+        # we need to know the plugin names to do some extra logic with reload
+        plugin_names = [x[0] for x in self.plugins]
+
+        # do reload(s)
+        if search_path:
+            plugins_to_reload = []
+            for module_name, module in sys.modules.items():
+                if module_name.startswith(search_path):
+                    if module_name in plugin_names:
+                        # we need to take the plugin reload path for this, but not
+                        # before all other code is reloaded, so we save this for the end
+                        # if we did this here, the ircplugin might still reference the old
+                        # version of its dependencies
+                        plugins_to_reload.append(module_name)
+                    else:
+                        log.debug("reloading %s", module_name)
+                        importlib.reload(module)
+
+                    if feedback:
+                        self.privmsg(dest, "'{0:s}' reloaded.".format(module_name))
+
+            # the references to event handlers will be stale (reload() doesn't go that deep)
+            # so we need to tear down and reinitialize plugins
+            for plugin_to_reload in plugins_to_reload:
+                log.debug("unloading plugin %s", plugin_to_reload)
+                self._unload_plugin(event, plugin_to_reload, False)
+                log.debug("loading plugin %s", plugin_to_reload)
+                self._load_plugin(connection, event, plugin_to_reload, False)
+
     def handle_unload(self, connection, event, match):
-        """Handle IRC requests to unload a plugin."""
+        """Handle IRC requests to unload a plugin.
+
+        Unloading includes disconnecting the IRC plugin hooks, so this is the
+        way to "remove" a plugin from the IRC bot. Note that this only
+        removes the module for the specified plugin, not its dependencies, so
+        if you're trying to unload in order to refresh modules, this won't
+        do what you want.
+
+        :param connection: connection for this unload request
+        :type connection: LenientServerConnection
+        :param event: associated irc event object
+        :type event: Event
+        :param match: the matched regex (via add_global_regex_handler)
+        :type match: Match
+        """
 
         has_perm = ircbotlib.has_permission(event.source, 'ircbot.manage_loaded_plugins')
         log.debug("has permission to unload?: %s", str(has_perm))
         if has_perm:
             plugin_path = match.group(1)
             log.debug("calling _unload_plugin on %s", plugin_path)
-            self._unload_plugin(connection, event, plugin_path)
+            self._unload_plugin(event, plugin_path)
 
-    def _unload_plugin(self, connection, event, plugin_path):
+    def _unload_plugin(self, event, plugin_path, feedback=True):
-        """Attempt to unload and del a module if it's loaded."""
+        """Attempt to unload and del a module if it's loaded.
 
-        log.debug("trying to unload %s", plugin_path)
+        This stops the plugin, which should (if it's coded properly) disconnect its
+        event handlers and kill threads and whatnot.
+
+        :param event: associated irc event object
+        :type event: Event
+        :param plugin_path: path (likely a relative one) of the plugin to attempt to unload
+        :type plugin_path: str
+        :param feedback: whether or not to send messages to IRC regarding the unload attempt
+        :type feedback: bool
+        """
+
+        log.debug("trying to unload plugin %s", plugin_path)
 
         dest = ircbotlib.reply_destination_for_event(event)
 
+        # loop over the sys.modules entries, for debugging
+        log.debug("loaded modules:")
+        for module_name, module in sys.modules.items():
+            log.debug("  %s", module_name)
+
+        unloaded = False
         for path, plugin in self.plugins:
             if plugin_path == path:
+                unloaded = True
                 self.plugins.remove((path, plugin))
                 plugin.stop()
                 del plugin
                 del sys.modules[plugin_path]
 
-                self.privmsg(dest, "Plugin '{0:s}' unloaded.".format(plugin_path))
+                if feedback:
-                return
+                    self.privmsg(dest, "Plugin '{0:s}' unloaded.".format(plugin_path))
 
-        # guess it was never loaded
+        if not unloaded and feedback:
-        self.privmsg(dest, "Plugin '{0:s}' is not loaded.".format(plugin_path))
+            self.privmsg(dest, "Plugin '{0:s}' is not loaded.".format(plugin_path))
+
+        # loop over the sys.modules entries, for debugging
+        log.debug("new list of loaded modules:")
+        for module_name, module in sys.modules.items():
+            log.debug("  %s", module_name)
 
     def privmsg(self, target, text):
         """Send a PRIVMSG command.