Misc: Clamp the number of lines returned by @more to the draft/multiline max-lines value, if any.

They matter in multiline messages, and there isn't much point to
stripping them anyway.

plugins/Misc/plugin.py

@@ -389,12 +389,31 @@ class Misc(callbacks.Plugin):
                       'to see someone else\'s more.  To do so, call this '
                       'command with that person\'s nick.'), Raise=True)
         number = self.registryValue('mores', msg.channel, irc.network)
+
+        if conf.supybot.protocols.irc.experimentalExtensions() \
+                and 'draft/multiline' in irc.state.capabilities_ack:
+            use_multiline = True
+            multiline_cap_values = ircutils.parseCapabilityKeyValue(
+                irc.state.capabilities_ls['draft/multiline'])
+            if multiline_cap_values.get('max-lines', '').isnumeric():
+                number = min(number, int(multiline_cap_values['max-lines']))
+        else:
+            use_multiline = False
+
         msgs = L[-number:]
         msgs.reverse()
         L[-number:] = []
         if msgs:
-            for msg in msgs:
-                irc.queueMsg(msg)
+            if use_multiline and len(msgs) > 1:
+                # If draft/multiline is available, use it.
+                # TODO: set concat=True. For now we can't, because every
+                # message has "(XX more messages)" at the end, so it would be
+                # unreadable if the messages were concatenated
+                irc.queueMultilineBatches(msgs, target=msgs[0].args[0],
+                        targetNick=msg.nick, concat=False)
+            else:
+                for msg in msgs:
+                    irc.queueMsg(msg)
         else:
             irc.error(_('That\'s all, there is no more.'))
     more = wrap(more, [additional('seenNick')])

plugins/Misc/test.py

@@ -236,6 +236,89 @@ class MiscTestCase(ChannelPluginTestCase):
             self.assertResponse('more',
                                 "Error: That's all, there is no more.")
 
+    def testMoreBatch(self):
+        self.irc.state.capabilities_ack.add('batch')
+        self.irc.state.capabilities_ack.add('draft/multiline')
+        self.irc.state.capabilities_ls['draft/multiline'] = 'max-bytes=4096'
+        with conf.supybot.protocols.irc.experimentalExtensions.context(True):
+            with conf.supybot.plugins.Misc.mores.context(2):
+                self.assertResponse('echo %s' % ('abc '*400),
+                                    'abc '*112 + ' \x02(3 more messages)\x02')
+                self.irc.feedMsg(ircmsgs.privmsg(
+                    self.channel, "@more", prefix=self.prefix))
+
+            # First message opens the batch
+            m = self.irc.takeMsg()
+            self.assertEqual(m.command, 'BATCH', m)
+            batch_name = m.args[0][1:]
+            self.assertEqual(
+                m, ircmsgs.IrcMsg(command='BATCH',
+                    args=('+' + batch_name,
+                        'draft/multiline', self.channel)))
+
+            # Second message, first PRIVMSG
+            m = self.irc.takeMsg()
+            self.assertEqual(
+                m, ircmsgs.IrcMsg(command='PRIVMSG',
+                    args=(self.channel, "abc " * 112 + " \x02(2 more messages)\x02"),
+                    server_tags={'batch': batch_name}))
+
+            # Third message, last PRIVMSG
+            m = self.irc.takeMsg()
+            self.assertEqual(
+                m, ircmsgs.IrcMsg(command='PRIVMSG',
+                    args=(self.channel,
+                        "abc " * 112 + " \x02(1 more message)\x02"),
+                    server_tags={'batch': batch_name}))
+
+            # Last message, closes the batch
+            m = self.irc.takeMsg()
+            self.assertEqual(
+                m, ircmsgs.IrcMsg(command='BATCH', args=(
+                    '-' + batch_name,)))
+
+    def testMoreBatchMaxLines(self):
+        self.irc.state.capabilities_ack.add('batch')
+        self.irc.state.capabilities_ack.add('draft/multiline')
+        self.irc.state.capabilities_ls['draft/multiline'] = \
+            'max-bytes=4096,max-lines=2'
+        with conf.supybot.protocols.irc.experimentalExtensions.context(True):
+            with conf.supybot.plugins.Misc.mores.context(3):
+                self.assertResponse('echo %s' % ('abc '*400),
+                                    'abc '*112 + ' \x02(3 more messages)\x02')
+                self.irc.feedMsg(ircmsgs.privmsg(
+                    self.channel, "@more", prefix=self.prefix))
+
+            # First message opens the batch
+            m = self.irc.takeMsg()
+            self.assertEqual(m.command, 'BATCH', m)
+            batch_name = m.args[0][1:]
+            self.assertEqual(
+                m, ircmsgs.IrcMsg(command='BATCH',
+                    args=('+' + batch_name,
+                        'draft/multiline', self.channel)))
+
+            # Second message, first PRIVMSG
+            m = self.irc.takeMsg()
+            self.assertEqual(
+                m, ircmsgs.IrcMsg(command='PRIVMSG',
+                    args=(self.channel, "abc " * 112 + " \x02(2 more messages)\x02"),
+                    server_tags={'batch': batch_name}))
+
+            # Third message, last PRIVMSG
+            m = self.irc.takeMsg()
+            self.assertEqual(
+                m, ircmsgs.IrcMsg(command='PRIVMSG',
+                    args=(self.channel,
+                        "abc " * 112 + " \x02(1 more message)\x02"),
+                    server_tags={'batch': batch_name}))
+
+            # Last message, closes the batch
+            m = self.irc.takeMsg()
+            self.assertEqual(
+                m, ircmsgs.IrcMsg(command='BATCH', args=(
+                    '-' + batch_name,)))
+
     def testClearMores(self):
         self.assertRegexp('echo %s' % ('abc'*700), 'more')
         self.assertRegexp('more', 'more')

src/callbacks.py

@@ -914,10 +914,6 @@ class NestedCommandsIrcProxy(ReplyIrcProxy):
         assert not isinstance(s, ircmsgs.IrcMsg), \
                'Old code alert: there is no longer a "msg" argument to reply.'
         self.repliedTo = True
-        if sendImmediately:
-            sendMsg = self.irc.sendMsg
-        else:
-            sendMsg = self.irc.queueMsg
         if msg is None:
             msg = self.msg
         if prefixNick is not None:
@@ -936,115 +932,14 @@ class NestedCommandsIrcProxy(ReplyIrcProxy):
         if not isinstance(s, minisix.string_types): # avoid trying to str() unicode
             s = str(s) # Allow non-string esses.
 
-        replyArgs = dict(
-            to=self.to,
-            notice=self.notice,
-            action=self.action,
-            private=self.private,
-            prefixNick=self.prefixNick,
-            stripCtcp=stripCtcp
-        )
-
         if self.finalEvaled:
             try:
-                if isinstance(self.irc, self.__class__):
-                    s = s[:conf.supybot.reply.maximumLength()]
-                    return self.irc.reply(s,
-                                          noLengthCheck=self.noLengthCheck,
-                                          **replyArgs)
-                elif self.noLengthCheck:
-                    # noLengthCheck only matters to NestedCommandsIrcProxy, so
-                    # it's not used here.  Just in case you were wondering.
-                    m = _makeReply(self, msg, s, **replyArgs)
-                    sendMsg(m)
-                    return m
-                else:
-                    s = ircutils.safeArgument(s)
-                    allowedLength = conf.get(conf.supybot.reply.mores.length,
-                        channel=target, network=self.irc.network)
-                    if not allowedLength: # 0 indicates this.
-                        allowedLength = (512
-                                - len(':') - len(self.irc.prefix)
-                                - len(' PRIVMSG ')
-                                - len(target)
-                                - len(' :')
-                                - len('\r\n')
-                                )
-                        if self.prefixNick:
-                            allowedLength -= len(msg.nick) + len(': ')
-                    maximumMores = conf.get(conf.supybot.reply.mores.maximum,
-                        channel=target, network=self.irc.network)
-                    maximumLength = allowedLength * maximumMores
-                    if len(s) > maximumLength:
-                        log.warning('Truncating to %s bytes from %s bytes.',
-                                    maximumLength, len(s))
-                        s = s[:maximumLength]
-                    s_size = len(s.encode()) if minisix.PY3 else len(s)
-                    if s_size <= allowedLength or \
-                       not conf.get(conf.supybot.reply.mores,
-                            channel=target, network=self.irc.network):
-                        # There's no need for action=self.action here because
-                        # action implies noLengthCheck, which has already been
-                        # handled.  Let's stick an assert in here just in case.
-                        assert not self.action
-                        m = _makeReply(self, msg, s, **replyArgs)
-                        sendMsg(m)
-                        return m
-                    # The '(XX more messages)' may have not the same
-                    # length in the current locale
-                    allowedLength -= len(_('(XX more messages)')) + 1 # bold
-                    chunks = ircutils.wrap(s, allowedLength)
-
-                    # Last messages to display at the beginning of the list
-                    # (which is used like a stack)
-                    chunks.reverse()
-
-                    instant = conf.get(conf.supybot.reply.mores.instant,
-                        channel=target, network=self.irc.network)
-
-                    msgs = []
-                    for (i, chunk) in enumerate(chunks):
-                        if i == 0:
-                            pass # last message, no suffix to add
-                        elif len(chunks) - i < instant:
-                            # one of the first messages, and the next one will
-                            # also be sent immediately, so no suffix
-                            pass
-                        else:
-                            if i == 1:
-                                more = _('more message')
-                            else:
-                                more = _('more messages')
-                            n = ircutils.bold('(%i %s)' % (len(msgs), more))
-                            chunk = '%s %s' % (chunk, n)
-                        msgs.append(_makeReply(self, msg, chunk, **replyArgs))
-
-                    while instant > 1 and msgs:
-                        instant -= 1
-                        response = msgs.pop()
-                        sendMsg(response)
-                        # XXX We should somehow allow these to be returned, but
-                        #     until someone complains, we'll be fine :)  We
-                        #     can't return from here, though, for obvious
-                        #     reasons.
-                        # return m
-                    if not msgs:
-                        return
-                    response = msgs.pop()
-                    prefix = msg.prefix
-                    if self.to and ircutils.isNick(self.to):
-                        try:
-                            state = self.getRealIrc().state
-                            prefix = state.nickToHostmask(self.to)
-                        except KeyError:
-                            pass # We'll leave it as it is.
-                    mask = prefix.split('!', 1)[1]
-                    self._mores[mask] = msgs
-                    public = bool(self.msg.channel)
-                    private = self.private or not public
-                    self._mores[msg.nick] = (private, msgs)
-                    sendMsg(response)
-                    return response
+                self._sendReply(
+                    s=s, target=target, msg=msg,
+                    sendImmediately=sendImmediately, stripCtcp=stripCtcp)
+            except:
+                log.exception('Error while sending reply')
+                raise
             finally:
                 self._resetReplyAttributes()
         else:
@@ -1058,6 +953,202 @@ class NestedCommandsIrcProxy(ReplyIrcProxy):
                 self.args[self.counter] = s
             self.evalArgs()
 
+    def _replyOverhead(self, target, targetNick):
+        """Returns the number of bytes added to a PRIVMSG payload, either by
+        Limnoria itself or by the server.
+        Ignores tag bytes, as they are accounted for separatly."""
+        overhead = (
+            len(':')
+            + len(self.irc.prefix.encode())
+            + len(' PRIVMSG ')
+            + len(target.encode())
+            + len(' :')
+            + len('\r\n')
+        )
+        if self.prefixNick and targetNick is not None:
+            overhead += len(targetNick) + len(': ')
+        return overhead
+
+    def _sendReply(self, s, target, msg, sendImmediately, stripCtcp):
+        if sendImmediately:
+            sendMsg = self.irc.sendMsg
+        else:
+            sendMsg = self.irc.queueMsg
+
+        replyArgs = dict(
+            to=self.to,
+            notice=self.notice,
+            action=self.action,
+            private=self.private,
+            prefixNick=self.prefixNick,
+            stripCtcp=stripCtcp
+        )
+
+        if isinstance(self.irc, self.__class__):
+            s = s[:conf.supybot.reply.maximumLength()]
+            return self.irc.reply(s,
+                                  noLengthCheck=self.noLengthCheck,
+                                  **replyArgs)
+        elif self.noLengthCheck:
+            # noLengthCheck only matters to NestedCommandsIrcProxy, so
+            # it's not used here.  Just in case you were wondering.
+            m = _makeReply(self, msg, s, **replyArgs)
+            sendMsg(m)
+            return m
+        else:
+            s = ircutils.safeArgument(s)
+            allowedLength = conf.get(conf.supybot.reply.mores.length,
+                channel=target, network=self.irc.network)
+            if not allowedLength: # 0 indicates this.
+                allowedLength = 512 - self._replyOverhead(target, msg.nick)
+            maximumMores = conf.get(conf.supybot.reply.mores.maximum,
+                channel=target, network=self.irc.network)
+            maximumLength = allowedLength * maximumMores
+            if len(s) > maximumLength:
+                log.warning('Truncating to %s bytes from %s bytes.',
+                            maximumLength, len(s))
+                s = s[:maximumLength]
+            s_size = len(s.encode()) if minisix.PY3 else len(s)
+            if s_size <= allowedLength or \
+               not conf.get(conf.supybot.reply.mores,
+                    channel=target, network=self.irc.network):
+                # There's no need for action=self.action here because
+                # action implies noLengthCheck, which has already been
+                # handled.  Let's stick an assert in here just in case.
+                assert not self.action
+                m = _makeReply(self, msg, s, **replyArgs)
+                sendMsg(m)
+                return m
+            # The '(XX more messages)' may have not the same
+            # length in the current locale
+            allowedLength -= len(_('(XX more messages)')) + 1 # bold
+            chunks = ircutils.wrap(s, allowedLength)
+
+            # Last messages to display at the beginning of the list
+            # (which is used like a stack)
+            chunks.reverse()
+
+            instant = conf.get(conf.supybot.reply.mores.instant,
+                channel=target, network=self.irc.network)
+
+            msgs = []
+            for (i, chunk) in enumerate(chunks):
+                if i == 0:
+                    pass # last message, no suffix to add
+                elif len(chunks) - i < instant:
+                    # one of the first messages, and the next one will
+                    # also be sent immediately, so no suffix
+                    pass
+                else:
+                    if i == 1:
+                        more = _('more message')
+                    else:
+                        more = _('more messages')
+                    n = ircutils.bold('(%i %s)' % (len(msgs), more))
+                    chunk = '%s %s' % (chunk, n)
+                msgs.append(_makeReply(self, msg, chunk, **replyArgs))
+
+            instant_messages = []
+
+            while instant > 0 and msgs:
+                instant -= 1
+                response = msgs.pop()
+                instant_messages.append(response)
+                # XXX We should somehow allow these to be returned, but
+                #     until someone complains, we'll be fine :)  We
+                #     can't return from here, though, for obvious
+                #     reasons.
+                # return m
+
+            if conf.supybot.protocols.irc.experimentalExtensions() \
+                    and 'draft/multiline' in self.state.capabilities_ack \
+                    and len(instant_messages) > 1:
+                # More than one message to send now, and we are allowed to use
+                # multiline batches, so let's do it
+                self.queueMultilineBatches(
+                    instant_messages, target, msg.nick, concat=True,
+                    allowedLength=allowedLength, sendImmediately=sendImmediately)
+            else:
+                for instant_msg in instant_messages:
+                    sendMsg(instant_msg)
+
+            if not msgs:
+                return
+            prefix = msg.prefix
+            if self.to and ircutils.isNick(self.to):
+                try:
+                    state = self.getRealIrc().state
+                    prefix = state.nickToHostmask(self.to)
+                except KeyError:
+                    pass # We'll leave it as it is.
+            mask = prefix.split('!', 1)[1]
+            self._mores[mask] = msgs
+            public = bool(self.msg.channel)
+            private = self.private or not public
+            self._mores[msg.nick] = (private, msgs)
+            return response
+
+    def queueMultilineBatches(self, msgs, target, targetNick, concat,
+                              allowedLength=0, sendImmediately=False):
+        """Queues the msgs passed as argument in batches using draft/multiline
+        batches.
+
+        This errors if experimentalExtensions is disabled or draft/multiline
+        was not negotiated."""
+        assert conf.supybot.protocols.irc.experimentalExtensions()
+        assert 'draft/multiline' in self.state.capabilities_ack
+
+        if not allowedLength: # 0 indicates this.
+            allowedLength = 512 - self._replyOverhead(target, targetNick)
+
+        multiline_cap_values = ircutils.parseCapabilityKeyValue(
+            self.state.capabilities_ls['draft/multiline'])
+
+        # All the messages in instant_messages are to be sent
+        # immediately, in multiline batches.
+        max_bytes_per_batch = int(multiline_cap_values['max-bytes'])
+
+        # We have to honor max_bytes_per_batch, but I don't want to
+        # encode messages again here just to have their length, so
+        # let's assume they all have the maximum length.
+        # It's not optimal, but close enough and simplifies the code.
+        messages_per_batch = max_bytes_per_batch // allowedLength
+
+        # "Clients MUST NOT send tags other than draft/multiline-concat and
+        # batch on messages within the batch. In particular, all client-only
+        # tags associated with the message must be sent attached to the initial
+        # BATCH command."
+        # -- <https://ircv3.net/specs/extensions/multiline>
+        # So we copy the tags of the first message, discard the tags of all
+        # other messages, and apply the tags to the opening BATCH
+        server_tags = msgs[0].server_tags
+
+        for batch_msgs in utils.iter.grouper(msgs, messages_per_batch):
+            # TODO: should use sendBatch instead of queueBatch if
+            # sendImmediately is True
+            batch_name = ircutils.makeLabel()
+            batch = []
+            batch.append(ircmsgs.IrcMsg(command='BATCH',
+                args=('+' + batch_name, 'draft/multiline', target),
+                server_tags=server_tags))
+
+            for (i, batch_msg) in enumerate(batch_msgs):
+                if batch_msg is None:
+                    continue  # 'grouper' generates None at the end
+                assert 'batch' not in batch_msg.server_tags
+
+                # Discard the existing tags, and add the batch ones.
+                batch_msg.server_tags = {'batch': batch_name}
+                if concat and i > 0:
+                    # Tell clients not to add a newline after this
+                    batch_msg.server_tags['draft/multiline-concat'] = None
+                batch.append(batch_msg)
+
+            batch.append(ircmsgs.IrcMsg(
+                command='BATCH', args=('-' + batch_name,)))
+
+            self.queueBatch(batch)
+
     def noReply(self, msg=None):
         if msg is None:
             msg = self.msg

src/drivers/__init__.py

@@ -240,7 +240,10 @@ def newDriver(irc, moduleName=None):
     return driver
 
 def parseMsg(s):
-    s = s.strip()
+    # It may be tempting to strip all whitespaces here, but it's important
+    # to preserve them, because they matter for multiline messages.
+    # https://ircv3.net/specs/extensions/multiline
+    s = s.strip('\r\n')
     if s:
         msg = ircmsgs.IrcMsg(s)
         return msg

src/irclib.py

@@ -1218,6 +1218,15 @@ class Irc(IrcCommandDispatcher, log.Firewalled):
         self.state = IrcState()
         self.queue = IrcMsgQueue()
         self.fastqueue = smallqueue()
+
+        # Messages of batches that are currently in one self.queue (not
+        # self.fastqueue).
+        # This works by adding only the first message of a batch in a queue,
+        # and when self.takeMsg pops that message from the queue, it will
+        # also pop the whole batch from self._queued_batches and atomically
+        # add it to self.fastqueue
+        self._queued_batches = {}
+
         self.driver = None # The driver should set this later.
         self._setNonResettingVariables()
         self._queueConnectMessages()
@@ -1306,6 +1315,9 @@ class Irc(IrcCommandDispatcher, log.Firewalled):
 
     def queueMsg(self, msg):
         """Queues a message to be sent to the server."""
+        if msg.command.upper() == 'BATCH':
+            log.error('Tried to send a BATCH message using queueMsg '
+                      'instead of queueBatch: %r', msg)
         if not self.zombie:
             return self.queue.enqueue(msg)
         else:
@@ -1314,11 +1326,67 @@ class Irc(IrcCommandDispatcher, log.Firewalled):
 
     def sendMsg(self, msg):
         """Queues a message to be sent to the server *immediately*"""
+        if msg.command.upper() == 'BATCH':
+            log.error('Tried to send a BATCH message using sendMsg '
+                      'instead of queueBatch: %r', msg)
         if not self.zombie:
             self.fastqueue.enqueue(msg)
         else:
             log.warning('Refusing to send %r; %s is a zombie.', msg, self)
 
+    def queueBatch(self, msgs):
+        """Queues a batch of messages to be sent to the server.
+        See <https://ircv3.net/specs/extensions/batch-3.2>
+
+        queueMsg/sendMsg must not be used repeatedly to send a batch, because
+        they do not guarantee the batch is send atomically, which is
+        required because "Clients MUST NOT send messages other than PRIVMSG
+        while a multiline batch is open."
+        -- <https://ircv3.net/specs/extensions/multiline>
+        """
+        if not conf.supybot.protocols.irc.experimentalExtensions():
+            raise ValueError(
+                'queueBatch is disabled because it depends on draft '
+                'IRC specifications. If you know what you are doing, '
+                'set supybot.protocols.irc.experimentalExtensions.')
+
+        if len(msgs) < 2:
+            raise ValueError(
+                'queueBatch called with less than two messages.')
+        if msgs[0].command.upper() != 'BATCH' or msgs[0].args[0][0] != '+':
+            raise ValueError(
+                'queueBatch called with non-"BATCH +" as first message.')
+        if msgs[-1].command.upper() != 'BATCH' or msgs[-1].args[0][0] != '-':
+            raise ValueError(
+                'queueBatch called with non-"BATCH -" as last message.')
+
+        batch_name = msgs[0].args[0][1:]
+
+        if msgs[-1].args[0][1:] != batch_name:
+            raise ValueError(
+                'queueBatch called with mismatched BATCH name args.')
+        if any(msg.server_tags['batch'] != batch_name for msg in msgs[1:-1]):
+            raise ValueError(
+                'queueBatch called with mismatched batch names.')
+            return
+        if batch_name in self._queued_batches:
+            raise ValueError(
+                'queueBatch called with a batch name already in flight')
+
+        self._queued_batches[batch_name] = msgs
+
+        # Enqueue only the start of the batch. When takeMsg sees it, it will
+        # enqueue the full batch in self.fastqueue.
+        # We don't enqueue the full batch in self.fastqueue here, because
+        # there is no reason for this batch to jump in front of all other
+        # queued messages.
+        # TODO: the batch will be ordered with the priority of a BATCH
+        # message (ie. normal), but if the batch is made only of low-priority
+        # messages like PRIVMSG, it should have that priority.
+        # (or maybe order on the batch type instead of commands inside
+        # the batch?)
+        self.queue.enqueue(msgs[0])
+
     def _truncateMsg(self, msg):
         msg_str = str(msg)
         if msg_str[0] == '@':
@@ -1381,7 +1449,37 @@ class Irc(IrcCommandDispatcher, log.Firewalled):
                 now = str(int(now))
                 self.outstandingPing = True
                 self.queueMsg(ircmsgs.ping(now))
+
         if msg:
+            if msg.command.upper() == 'BATCH':
+                if not conf.supybot.protocols.irc.experimentalExtensions():
+                    log.error('Dropping outgoing batch. '
+                              'supybot.protocols.irc.experimentalExtensions '
+                              'is disabled, so plugins should not send '
+                              'batches. This is a bug, please report it.')
+                    return None
+                if msg.args[0].startswith('+'):
+                    # Start of a batch; created by self.queueBatch. We need to
+                    # *prepend* the rest of the batch to the fastqueue
+                    # so that no other message is sent while the batch is
+                    # open.
+                    # "Clients MUST NOT send messages other than PRIVMSG while
+                    # a multiline batch is open."
+                    # -- <https://ircv3.net/specs/extensions/multiline>
+                    #
+                    # (Yes, *prepend* to the queue. Fortunately, it should be
+                    # empty, because BATCH cannot be queued in the fastqueue
+                    # and we just got a BATCH, which means it's from the
+                    # regular queue, which means the fastqueue is empty.
+                    # But let's not take any risk, eg. if race condition
+                    # with a plugin appending directly to the fastqueue.)
+                    batch_name = msg.args[0][1:]
+                    batch_messages = self._queued_batches.pop(batch_name)
+                    if batch_messages[0] != msg:
+                        log.error('Enqueue "BATCH +" message does not match '
+                                  'the one of the batch in flight.')
+                    self.fastqueue[:0] = batch_messages[1:]
+
             if not world.testing and 'label' not in msg.server_tags \
                     and 'labeled-response' in self.state.capabilities_ack:
                 # Not adding labels while testing, because it would break

src/ircutils.py

@@ -936,14 +936,22 @@ class AuthenticateDecoder(object):
         return base64.b64decode(b''.join(self.chunks))
 
 
-def parseStsPolicy(logger, policy, parseDuration):
-    parsed_policy = {}
-    for kv in policy.split(','):
+def parseCapabilityKeyValue(s):
+    """Parses a key-value string, in the format used by 'sts' and
+    'draft/multiline."""
+    d = {}
+    for kv in s.split(','):
         if '=' in kv:
             (k, v) = kv.split('=', 1)
-            parsed_policy[k] = v
+            d[k] = v
         else:
-            parsed_policy[kv] = None
+            d[kv] = None
+
+    return d
+
+
+def parseStsPolicy(logger, policy, parseDuration):
+    parsed_policy = parseCapabilityKeyValue(policy)
 
     for key in ('port', 'duration'):
         if key == 'duration' and not parseDuration:

src/utils/iter.py

@@ -161,4 +161,14 @@ def limited(iterable, limit):
         raise ValueError('Expected %s elements in iterable (%r), got %s.' % \
               (limit, iterable, limit-i))
 
+
+def grouper(iterable, n, fillvalue=None):
+    """Collect data into fixed-length chunks or blocks
+
+    grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx
+
+    From https://docs.python.org/3/library/itertools.html#itertools-recipes"""
+    args = [iter(iterable)] * n
+    return zip_longest(*args, fillvalue=fillvalue)
+
 # vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79:

test/test_callbacks.py

@@ -318,7 +318,6 @@ class FunctionsTestCase(SupyTestCase):
         irc.state.supported['statusmsg'] = '+'
         msg = ircmsgs.privmsg('+#foo', 'bar baz', prefix=prefix)
         irc._tagMsg(msg)
-        print(msg.channel)
         self.assertEqual(ircmsgs.privmsg('+#foo', '%s: foo' % msg.nick),
                          callbacks._makeReply(irc, msg, 'foo'))
 
@@ -536,7 +535,6 @@ class PrivmsgTestCase(ChannelPluginTestCase):
 
     def testReplyInstant(self):
         self.assertNoResponse(' ')
-        print(conf.supybot.reply.mores.instant())
         self.assertResponse(
             "eval 'foo '*300",
             "'" + "foo " * 110 + " \x02(2 more messages)\x02")
@@ -695,6 +693,234 @@ class PrivmsgTestCase(ChannelPluginTestCase):
             conf.supybot.reply.whenNotCommand.set(original)
 
 
+class MultilinePrivmsgTestCase(ChannelPluginTestCase):
+    plugins = ('Utilities', 'Misc', 'Web', 'String')
+    conf.allowEval = True
+
+    def setUp(self):
+        super().setUp()
+
+        # Enable multiline
+        self.irc.state.capabilities_ack.add('batch')
+        self.irc.state.capabilities_ack.add('draft/multiline')
+        self.irc.state.capabilities_ls['draft/multiline'] = 'max-bytes=4096'
+
+        # Enable msgid and +draft/reply
+        self.irc.state.capabilities_ack.add('message-tags')
+
+        conf.supybot.protocols.irc.experimentalExtensions.setValue(True)
+
+    def tearDown(self):
+        conf.supybot.protocols.irc.experimentalExtensions.setValue(False)
+        conf.supybot.reply.mores.instant.setValue(1)
+        super().tearDown()
+
+    def testReplyInstantSingle(self):
+        self.assertIsNone(self.irc.takeMsg())
+
+        # Single message as reply, no batch
+        self.irc.feedMsg(ircmsgs.privmsg(
+            self.channel, "@eval 'foo '*300", prefix=self.prefix))
+        m = self.irc.takeMsg()
+        self.assertEqual(
+            m, ircmsgs.IrcMsg(command='PRIVMSG', args=(self.channel,
+                "test: '" + "foo " * 110 + " \x02(2 more messages)\x02")))
+        self.assertIsNone(self.irc.takeMsg())
+
+    def testReplyInstantBatchPartial(self):
+        """response is shared as a batch + (1 more message)"""
+        self.assertIsNone(self.irc.takeMsg())
+
+        conf.supybot.reply.mores.instant.setValue(2)
+        self.irc.feedMsg(ircmsgs.privmsg(
+            self.channel, "@eval 'foo '*300", prefix=self.prefix))
+
+        # First message opens the batch
+        m = self.irc.takeMsg()
+        self.assertEqual(m.command, 'BATCH', m)
+        batch_name = m.args[0][1:]
+        self.assertEqual(
+            m, ircmsgs.IrcMsg(command='BATCH',
+                args=('+' + batch_name,
+                    'draft/multiline', self.channel)))
+
+        # Second message, first PRIVMSG
+        m = self.irc.takeMsg()
+        self.assertEqual(
+            m, ircmsgs.IrcMsg(command='PRIVMSG',
+                args=(self.channel, "test: '" + "foo " * 110),
+                server_tags={'batch': batch_name}))
+
+        # Third message, last PRIVMSG
+        m = self.irc.takeMsg()
+        self.assertEqual(
+            m, ircmsgs.IrcMsg(command='PRIVMSG',
+                args=(self.channel,
+                    "test: " + "foo " * 111 + "\x02(1 more message)\x02"),
+                server_tags={'batch': batch_name,
+                    'draft/multiline-concat': None}))
+
+        # Last message, closes the batch
+        m = self.irc.takeMsg()
+        self.assertEqual(
+            m, ircmsgs.IrcMsg(command='BATCH', args=(
+                '-' + batch_name,)))
+
+    def testReplyInstantBatchFull(self):
+        """response is entirely instant"""
+        self.assertIsNone(self.irc.takeMsg())
+
+        conf.supybot.reply.mores.instant.setValue(3)
+        self.irc.feedMsg(ircmsgs.privmsg(
+            self.channel, "@eval 'foo '*300", prefix=self.prefix))
+
+        # First message opens the batch
+        m = self.irc.takeMsg()
+        self.assertEqual(m.command, 'BATCH', m)
+        batch_name = m.args[0][1:]
+        self.assertEqual(
+            m, ircmsgs.IrcMsg(command='BATCH',
+                args=('+' + batch_name,
+                    'draft/multiline', self.channel)))
+
+        # Second message, first PRIVMSG
+        m = self.irc.takeMsg()
+        self.assertEqual(
+            m, ircmsgs.IrcMsg(command='PRIVMSG',
+                args=(self.channel, "test: '" + "foo " * 110),
+                server_tags={'batch': batch_name}))
+
+        # Third message, a PRIVMSG
+        m = self.irc.takeMsg()
+        self.assertEqual(
+            m, ircmsgs.IrcMsg(command='PRIVMSG',
+                args=(self.channel,
+                    "test: " + "foo " * 110 + "foo"),
+                server_tags={'batch': batch_name,
+                    'draft/multiline-concat': None}))
+
+        # Fourth message, last PRIVMSG
+        m = self.irc.takeMsg()
+        self.assertEqual(
+            m, ircmsgs.IrcMsg(command='PRIVMSG',
+                args=(self.channel,
+                    "test:  " + "foo " * 79 + "'"),
+                server_tags={'batch': batch_name,
+                    'draft/multiline-concat': None}))
+
+        # Last message, closes the batch
+        m = self.irc.takeMsg()
+        self.assertEqual(
+            m, ircmsgs.IrcMsg(command='BATCH', args=(
+                '-' + batch_name,)))
+
+    def testReplyInstantBatchFullMaxBytes(self):
+        """response is entirely instant, but does not fit in a single batch"""
+        self.irc.state.capabilities_ls['draft/multiline'] = 'max-bytes=900'
+        self.assertIsNone(self.irc.takeMsg())
+
+        conf.supybot.reply.mores.instant.setValue(3)
+        self.irc.feedMsg(ircmsgs.privmsg(
+            self.channel, "@eval 'foo '*300", prefix=self.prefix))
+
+        # First message opens the first batch
+        m = self.irc.takeMsg()
+        self.assertEqual(m.command, 'BATCH', m)
+        batch_name = m.args[0][1:]
+        self.assertEqual(
+            m, ircmsgs.IrcMsg(command='BATCH',
+                args=('+' + batch_name,
+                    'draft/multiline', self.channel)))
+
+        # Second message, first PRIVMSG
+        m = self.irc.takeMsg()
+        self.assertEqual(
+            m, ircmsgs.IrcMsg(command='PRIVMSG',
+                args=(self.channel, "test: '" + "foo " * 110),
+                server_tags={'batch': batch_name}))
+
+        # Third message, a PRIVMSG
+        m = self.irc.takeMsg()
+        self.assertEqual(
+            m, ircmsgs.IrcMsg(command='PRIVMSG',
+                args=(self.channel,
+                    "test: " + "foo " * 110 + "foo"),
+                server_tags={'batch': batch_name,
+                    'draft/multiline-concat': None}))
+
+        # closes the first batch
+        m = self.irc.takeMsg()
+        self.assertEqual(
+            m, ircmsgs.IrcMsg(command='BATCH', args=(
+                '-' + batch_name,)))
+
+        # opens the second batch
+        m = self.irc.takeMsg()
+        self.assertEqual(m.command, 'BATCH', m)
+        batch_name = m.args[0][1:]
+        self.assertEqual(
+            m, ircmsgs.IrcMsg(command='BATCH',
+                args=('+' + batch_name,
+                    'draft/multiline', self.channel)))
+
+        # last PRIVMSG (and also the first of its batch)
+        m = self.irc.takeMsg()
+        self.assertEqual(
+            m, ircmsgs.IrcMsg(command='PRIVMSG',
+                args=(self.channel,
+                    "test:  " + "foo " * 79 + "'"),
+                server_tags={'batch': batch_name}))
+
+        # Last message, closes the second batch
+        m = self.irc.takeMsg()
+        self.assertEqual(
+            m, ircmsgs.IrcMsg(command='BATCH', args=(
+                '-' + batch_name,)))
+
+    def testReplyInstantBatchTags(self):
+        """check a message's tags are 'lifted' to the initial BATCH
+        command."""
+        self.assertIsNone(self.irc.takeMsg())
+
+        conf.supybot.reply.mores.instant.setValue(2)
+        m = ircmsgs.privmsg(
+            self.channel, "@eval 'foo '*300", prefix=self.prefix)
+        m.server_tags['msgid'] = 'initialmsgid'
+        self.irc.feedMsg(m)
+
+        # First message opens the batch
+        m = self.irc.takeMsg()
+        self.assertEqual(m.command, 'BATCH', m)
+        batch_name = m.args[0][1:]
+        self.assertEqual(
+            m, ircmsgs.IrcMsg(command='BATCH',
+                args=('+' + batch_name,
+                    'draft/multiline', self.channel),
+                server_tags={'+draft/reply': 'initialmsgid'}))
+
+        # Second message, first PRIVMSG
+        m = self.irc.takeMsg()
+        self.assertEqual(
+            m, ircmsgs.IrcMsg(command='PRIVMSG',
+                args=(self.channel, "test: '" + "foo " * 110),
+                server_tags={'batch': batch_name}))
+
+        # Third message, last PRIVMSG
+        m = self.irc.takeMsg()
+        self.assertEqual(
+            m, ircmsgs.IrcMsg(command='PRIVMSG',
+                args=(self.channel,
+                    "test: " + "foo " * 111 + "\x02(1 more message)\x02"),
+                server_tags={'batch': batch_name,
+                    'draft/multiline-concat': None}))
+
+        # Last message, closes the batch
+        m = self.irc.takeMsg()
+        self.assertEqual(
+            m, ircmsgs.IrcMsg(command='BATCH', args=(
+                '-' + batch_name,)))
+
+
 class PluginRegexpTestCase(PluginTestCase):
     plugins = ()
     class PCAR(callbacks.PluginRegexp):

test/test_irclib.py

@@ -1131,6 +1131,153 @@ class IrcTestCase(SupyTestCase):
                 str(m), 'PRIVMSG #test :%s\r\n' % remaining_payload)
 
 
+class BatchTestCase(SupyTestCase):
+    def setUp(self):
+        self.irc = irclib.Irc('test')
+        conf.supybot.protocols.irc.experimentalExtensions.setValue(True)
+        while self.irc.takeMsg() is not None:
+            self.irc.takeMsg()
+
+    def tearDown(self):
+        conf.supybot.protocols.irc.experimentalExtensions.setValue(False)
+
+    def testQueueBatch(self):
+        """Basic operation of queueBatch"""
+        msgs = [
+            ircmsgs.IrcMsg('BATCH +label batchtype'),
+            ircmsgs.IrcMsg('@batch=label PRIVMSG #channel :hello'),
+            ircmsgs.IrcMsg('@batch=label PRIVMSG #channel :there'),
+            ircmsgs.IrcMsg('BATCH -label'),
+        ]
+
+        self.irc.queueBatch(copy.deepcopy(msgs))
+        for msg in msgs:
+            self.assertEqual(msg, self.irc.takeMsg())
+
+    def testQueueBatchStartMinus(self):
+        msgs = [
+            ircmsgs.IrcMsg('BATCH -label batchtype'),
+            ircmsgs.IrcMsg('@batch=label PRIVMSG #channel :hello'),
+            ircmsgs.IrcMsg('BATCH -label'),
+        ]
+
+        with self.assertRaises(ValueError):
+            self.irc.queueBatch(msgs)
+        self.assertIsNone(self.irc.takeMsg())
+
+    def testQueueBatchEndPlus(self):
+        msgs = [
+            ircmsgs.IrcMsg('BATCH +label batchtype'),
+            ircmsgs.IrcMsg('@batch=label PRIVMSG #channel :hello'),
+            ircmsgs.IrcMsg('BATCH +label'),
+        ]
+
+        with self.assertRaises(ValueError):
+            self.irc.queueBatch(msgs)
+        self.assertIsNone(self.irc.takeMsg())
+
+    def testQueueBatchMismatchStartEnd(self):
+        msgs = [
+            ircmsgs.IrcMsg('BATCH +label batchtype'),
+            ircmsgs.IrcMsg('@batch=label PRIVMSG #channel :hello'),
+            ircmsgs.IrcMsg('BATCH -label2'),
+        ]
+
+        with self.assertRaises(ValueError):
+            self.irc.queueBatch(msgs)
+        self.assertIsNone(self.irc.takeMsg())
+
+    def testQueueBatchMismatchInner(self):
+        msgs = [
+            ircmsgs.IrcMsg('BATCH +label batchtype'),
+            ircmsgs.IrcMsg('@batch=label2 PRIVMSG #channel :hello'),
+            ircmsgs.IrcMsg('BATCH -label'),
+        ]
+
+        with self.assertRaises(ValueError):
+            self.irc.queueBatch(msgs)
+        self.assertIsNone(self.irc.takeMsg())
+
+    def testQueueBatchTwice(self):
+        """Basic operation of queueBatch"""
+        all_msgs = []
+        for label in ('label1', 'label2'):
+            msgs = [
+                ircmsgs.IrcMsg('BATCH +%s batchtype' % label),
+                ircmsgs.IrcMsg('@batch=%s PRIVMSG #channel :hello' % label),
+                ircmsgs.IrcMsg('@batch=%s PRIVMSG #channel :there' % label),
+                ircmsgs.IrcMsg('BATCH -%s' % label),
+            ]
+            all_msgs.extend(msgs)
+            self.irc.queueBatch(copy.deepcopy(msgs))
+
+        for msg in all_msgs:
+            self.assertEqual(msg, self.irc.takeMsg())
+
+    def testQueueBatchDuplicate(self):
+        msgs = [
+            ircmsgs.IrcMsg('BATCH +label batchtype'),
+            ircmsgs.IrcMsg('@batch=label PRIVMSG #channel :hello'),
+            ircmsgs.IrcMsg('BATCH -label'),
+        ]
+
+        self.irc.queueBatch(copy.deepcopy(msgs))
+
+        with self.assertRaises(ValueError):
+            self.irc.queueBatch(copy.deepcopy(msgs))
+
+        for msg in msgs:
+            self.assertEqual(msg, self.irc.takeMsg())
+        self.assertIsNone(self.irc.takeMsg())
+
+    def testQueueBatchReuse(self):
+        """We can reuse the same label after the batch is closed."""
+        msgs = [
+            ircmsgs.IrcMsg('BATCH +label batchtype'),
+            ircmsgs.IrcMsg('@batch=label PRIVMSG #channel :hello'),
+            ircmsgs.IrcMsg('BATCH -label'),
+        ]
+
+        self.irc.queueBatch(copy.deepcopy(msgs))
+        for msg in msgs:
+            self.assertEqual(msg, self.irc.takeMsg())
+
+        self.irc.queueBatch(copy.deepcopy(msgs))
+        for msg in msgs:
+            self.assertEqual(msg, self.irc.takeMsg())
+
+    def testBatchInterleaved(self):
+        """Make sure it's not possible for an unrelated message to be sent
+        while a batch is open"""
+        msgs = [
+            ircmsgs.IrcMsg('BATCH +label batchtype'),
+            ircmsgs.IrcMsg('@batch=label PRIVMSG #channel :hello'),
+            ircmsgs.IrcMsg('BATCH -label'),
+        ]
+        msg = ircmsgs.IrcMsg('PRIVMSG #channel :unrelated message')
+
+        with self.subTest('sendMsg called before "BATCH +" is dequeued'):
+            self.irc.queueBatch(copy.deepcopy(msgs))
+            self.irc.sendMsg(msg)
+
+            self.assertEqual(msg, self.irc.takeMsg())
+            self.assertEqual(msgs[0], self.irc.takeMsg())
+            self.assertEqual(msgs[1], self.irc.takeMsg())
+            self.assertEqual(msgs[2], self.irc.takeMsg())
+            self.assertIsNone(self.irc.takeMsg())
+
+        with self.subTest('sendMsg called after "BATCH +" is dequeued'):
+            self.irc.queueBatch(copy.deepcopy(msgs))
+            self.assertEqual(msgs[0], self.irc.takeMsg())
+
+            self.irc.sendMsg(msg)
+
+            self.assertEqual(msgs[1], self.irc.takeMsg())
+            self.assertEqual(msgs[2], self.irc.takeMsg())
+            self.assertEqual(msg, self.irc.takeMsg())
+            self.assertIsNone(self.irc.takeMsg())
+
+
 class SaslTestCase(SupyTestCase, CapNegMixin):
     def setUp(self):
         pass