net_imap: Finish implementing missing or broken parts of FETCH BODY[]…

… command.

Major refactoring and additions in order to better comply with IMAP4rev1:

* Make MIME structure in mod_mimeparse reusable across operations,
  which can increase efficiency.
* Allow multiple BODY[]/BODY.PEEK[] items to be requested in FETCH.
  As part of this, completely refactor process_fetch_finalize.
* Implement part specifier support. This was a major issue previously
  that would cause some clients to fail to retrieve certain messages.
* Fix bugs with partial fetch support.
* Never return BODY.PEEK in responses.
* Wait for final 220 responses in any SMTP-interacting tests.
* Add more comprehensive tests for FETCH BODY[].

bbs/socket.c

@@ -34,7 +34,6 @@
 #include <poll.h>
 
 #ifdef __linux__
-#define SOL_TCP  6 /* TCP level */
 /* <linux/tcp.h> includes some of the same stuff as <netinet/tcp.h>, so don't include both, just one or the other */
 #include <linux/tcp.h>
 #else
@@ -315,6 +314,24 @@ int bbs_block_fd(int fd)
 	return 0;
 }
 
+int bbs_node_cork(struct bbs_node *node, int enabled)
+{
+	int i = enabled;
+	/* This is TCP-specific, so it has to operate on the actual socket file descriptor,
+	 * not any of the other node file descriptors. */
+#ifdef __FreeBSD__
+	if (setsockopt(node->sfd, IPPROTO_TCP, TCP_NOPUSH, &i, sizeof(i))) { /* FreeBSD */
+#else
+	if (setsockopt(node->sfd, IPPROTO_TCP, TCP_CORK, &i, sizeof(i))) { /* Linux */
+#endif
+		bbs_error("setsockopt failed: %s\n", strerror(errno));
+		return -1;
+	} else {
+		bbs_debug(3, "%s corking on node %d\n", enabled ? "Enabled" : "Disabled", node->id);
+	}
+	return 0;
+}
+
 int bbs_set_fd_tcp_nodelay(int fd, int enabled)
 {
 	int i = enabled;
@@ -380,7 +397,7 @@ int bbs_node_wait_until_output_sent(struct bbs_node *node)
 	/* If it's the network link that's actually slow,
 	 * then the acks will take some time to come back. */
 	for (;;) {
-		if (getsockopt(node->sfd, SOL_TCP, TCP_INFO, &info, &len)) {
+		if (getsockopt(node->sfd, IPPROTO_TCP, TCP_INFO, &info, &len)) {
 			bbs_error("getsockopt failed: %s\n", strerror(errno));
 			return -1;
 		}

include/mod_mimeparse.h

@@ -1,22 +1,52 @@
-/*
- * LBBS -- The Lightweight Bulletin Board System
- *
- * Copyright (C) 2023, Naveen Albert
- *
- * Naveen Albert <[email protected]>
- *
- */
-
-/*! \file
- *
- * \brief MIME Parser Supplements for IMAP Server
- *
- */
-
-/*!
- * \brief Generate the BODY/BODYSTRUCTURE data item for FETCH responses
- * \param itemname BODY or BODYSTRUCTURE
- * \param file File containing email message
- * \returns NULL on failure, BODYSTRUCTURE text on success, which must be be freed using free()
- */
-char *mime_make_bodystructure(const char *itemname, const char *file);
+/*
+ * LBBS -- The Lightweight Bulletin Board System
+ *
+ * Copyright (C) 2023, Naveen Albert
+ *
+ * Naveen Albert <[email protected]>
+ *
+ */
+
+/*! \file
+ *
+ * \brief MIME Parser Supplements for IMAP Server
+ *
+ */
+
+struct bbs_mime_message;
+
+/*!
+ * \brief Create a MIME message structure by parsing a message file
+ * \param filename Path to email message
+ * \return NULL on failure, opaque MIME structure on success which must be destroyed using bbs_mime_message_parse
+ */
+struct bbs_mime_message *bbs_mime_message_parse(const char *filename) __attribute__((nonnull (1)));
+
+/*!
+ * \brief Destroy a MIME message structure
+ */
+void bbs_mime_message_destroy(struct bbs_mime_message *mime) __attribute__((nonnull (1)));
+
+/*!
+ * \brief Generate the BODY/BODYSTRUCTURE data item for FETCH responses
+ * \param mime
+ * \returns NULL on failure, BODYSTRUCTURE text on success, which must be be freed using free()
+ */
+char *bbs_mime_make_bodystructure(struct bbs_mime_message *mime) __attribute__((nonnull (1)));
+
+enum mime_part_filter {
+	MIME_PART_FILTER_ALL = 0,		/* Retrieve the entire part */
+	MIME_PART_FILTER_MIME,			/* Retrieve just the MIME of the part */
+	MIME_PART_FILTER_HEADERS,		/* Retrieve the headers, but only if the part's Content-Type is message/rfc822 */
+	MIME_PART_FILTER_TEXT,			/* Retrieve the text, but only if the part's Content-Type is message/rfc822 */
+};
+
+/*!
+ * \brief Retrieve a particular part of the body, by part specification
+ * \param mime
+ * \param spec Part specification, e.g. 2.3. This should be ONLY the part number portion of the spec (e.g. not including .MIME, .TEXT, etc.)
+ * \param[out] outlen Length of returned part, if return value is non-NULL
+ * \param[in] filter What to retrieve and return
+ * \returns NULL on failure, requested section on success, which must be freed using free()
+ */
+char *bbs_mime_get_part(struct bbs_mime_message *mime, const char *spec, size_t *restrict outlen, enum mime_part_filter filter) __attribute__((nonnull (1, 2, 3)));

include/socket.h

@@ -55,6 +55,16 @@ int bbs_unblock_fd(int fd);
 /*! \brief Put a socket in blocking mode */
 int bbs_block_fd(int fd);
 
+/*!
+ * \brief Cork or uncork a node's TCP session
+ * \param node
+ * \param enabled 1 to buffer data in the kernel until full packets are available to send, 0 to disable
+ * \note If enabled, this MUST be disabled at some point to ensure pending data is fully written!
+ * \note You should not use this function unless no better alternative is available, use with caution!
+ * \retval 0 on success, -1 on failure
+ */
+int bbs_node_cork(struct bbs_node *node, int enabled);
+
 /*!
  * \brief Enable or disable Nagle's algorithm
  * \param fd

modules/mod_mimeparse.c

@@ -17,7 +17,7 @@
  * \author Naveen Albert <[email protected]>
  */
 
-/* XXX Not sure which this is necessary, but it is... */
+/* XXX Not sure why this is necessary, but it is... */
 #define BBS_LOCK_WRAPPERS_NOWARN
 
 #include "include/bbs.h"
@@ -343,20 +343,13 @@ static void write_part_bodystructure(GMimeObject *part, GString *gs, int level)
 	g_string_append_c(gs, ')');
 }
 
-char *mime_make_bodystructure(const char *itemname, const char *file)
+static GMimeMessage *mk_mime(const char *file)
 {
 	GMimeFormat format = GMIME_FORMAT_MESSAGE;
 	GMimeMessage *message;
 	GMimeParser *parser;
 	GMimeStream *stream;
-	GString *str;
-	gchar *result;
 	int fd;
-#ifdef CHECK_VALIDITY
-	int p = 0;
-	int in_quoted = 0;
-	char *s;
-#endif
 
 	fd = open(file, O_RDONLY, 0);
 	if (fd < 0) {
@@ -376,12 +369,45 @@ char *mime_make_bodystructure(const char *itemname, const char *file)
 	close(fd);
 
 	if (!message) {
-		bbs_error("Failed to parse message as MIME\n");
+		bbs_error("Failed to parse message %s as MIME\n", file);
+	}
+
+	return message;
+}
+
+/* Opaque structure for MIME message, so other modules can reuse this for multiple operations */
+struct bbs_mime_message {
+	GMimeMessage *message;
+};
+
+struct bbs_mime_message *bbs_mime_message_parse(const char *filename)
+{
+	struct bbs_mime_message *mime = malloc(sizeof(*mime));
+	if (ALLOC_FAILURE(mime)) {
 		return NULL;
 	}
+	mime->message = mk_mime(filename);
+	return mime;
+}
+
+void bbs_mime_message_destroy(struct bbs_mime_message *mime)
+{
+	g_object_unref(mime->message);
+	free(mime);
+}
+
+char *bbs_mime_make_bodystructure(struct bbs_mime_message *mime)
+{
+	GMimeMessage *message = mime->message;
+	GString *str;
+	gchar *result;
+#ifdef CHECK_VALIDITY
+	int p = 0;
+	int in_quoted = 0;
+	char *s;
+#endif
 
 	str = g_string_new("");
-	g_string_append_printf(str, "%s ", itemname);
 	write_part_bodystructure(message->mime_part, str, 0);
 
 #ifdef CHECK_VALIDITY
@@ -420,10 +446,154 @@ char *mime_make_bodystructure(const char *itemname, const char *file)
 #endif
 
 	result = g_string_free(str, FALSE); /* Free the g_string but not the buffer */
-	g_object_unref(message);
 	return result; /* gchar is just a typedef for char, so this returns a char */
 }
 
+#if SEMVER_VERSION(GMIME_MAJOR_VERSION, GMIME_MINOR_VERSION, GMIME_MICRO_VERSION) < SEMVER_VERSION(3, 2, 8)
+/* This function was only added in gmime commit 3efc24a6cdc88198e11e43f23e03e32f12b13bd8,
+ * so older packages of the library don't have it. Define it manually for those cases. */
+static ssize_t g_mime_object_write_content_to_stream(GMimeObject *object, GMimeFormatOptions *options, GMimeStream *stream)
+{
+	g_return_val_if_fail (GMIME_IS_OBJECT (object), -1);
+	g_return_val_if_fail (GMIME_IS_STREAM (stream), -1);
+
+	return GMIME_OBJECT_GET_CLASS(object)->write_to_stream(object, options, TRUE, stream);
+}
+#endif
+
+static void write_part(GMimeObject *part, GMimeStream *stream, enum mime_part_filter filter)
+{
+	char *buf;
+	GMimeMessage *message;
+	GMimeContentType *content_type;
+	const char *subtype;
+	GMimeFormatOptions *format = g_mime_format_options_get_default();
+
+#define IS_RFC822(part) (!strcasecmp(g_mime_content_type_get_media_type(content_type), "message") && subtype && !strcasecmp(subtype, "rfc822"))
+
+	content_type = g_mime_object_get_content_type(part);
+	subtype = g_mime_content_type_get_media_subtype(content_type);
+
+	switch (filter) {
+	case MIME_PART_FILTER_TEXT:
+		/* If the type is message/rfc822, proceed. If not, illegal (just return empty). */
+		if (!IS_RFC822(part)) {
+			bbs_debug(1, "Ignoring request for TEXT, since its type is %s/%s\n", g_mime_content_type_get_media_type(content_type), S_IF(subtype));
+			return;
+		}
+		/* Might be a more elegant way to do this with gmime, but not sure what it is at the moment.
+		 * For now, return the whole thing, and later strip out the headers. */
+		/* Fall through */
+	case MIME_PART_FILTER_ALL:
+		if (g_mime_object_write_content_to_stream((GMimeObject *) part, format, stream) == -1) {
+			bbs_warning("Failed to write part to stream\n");
+		}
+		break;
+	case MIME_PART_FILTER_MIME:
+		/* XXX In theory, since we get a string, we could return this directly,
+		 * rather than adding to a stream and then allocating another string.
+		 * However, the stream allows us to be a little more abstract here. */
+		buf = g_mime_object_get_headers(part, format);
+		if (!strlen_zero(buf)) {
+			g_mime_stream_printf(stream, "%s\r\n", buf); /* Need to include end of headers */
+		}
+		g_free(buf);
+		break;
+	case MIME_PART_FILTER_HEADERS:
+		/* If the type is message/rfc822, proceed. If not, illegal (just return empty). */
+		if (!IS_RFC822(part)) {
+			bbs_debug(1, "Ignoring request for HEADERS, since its type is %s/%s\n", g_mime_content_type_get_media_type(content_type), S_IF(subtype));
+			return;
+		}
+		/* XXX Same thing here about returning a string directly */
+		message = g_mime_message_part_get_message((GMimeMessagePart *) part);
+		buf = g_mime_object_get_headers((GMimeObject *) message, format);
+		if (!strlen_zero(buf)) {
+			g_mime_stream_printf(stream, "%s\r\n", buf); /* Need to include end of headers */
+		}
+		g_free(buf);
+		break;
+	}
+}
+
+/*! * \brief Get the contents of a MIME part by part number */
+static int get_part(GMimeMessage *message, GMimeStream *mem, const char *spec, enum mime_part_filter filter)
+{
+	GMimePartIter *iter;
+	GMimeObject *part;
+
+	iter = g_mime_part_iter_new((GMimeObject *) message);
+	if (!g_mime_part_iter_is_valid(iter)) {
+		bbs_warning("Part iteration is invalid\n");
+		g_mime_part_iter_free(iter);
+		return -1;
+	}
+	if (!g_mime_part_iter_jump_to(iter, spec)) {
+		bbs_warning("Failed to fetch part number %s\n", spec);
+		g_mime_part_iter_free(iter);
+		return -1;
+	}
+
+	part = g_mime_part_iter_get_current(iter);
+	write_part(part, mem, filter);
+	g_mime_part_iter_free(iter);
+	return 0;
+}
+
+char *bbs_mime_get_part(struct bbs_mime_message *mime, const char *spec, size_t *restrict outlen, enum mime_part_filter filter)
+{
+	GMimeMessage *message = mime->message;
+	GMimeStream *mem;
+	GByteArray *buffer;
+	char *buf;
+	unsigned char *bufdata;
+	size_t buflen;
+
+	mem = g_mime_stream_mem_new();
+	if (!mem) {
+		bbs_error("Failed to allocate stream buffer\n");
+		return NULL;
+	}
+
+	if (get_part(message, mem, spec, filter)) {
+		g_object_unref(mem);
+		return NULL;
+	}
+
+	buffer = g_mime_stream_mem_get_byte_array((GMimeStreamMem *) mem);
+
+	if (filter == MIME_PART_FILTER_TEXT) {
+		char *eoh;
+		size_t diff;
+		/* Now we have to pay the piper... skip past the headers. */
+		eoh = memmem(buffer->data, buffer->len, "\r\n\r\n", STRLEN("\r\n\r\n"));
+		if (!eoh) {
+			bbs_debug(3, "Message has no body, just headers, if that...\n");
+			g_object_unref(mem);
+			return NULL;
+		}
+		diff = (size_t) (eoh - (char*) buffer->data);
+		diff += STRLEN("\r\n\r\n");
+		bufdata = buffer->data + diff;
+		buflen = buffer->len - diff;
+	} else {
+		bufdata = buffer->data;
+		buflen = buffer->len;
+	}
+
+	buf = malloc(buflen + 1);
+	if (ALLOC_FAILURE(buf)) {
+		g_object_unref(mem);
+		return NULL;
+	}
+	memcpy(buf, bufdata, buflen);
+	buf[buflen] = '\0';
+	*outlen = buflen;
+	g_object_unref(mem);
+
+	return buf; /* gchar is just a typedef for char, so this returns a char */
+}
+
 static int load_module(void)
 {
 	g_mime_init();
@@ -434,7 +604,7 @@ static int load_module(void)
 
 static int unload_module(void)
 {
-	/* This doesn't free everything (possibly lost leaks in valgrind . See:
+	/* This doesn't free everything (possibly lost leaks in valgrind). See:
 	 * Q: https://mail.gnome.org/archives/gmime-devel-list/2012-November/msg00000.html
 	 * A: https://mail.gnome.org/archives/gmime-devel-list/2012-November/msg00001.html
 	 */

nets/net_imap/imap.h

@@ -168,8 +168,10 @@ extern int imap_debug_level;
 #define __imap_parallel_reply(imap, fmt, ...) imap_debug(4, "%p <= " fmt, imap, ## __VA_ARGS__); bbs_node_any_fd_writef(imap->node, imap->node->wfd, fmt, ## __VA_ARGS__);
 
 #define _imap_reply_nolock_fd(imap, fd, fmt, ...) imap_debug(4, "%p <= " fmt, imap, ## __VA_ARGS__); bbs_node_fd_writef(imap->node, fd, fmt, ## __VA_ARGS__);
+#define _imap_reply_nolock_fd_lognewline(imap, fd, fmt, ...) imap_debug(4, "%p <= " fmt "\r\n", imap, ## __VA_ARGS__); bbs_node_fd_writef(imap->node, fd, fmt, ## __VA_ARGS__);
 #define _imap_reply_nolock(imap, fmt, ...) _imap_reply_nolock_fd(imap, imap->node->wfd, fmt, ## __VA_ARGS__);
 #define _imap_reply(imap, fmt, ...) _imap_reply_nolock(imap, fmt, ## __VA_ARGS__)
+#define imap_send_nocrlf(imap, fmt, ...) _imap_reply_nolock_fd_lognewline(imap, imap->node->wfd, fmt, ## __VA_ARGS__);
 #define imap_send(imap, fmt, ...) _imap_reply(imap, "%s " fmt "\r\n", "*", ## __VA_ARGS__)
 #define imap_reply(imap, fmt, ...) _imap_reply(imap, "%s " fmt "\r\n", imap->tag, ## __VA_ARGS__)