l2md: add support to pipe new mails to stdout for procmail et al

Add very initial/rough support for piping new mails to external MDAs.

Konstantin says:

  Thanks for working on this -- I've started on a similar tool in the
  past, but got distracted and never completed it. In my implementation,
  it was piping messages to procmail, which allowed writing complex rules
  for folders/pre-processing, etc. May I suggest that your tool also
  offers a stdout that can be piped to procmail?

Used test config:

$ cat ~/.l2mdconfig
[general]
	base = /tmp/test/
	pipe = /usr/bin/procmail
	period = 30

[repo bpf]
	url = https://lore.kernel.org/bpf/0
	initial_import = 10

Tested l2md -> procmail -> mutt locally which seems to work fine.

Suggested-by: Konstantin Ryabitsev <konstantin@linuxfoundation.org>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Link: https://lore.kernel.org/workflows/20190930212410.GE14403@pure.paranoia.local/

10 files changed, 214 insertions, 32 deletions

diff --git a/Makefile b/Makefile
index 8ef22d9..a189938 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
 CFLAGS = -O2 -Wall -Werror
 LDFLAGS = -lgit2
 
-l2md: l2md.o config.o env.o utils.o repo.o mail.o maildir.o
+l2md: l2md.o config.o env.o utils.o repo.o mail.o maildir.o pipe.o
 	$(CC) -o $@ $^ $(LDFLAGS)
 
 install:
diff --git a/README b/README
index e8ac03c..e061e20 100644
--- a/README
+++ b/README
@@ -2,10 +2,13 @@ l2md - lore2maildir
 -------------------
 
 Quick and dirty hack to import lore.kernel.org list archives via git,
-export them in maildir format and keep them periodically synced.
+to export them in maildir format or through a pipe, and to keep them
+periodically synced.
 
 It can then be used in whichever mail client that supports maildir
-format, for example, mutt.
+format, for example, mutt. Alternatively, it can also pipe new mails
+to external MDAs like procmail and friends which then deliver it to
+your favorite mail client eventually.
 
 Essentially, it avoids the need to subscribe to any of the lore lists
 via mail since all messages are now imported through git transport.
@@ -55,20 +58,27 @@ Redirecting to /bin/systemctl status l2md.service
 The muttrc file contains an example mutt config for importing the generated
 maildir directories.
 
-Howto
------
+The procmailrc file contains an example procmail config for getting started,
+there are plenty of howtos online for setting up filtering rules.
+
+For l2md, the repository here ships with two example configs, that is,
+l2mdconfig.maildir and l2mdconfig.procmail. As the name says, the former
+is for exporting new mails in maildir format directly, and the latter is
+one example where l2md pipes new mails via stdin to an external MDA like
+procmail. Copy the one of your choice into ~/.l2mdconfig to get started.
 
-The repo has a few example configs: l2mdconfig and muttrc. The latter
-needs most likey no further explanation, but just to provide an example
-on how to import the directories into mutt.
+Howto 1: maildir
+----------------
 
-The l2mdconfig is an example l2md config which needs to be placed under
-~/.l2mdconfig .
+The l2mdconfig.maildir is an example l2md config which needs to be placed
+under ~/.l2mdconfig . The muttrc is an example config to get started for
+reading the maildir content.
 
 $ cat ~/.l2mdconfig
 [general]
 	maildir = ~/.l2md/maildir/common
 	period = 30
+	base = ~/.l2md/
 
 # bpf@vger.kernel.org list
 [repo bpf]
@@ -85,9 +95,38 @@ The general section contains a sync period in seconds where l2md refetches
 all the git repos and looks for new messages to export into the configured
 maildirs. The maildir under general is a path to a shared maildir where
 l2md exports new mails into. This can also be specified on a per repository
-basis.
+basis. Specifying the maildir under general is optional. It will default
+to ~/.l2md/maildir or <base-path>/maildir if the base deviates from the
+default one. Specifying base is optional as well. This is the working dir
+of l2md where it places its git repos and other meta data for record keeping.
+The default is at ~/.l2md/.
 
 The repo sections with subsequent name define a repository (duh!) with
 one or more git urls to lore and optional maildir export path as mentioned.
 If initial_import is set to >0, then it will only import first x mails upon
 initial repository creation instead of the entire archive.
+
+Howto 2: procmail
+-----------------
+
+The l2mdconfig.procmail is an example l2md config which needs to be placed
+under ~/.l2mdconfig . The procmailrc is an example config to get started
+with a basic config for procmail. The provided muttrc can also be used here
+in order to get started for reading the maildir content preprocessed via
+procmail (the folder needs to point to procmail's MAILDIR of course). Other
+MDAs should work as well, but not tested at this point.
+
+$ cat ~/.l2mdconfig
+[general]
+	base = ~/.l2md/
+	pipe = /usr/bin/procmail
+	period = 30
+
+[repo bpf]
+	url = https://lore.kernel.org/bpf/0
+	initial_import = 1000
+
+See Howto 1 for basics. Instead of maildir, the general section here has a
+setting which is set to pipe. It is pointing to the /usr/bin/procmail MDA in
+the example, and generally executed as the same user as l2md. Similar as with
+maildir, pipe can optionally be specified on a per repository basis.
diff --git a/config.c b/config.c
index 129aa75..d8d55da 100644
--- a/config.c
+++ b/config.c
@@ -71,10 +71,24 @@ static void config_set_basedir(struct config *cfg, const char *dir)
 	wordfree(&p);
 }
 
+static void config_set_ops(struct config *cfg, const struct mail_ops* ops)
+{
+	cfg->ops = ops;
+}
+
+static void config_check_ops(struct config *cfg, const struct mail_ops* ops)
+{
+	if (cfg->ops != ops)
+		panic("mode %s in [general] must match [repo *] mode %s\n",
+		      cfg->ops->name, ops->name);
+}
+
 static void config_set_mode(struct config *cfg, const char *mode)
 {
 	if (!strncmp(mode, "maildir", sizeof("maildir")))
-		cfg->ops = &ops_maildir;
+		config_set_ops(cfg, &ops_maildir);
+	else if (!strncmp(mode, "pipe", sizeof("pipe")))
+		config_set_ops(cfg, &ops_pipe);
 	else
 		panic("Unknown mode: %s!\n", mode);
 }
@@ -130,14 +144,13 @@ static void config_set_defaults(struct config *cfg, const char *homedir)
 {
 	char path[PATH_MAX];
 
-	cfg->ops = &ops_maildir;
 	cfg->general.period = 60;
 
 	slprintf(path, sizeof(path), "%s/.l2md", homedir);
 	strlcpy(cfg->general.base, path, sizeof(cfg->general.base));
 
-	slprintf(path, sizeof(path), "%s/maildir", cfg->general.base);
-	strlcpy(cfg->general.out, path, sizeof(cfg->general.out));
+	config_set_ops(cfg, &ops_maildir);
+	cfg->ops->set_defaults(cfg);
 }
 
 void config_uninit(struct config *cfg)
@@ -202,29 +215,39 @@ struct config *config_init(int argc, char **argv)
 			break;
 
 		case STATE_GENERAL:
-			if (seen[STATE_REPO])
-				panic("[general] config must be before [repo *] config");
-			else if (sscanf(buff, "\tperiod = %u", &val) == 1)
+			if (seen[STATE_REPO]) {
+				panic("[general] config must be before [repo *] config\n");
+			} else if (sscanf(buff, "\tperiod = %u", &val) == 1) {
 				cfg->general.period = val;
-			else if (sscanf(buff, "\tmode = %1023s", tmp) == 1)
+			} else if (sscanf(buff, "\tmode = %1023s", tmp) == 1) {
 				config_set_mode(cfg, tmp);
-			else if (sscanf(buff, "\tmaildir = %1023s", tmp) == 1)
+			} else if (sscanf(buff, "\tmaildir = %1023s", tmp) == 1) {
+				config_set_ops(cfg, &ops_maildir);
 				config_set_out(cfg, tmp, true);
-			else if (sscanf(buff, "\tbase = %1023s", tmp) == 1)
+			} else if (sscanf(buff, "\tpipe = %1023s", tmp) == 1) {
+				config_set_ops(cfg, &ops_pipe);
+				config_set_out(cfg, tmp, true);
+			} else if (sscanf(buff, "\tbase = %1023s", tmp) == 1) {
 				config_set_basedir(cfg, tmp);
-			else
+			} else {
 				goto state_next;
+			}
 			break;
 
 		case STATE_REPO:
-			if (sscanf(buff, "\turl = %1023s", tmp) == 1)
+			if (sscanf(buff, "\turl = %1023s", tmp) == 1) {
 				config_new_url(cfg, tmp);
-			else if (sscanf(buff, "\tmaildir = %1023s", tmp) == 1)
+			} else if (sscanf(buff, "\tmaildir = %1023s", tmp) == 1) {
+				config_check_ops(cfg, &ops_maildir);
 				config_set_out(cfg, tmp, false);
-			else if (sscanf(buff, "\tinitial_import = %u", &val) == 1)
+			} else if (sscanf(buff, "\tpipe = %1023s", tmp) == 1) {
+				config_check_ops(cfg, &ops_pipe);
+				config_set_out(cfg, tmp, false);
+			} else if (sscanf(buff, "\tinitial_import = %u", &val) == 1) {
 				config_set_initial_import(cfg, val);
-			else
+			} else {
 				goto state_next;
+			}
 			break;
 
 		default:
diff --git a/l2md.h b/l2md.h
index 7389d8c..c127937 100644
--- a/l2md.h
+++ b/l2md.h
@@ -42,6 +42,7 @@ struct mail_ops {
 	void (*bootstrap)(struct config *cfg);
 	void (*new_mail)(struct config_repo *repo, uint32_t url,
 			 const char *oid, const void *raw, size_t len);
+	void (*set_defaults)(struct config *cfg);
 };
 
 struct config_general {
@@ -90,7 +91,8 @@ struct config {
 extern pid_t own_pid;
 extern bool verbose_enabled;
 
-extern struct mail_ops ops_maildir;
+extern const struct mail_ops ops_maildir;
+extern const struct mail_ops ops_pipe;
 
 struct config *config_init(int argc, char **argv);
 void config_uninit(struct config *cfg);
@@ -123,6 +125,9 @@ void xmkdir1(const char *path);
 void xmkdir2(const char *base, const char *name);
 void xmkdir1_with_subdirs(const char *path);
 
+void xpipe(int pipefd[2]);
+void xwrite(int fd, const char *to, size_t len);
+
 int xread_file(const char *file, char *to, size_t len, bool fatal);
 int xwrite_file(const char *file, const char *to, size_t len, bool fatal);
 
diff --git a/l2mdconfig b/l2mdconfig.maildir
index d273d9c..d273d9c 100644
--- a/l2mdconfig
+++ b/l2mdconfig.maildir
diff --git a/l2mdconfig.procmail b/l2mdconfig.procmail
new file mode 100644
index 0000000..a951169
--- /dev/null
+++ b/l2mdconfig.procmail
@@ -0,0 +1,8 @@
+[general]
+	base = ~/.l2md/
+	pipe = /usr/bin/procmail
+	period = 30
+
+[repo bpf]
+	url = https://lore.kernel.org/bpf/0
+	initial_import = 1000
diff --git a/maildir.c b/maildir.c
index 5228650..7b7cc41 100644
--- a/maildir.c
+++ b/maildir.c
@@ -33,8 +33,17 @@ static void maildir_bootstrap(struct config *cfg)
 	}
 }
 
-struct mail_ops ops_maildir = {
+static void maildir_set_defaults(struct config *cfg)
+{
+	char path[PATH_MAX];
+
+	slprintf(path, sizeof(path), "%s/maildir", cfg->general.base);
+	strlcpy(cfg->general.out, path, sizeof(cfg->general.out));
+}
+
+const struct mail_ops ops_maildir = {
 	.name		= "maildir",
 	.bootstrap	= maildir_bootstrap,
 	.new_mail	= maildir_new_mail,
+	.set_defaults	= maildir_set_defaults,
 };
diff --git a/pipe.c b/pipe.c
new file mode 100644
index 0000000..ec64769
--- /dev/null
+++ b/pipe.c
@@ -0,0 +1,71 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (C) 2019 Daniel Borkmann <daniel@iogearbox.net> */
+
+#include <unistd.h>
+#include <errno.h>
+#include <string.h>
+#include <libgen.h>
+
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <sys/stat.h>
+
+#include "l2md.h"
+
+enum {
+	FORK_ERROR = -1,
+	FORK_CHILD =  0,
+};
+
+static void pipe_new_mail(struct config_repo *repo, uint32_t which,
+			  const char *oid, const void *raw, size_t len)
+{
+	char tmp[PATH_MAX];
+	int fd[2];
+
+	xpipe(fd);
+	switch(fork()) {
+	case FORK_ERROR:
+		panic("Cannot fork: %s\n", strerror(errno));
+	case FORK_CHILD:
+		close(fd[1]);
+		dup2(fd[0], STDIN_FILENO);
+		close(fd[0]);
+		strcpy(tmp, repo->out);
+		execl(repo->out, basename(tmp), NULL);
+		break;
+	default:
+		close(fd[0]);
+		xwrite(fd[1], raw, len);
+		close(fd[1]);
+		wait(NULL);
+		break;
+	}
+}
+
+static void pipe_bootstrap(struct config *cfg)
+{
+	struct config_repo *repo;
+	struct stat sb;
+	uint32_t i;
+
+	repo_for_each(cfg, repo, i) {
+		if (stat(repo->out, &sb) != 0)
+			panic("Cannot stat %s: %s\n", repo->out,
+			      strerror(errno));
+		if (!(sb.st_mode & S_IXUSR))
+			panic("%s is not an executable!\n", repo->out);
+	}
+}
+
+static void pipe_set_defaults(struct config *cfg)
+{
+}
+
+const struct mail_ops ops_pipe = {
+	.name		= "pipe",
+	.bootstrap	= pipe_bootstrap,
+	.new_mail	= pipe_new_mail,
+	.set_defaults	= pipe_set_defaults,
+};
diff --git a/procmailrc b/procmailrc
new file mode 100644
index 0000000..3334386
--- /dev/null
+++ b/procmailrc
@@ -0,0 +1,7 @@
+SHELL=/bin/sh
+PATH=/usr/sbin:/usr/bin
+MAILDIR=$HOME/maildir/       # Make sure it exists!
+DEFAULT=$MAILDIR
+LOGFILE=$HOME/.procmail.log
+LOG=""
+VERBOSE=yes
diff --git a/utils.c b/utils.c
index 44004a4..9e21947 100644
--- a/utils.c
+++ b/utils.c
@@ -102,7 +102,7 @@ int xread_file(const char *file, char *to, size_t len, bool fatal)
 	fd = open(file, O_RDONLY);
 	if (fd < 0) {
 		if (fatal)
-			panic("Cannot open %s: %s", file, strerror(errno));
+			panic("Cannot open %s: %s\n", file, strerror(errno));
 		else
 			return fd;
 	}
@@ -111,7 +111,7 @@ int xread_file(const char *file, char *to, size_t len, bool fatal)
 		ret = read(fd, to, len);
 		if (ret < 0) {
 			if (fatal) {
-				panic("Cannot read file %s: %s", file,
+				panic("Cannot read file %s: %s\n", file,
 				      strerror(errno));
 			} else {
 				close(fd);
@@ -134,7 +134,7 @@ int xwrite_file(const char *file, const char *to, size_t len, bool fatal)
 	fd = open(file, O_CREAT | O_WRONLY | O_TRUNC, 0600);
 	if (fd < 0) {
 		if (fatal)
-			panic("Cannot open %s: %s", file, strerror(errno));
+			panic("Cannot open %s: %s\n", file, strerror(errno));
 		else
 			return fd;
 	}
@@ -143,7 +143,7 @@ int xwrite_file(const char *file, const char *to, size_t len, bool fatal)
 		ret = write(fd, to, len);
 		if (ret < 0) {
 			if (fatal) {
-				panic("Cannot write file %s: %s", file,
+				panic("Cannot write file %s: %s\n", file,
 				      strerror(errno));
 			} else {
 				close(fd);
@@ -158,6 +158,26 @@ int xwrite_file(const char *file, const char *to, size_t len, bool fatal)
 	return 0;
 }
 
+void xpipe(int pipefd[2])
+{
+	if (pipe(pipefd) < 0)
+		panic("Cannot create pipe: %s\n", strerror(errno));
+}
+
+void xwrite(int fd, const char *to, size_t len)
+{
+	loff_t ret;
+
+	do {
+		ret = write(fd, to, len);
+		if (ret < 0)
+			panic("Cannot write to file descriptor %d: %s",
+			      fd, strerror(errno));
+		to  += ret;
+		len -= ret;
+	} while (len > 0);
+}
+
 int timeval_sub(struct timeval *res, struct timeval *x,
 		struct timeval *y)
 {