email: Added per-user spam checking and training

This commit adds per-user bayesian spam filtering and training to the
email module. It does this by making spam filtering a two part process
where the first is run as part of a postfix milter and the second is
run by dovecot as part of a sieve script that is run before any other
sieve scripts. And by training the bayesian filter using the Dovecot
IMAPSieve Plugin so that moving a mail to the Spam folder trains that
mail as spam and moving a mail from Spam folder to any other folder
trains as ham.

This commit also adds an assertion that Rspamd version is at least
1.7.3 since older versions did not support the settings option in the
proxy worker which is used to exclude the rules checked as part of final
delivery from the milter run.

To support the custom rule that loads the spam score from the first run
into the second run I needed Rspamd to support storing the custom Lua
rules in LOCAL_CONFDIR. This change was not added to Rspamd until after
version 1.8.0 and so to support the older versions of Rspamd found in
NixOS 18.09 and NixOS Unstable I have back-ported the change to versions
1.7.3, 1.7.9 and 1.8.0 of Rspamd which are versions I have come across
in NixOS.

The latest version of the Rspamd module in NixOS Unstable include
several changes, made by me, that made it possible to implement this
feature and so to also support NixOS 18.09 this commit also includes a
replacement module for Rspamd that disables the Rspamd module in nixpkgs
and instead downloads and uses a newer one from NixOS Unstable.

modules/core/packages.nix

@@ -11,6 +11,13 @@ with pkgs;
       #  src = /etc/nixos/pkgs/lxc;
       #  patches = [  ];
       #});
+      rspamd = let
+          version = super.rspamd.version;
+        in if (lib.versionAtLeast version "1.7.3") && (lib.versionOlder version "1.8.1")
+          then super.rspamd.overrideAttrs (oldAttrs: rec {
+            patches = ["${./rspamd}/rspamd-${version}-local-rules.patch"];
+          })
+          else super.rspamd;
     })
   ];
 }

modules/core/rspamd/rspamd-1.7.3-local-rules.patch

@@ -0,0 +1,72 @@
+diff --git a/rules/rspamd.lua b/rules/rspamd.lua
+index 6b53828ee..0f7990d58 100644
+--- a/rules/rspamd.lua
++++ b/rules/rspamd.lua
+@@ -21,7 +21,7 @@ require "global_functions" ()
+ config['regexp'] = {}
+ rspamd_maps = {} -- Global maps
+
+-local local_conf = rspamd_paths['CONFDIR']
++local local_conf = rspamd_paths['LOCAL_CONFDIR']
+ local local_rules = rspamd_paths['RULESDIR']
+
+ dofile(local_rules .. '/regexp/headers.lua')
+@@ -74,4 +74,4 @@ if rmaps and type(rmaps) == 'table' then
+ end
+
+ local rspamd_nn = require "lua_nn"
+-rspamd_nn.load_rspamd_nn() -- Load defined models
+\ No newline at end of file
++rspamd_nn.load_rspamd_nn() -- Load defined models
+diff --git a/src/libserver/cfg_rcl.c b/src/libserver/cfg_rcl.c
+index 6019d7c7d..0e17afa29 100644
+--- a/src/libserver/cfg_rcl.c
++++ b/src/libserver/cfg_rcl.c
+@@ -612,6 +612,7 @@ rspamd_rcl_worker_handler (rspamd_mempool_t *pool, const ucl_object_t *obj,
+ }
+
+ #define RSPAMD_CONFDIR_INDEX "CONFDIR"
++#define RSPAMD_LOCAL_CONFDIR_INDEX "LOCAL_CONFDIR"
+ #define RSPAMD_RUNDIR_INDEX "RUNDIR"
+ #define RSPAMD_DBDIR_INDEX "DBDIR"
+ #define RSPAMD_LOGDIR_INDEX "LOGDIR"
+@@ -819,6 +820,7 @@ rspamd_rcl_set_lua_globals (struct rspamd_config *cfg, lua_State *L,
+ 	lua_getglobal (L, "rspamd_paths");
+ 	if (lua_isnil (L, -1)) {
+ 		const gchar *confdir = RSPAMD_CONFDIR, *rundir = RSPAMD_RUNDIR,
++				*local_confdir = RSPAMD_LOCAL_CONFDIR,
+ 				*dbdir = RSPAMD_DBDIR, *logdir = RSPAMD_LOGDIR,
+ 				*wwwdir = RSPAMD_WWWDIR, *pluginsdir = RSPAMD_PLUGINSDIR,
+ 				*rulesdir = RSPAMD_RULESDIR, *lualibdir = RSPAMD_LUALIBDIR,
+@@ -866,6 +868,11 @@ rspamd_rcl_set_lua_globals (struct rspamd_config *cfg, lua_State *L,
+ 			confdir = t;
+ 		}
+
++		t = getenv ("LOCAL_CONFDIR");
++		if (t) {
++			local_confdir = t;
++		}
++
+
+ 		if (vars) {
+ 			t = g_hash_table_lookup (vars, "PLUGINSDIR");
+@@ -898,6 +905,11 @@ rspamd_rcl_set_lua_globals (struct rspamd_config *cfg, lua_State *L,
+ 				confdir = t;
+ 			}
+
++			t = g_hash_table_lookup (vars, "LOCAL_CONFDIR");
++			if (t) {
++				local_confdir = t;
++			}
++
+ 			t = g_hash_table_lookup (vars, "DBDIR");
+ 			if (t) {
+ 				dbdir = t;
+@@ -912,6 +924,7 @@ rspamd_rcl_set_lua_globals (struct rspamd_config *cfg, lua_State *L,
+ 		lua_createtable (L, 0, 9);
+
+ 		rspamd_lua_table_set (L, RSPAMD_CONFDIR_INDEX, confdir);
++		rspamd_lua_table_set (L, RSPAMD_LOCAL_CONFDIR_INDEX, local_confdir);
+ 		rspamd_lua_table_set (L, RSPAMD_RUNDIR_INDEX, rundir);
+ 		rspamd_lua_table_set (L, RSPAMD_DBDIR_INDEX, dbdir);
+ 		rspamd_lua_table_set (L, RSPAMD_LOGDIR_INDEX, logdir);

modules/core/rspamd/rspamd-1.7.9-local-rules.patch

@@ -0,0 +1,81 @@
+diff --git a/rules/rspamd.lua b/rules/rspamd.lua
+index a193eb495..40b6af704 100644
+--- a/rules/rspamd.lua
++++ b/rules/rspamd.lua
+@@ -21,7 +21,7 @@ require "global_functions" ()
+ config['regexp'] = {}
+ rspamd_maps = {} -- Global maps
+
+-local local_conf = rspamd_paths['CONFDIR']
++local local_conf = rspamd_paths['LOCAL_CONFDIR']
+ local local_rules = rspamd_paths['RULESDIR']
+ local rspamd_util = require "rspamd_util"
+
+diff --git a/src/lua/lua_common.c b/src/lua/lua_common.c
+index 323957086..1e4538b1c 100644
+--- a/src/lua/lua_common.c
++++ b/src/lua/lua_common.c
+@@ -576,6 +576,7 @@ rspamd_lua_set_globals (struct rspamd_config *cfg, lua_State *L,
+ 	lua_getglobal (L, "rspamd_paths");
+ 	if (lua_isnil (L, -1)) {
+ 		const gchar *confdir = RSPAMD_CONFDIR, *rundir = RSPAMD_RUNDIR,
++				*local_confdir = RSPAMD_LOCAL_CONFDIR,
+ 				*dbdir = RSPAMD_DBDIR, *logdir = RSPAMD_LOGDIR,
+ 				*wwwdir = RSPAMD_WWWDIR, *pluginsdir = RSPAMD_PLUGINSDIR,
+ 				*rulesdir = RSPAMD_RULESDIR, *lualibdir = RSPAMD_LUALIBDIR,
+@@ -623,6 +624,11 @@ rspamd_lua_set_globals (struct rspamd_config *cfg, lua_State *L,
+ 			confdir = t;
+ 		}
+
++		t = getenv ("LOCAL_CONFDIR");
++		if (t) {
++			local_confdir = t;
++		}
++
+
+ 		if (vars) {
+ 			t = g_hash_table_lookup (vars, "PLUGINSDIR");
+@@ -655,6 +661,11 @@ rspamd_lua_set_globals (struct rspamd_config *cfg, lua_State *L,
+ 				confdir = t;
+ 			}
+
++			t = g_hash_table_lookup (vars, "LOCAL_CONFDIR");
++			if (t) {
++				local_confdir = t;
++			}
++
+ 			t = g_hash_table_lookup (vars, "DBDIR");
+ 			if (t) {
+ 				dbdir = t;
+@@ -669,6 +680,7 @@ rspamd_lua_set_globals (struct rspamd_config *cfg, lua_State *L,
+ 		lua_createtable (L, 0, 9);
+
+ 		rspamd_lua_table_set (L, RSPAMD_CONFDIR_INDEX, confdir);
++		rspamd_lua_table_set (L, RSPAMD_LOCAL_CONFDIR_INDEX, local_confdir);
+ 		rspamd_lua_table_set (L, RSPAMD_RUNDIR_INDEX, rundir);
+ 		rspamd_lua_table_set (L, RSPAMD_DBDIR_INDEX, dbdir);
+ 		rspamd_lua_table_set (L, RSPAMD_LOGDIR_INDEX, logdir);
+diff --git a/src/lua/lua_common.h b/src/lua/lua_common.h
+index 838e0fe7a..5810184b7 100644
+--- a/src/lua/lua_common.h
++++ b/src/lua/lua_common.h
+@@ -410,6 +410,7 @@ gboolean rspamd_lua_require_function (lua_State *L, const gchar *modname,
+
+ /* Paths defs */
+ #define RSPAMD_CONFDIR_INDEX "CONFDIR"
++#define RSPAMD_LOCAL_CONFDIR_INDEX "LOCAL_CONFDIR"
+ #define RSPAMD_RUNDIR_INDEX "RUNDIR"
+ #define RSPAMD_DBDIR_INDEX "DBDIR"
+ #define RSPAMD_LOGDIR_INDEX "LOGDIR"
+diff --git a/src/lua/lua_config.c b/src/lua/lua_config.c
+index 2093cbe01..d5d80914b 100644
+--- a/src/lua/lua_config.c
++++ b/src/lua/lua_config.c
+@@ -3339,6 +3339,7 @@ lua_config_load_ucl (lua_State *L)
+
+ 		if (lua_istable (L, -1)) {
+ 			LUA_TABLE_TO_HASH(paths, RSPAMD_CONFDIR_INDEX);
++			LUA_TABLE_TO_HASH(paths, RSPAMD_LOCAL_CONFDIR_INDEX);
+ 			LUA_TABLE_TO_HASH(paths, RSPAMD_RUNDIR_INDEX);
+ 			LUA_TABLE_TO_HASH(paths, RSPAMD_DBDIR_INDEX);
+ 			LUA_TABLE_TO_HASH(paths, RSPAMD_LOGDIR_INDEX);

modules/core/rspamd/rspamd-1.8.0-local-rules.patch

@@ -0,0 +1,81 @@
+diff --git a/rules/rspamd.lua b/rules/rspamd.lua
+index a193eb495..67136cc6e 100644
+--- a/rules/rspamd.lua
++++ b/rules/rspamd.lua
+@@ -21,7 +21,7 @@ require "global_functions" ()
+ config['regexp'] = {}
+ rspamd_maps = {} -- Global maps
+
+-local local_conf = rspamd_paths['CONFDIR']
++local local_conf = rspamd_paths['LOCAL_CONFDIR']
+ local local_rules = rspamd_paths['RULESDIR']
+ local rspamd_util = require "rspamd_util"
+
+diff --git a/src/lua/lua_common.c b/src/lua/lua_common.c
+index ac7a393b8..233397fc0 100644
+--- a/src/lua/lua_common.c
++++ b/src/lua/lua_common.c
+@@ -581,6 +581,7 @@ rspamd_lua_set_globals (struct rspamd_config *cfg, lua_State *L,
+ 	lua_getglobal (L, "rspamd_paths");
+ 	if (lua_isnil (L, -1)) {
+ 		const gchar *confdir = RSPAMD_CONFDIR, *rundir = RSPAMD_RUNDIR,
++				*local_confdir = RSPAMD_LOCAL_CONFDIR,
+ 				*dbdir = RSPAMD_DBDIR, *logdir = RSPAMD_LOGDIR,
+ 				*wwwdir = RSPAMD_WWWDIR, *pluginsdir = RSPAMD_PLUGINSDIR,
+ 				*rulesdir = RSPAMD_RULESDIR, *lualibdir = RSPAMD_LUALIBDIR,
+@@ -628,6 +629,11 @@ rspamd_lua_set_globals (struct rspamd_config *cfg, lua_State *L,
+ 			confdir = t;
+ 		}
+
++		t = getenv ("LOCAL_CONFDIR");
++		if (t) {
++			local_confdir = t;
++		}
++
+
+ 		if (vars) {
+ 			t = g_hash_table_lookup (vars, "PLUGINSDIR");
+@@ -660,6 +666,11 @@ rspamd_lua_set_globals (struct rspamd_config *cfg, lua_State *L,
+ 				confdir = t;
+ 			}
+
++			t = g_hash_table_lookup (vars, "LOCAL_CONFDIR");
++			if (t) {
++				local_confdir = t;
++			}
++
+ 			t = g_hash_table_lookup (vars, "DBDIR");
+ 			if (t) {
+ 				dbdir = t;
+@@ -674,6 +685,7 @@ rspamd_lua_set_globals (struct rspamd_config *cfg, lua_State *L,
+ 		lua_createtable (L, 0, 9);
+
+ 		rspamd_lua_table_set (L, RSPAMD_CONFDIR_INDEX, confdir);
++		rspamd_lua_table_set (L, RSPAMD_LOCAL_CONFDIR_INDEX, local_confdir);
+ 		rspamd_lua_table_set (L, RSPAMD_RUNDIR_INDEX, rundir);
+ 		rspamd_lua_table_set (L, RSPAMD_DBDIR_INDEX, dbdir);
+ 		rspamd_lua_table_set (L, RSPAMD_LOGDIR_INDEX, logdir);
+diff --git a/src/lua/lua_common.h b/src/lua/lua_common.h
+index af0b5f824..fccbf5115 100644
+--- a/src/lua/lua_common.h
++++ b/src/lua/lua_common.h
+@@ -425,6 +425,7 @@ gboolean rspamd_lua_require_function (lua_State *L, const gchar *modname,
+
+ /* Paths defs */
+ #define RSPAMD_CONFDIR_INDEX "CONFDIR"
++#define RSPAMD_LOCAL_CONFDIR_INDEX "LOCAL_CONFDIR"
+ #define RSPAMD_RUNDIR_INDEX "RUNDIR"
+ #define RSPAMD_DBDIR_INDEX "DBDIR"
+ #define RSPAMD_LOGDIR_INDEX "LOGDIR"
+diff --git a/src/lua/lua_config.c b/src/lua/lua_config.c
+index d72c5ff66..b609348fe 100644
+--- a/src/lua/lua_config.c
++++ b/src/lua/lua_config.c
+@@ -3441,6 +3441,7 @@ lua_config_load_ucl (lua_State *L)
+
+ 		if (lua_istable (L, -1)) {
+ 			LUA_TABLE_TO_HASH(paths, RSPAMD_CONFDIR_INDEX);
++			LUA_TABLE_TO_HASH(paths, RSPAMD_LOCAL_CONFDIR_INDEX);
+ 			LUA_TABLE_TO_HASH(paths, RSPAMD_RUNDIR_INDEX);
+ 			LUA_TABLE_TO_HASH(paths, RSPAMD_DBDIR_INDEX);
+ 			LUA_TABLE_TO_HASH(paths, RSPAMD_LOGDIR_INDEX);

modules/default.nix

@@ -17,6 +17,7 @@
     core/version.nix
     services/reverse-proxy
     services/TLS
+    services/email/rspamd.nix
     services/email/opendkim.nix
     services/email/postfix.nix
     services/email/pfix-srsd.nix

modules/services/email/dovecot/filter_bin/rspamd.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+set -o errexit
+cat | rspamc --header="settings-id=delivery" -d "$1" -h /run/rspamd/worker-controller.sock -m

modules/services/email/dovecot/imap_sieve/report-ham.sieve

@@ -0,0 +1,19 @@
+require ["vnd.dovecot.pipe", "copy", "imapsieve", "environment", "variables"];
+
+if environment :matches "imap.mailbox" "*" {
+  set "mailbox" "${1}";
+}
+
+if string "${mailbox}" "Trash" {
+  stop;
+}
+
+if environment :matches "imap.user" "*" {
+  set "username" "${1}";
+}
+
+if environment :matches "imap.email" "*" {
+  set "email" "${1}";
+}
+
+pipe :copy "learn-ham.sh" [ "${username}", "${email}" ];

modules/services/email/dovecot/imap_sieve/report-spam.sieve

@@ -0,0 +1,11 @@
+require ["vnd.dovecot.pipe", "copy", "imapsieve", "environment", "variables"];
+
+if environment :matches "imap.user" "*" {
+  set "username" "${1}";
+}
+
+if environment :matches "imap.email" "*" {
+  set "email" "${1}";
+}
+
+pipe :copy "learn-spam.sh" [ "${username}", "${email}" ];

modules/services/email/dovecot/pipe_bin/learn-ham.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+set -o errexit
+exec rspamc -c bayes_users -d "$2" -h /run/rspamd/worker-controller.sock learn_ham

modules/services/email/dovecot/pipe_bin/learn-spam.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+set -o errexit
+exec rspamc -c bayes_users -d "$2" -h /run/rspamd/worker-controller.sock learn_spam

modules/services/email/dovecot/sieve/file-spam.sieve

@@ -0,0 +1,12 @@
+require ["fileinto", "reject", "envelope", "mailbox", "reject"];
+
+# spamassassin
+if header :contains "X-Spam-Flag" "YES" {
+  fileinto :create "Spam";
+  stop;
+}
+# rspamd
+if header :contains "X-Spam" "YES" {
+  fileinto :create "Spam";
+  stop;
+}

modules/services/email/dovecot/sieve/rspamd.sieve

@@ -0,0 +1,7 @@
+require ["vnd.dovecot.filter", "envelope", "variables"];
+
+if envelope :matches "to" "*" {
+  set "destination" "${1}";
+}
+
+filter "rspamd.sh" [ "${destination}" ];