1 ;;; mh-junk.el --- Interface to anti-spam measures
3 ;; Copyright (C) 2003, 2004, 2005, 2006 Free Software Foundation, Inc.
5 ;; Author: Satyaki Das <satyaki@theforce.stanford.edu>,
6 ;; Bill Wohler <wohler@newt.com>
7 ;; Maintainer: Bill Wohler <wohler@newt.com>
8 ;; Keywords: mail, spam
10 ;; This file is part of GNU Emacs.
12 ;; GNU Emacs is free software; you can redistribute it and/or modify
13 ;; it under the terms of the GNU General Public License as published by
14 ;; the Free Software Foundation; either version 2, or (at your option)
17 ;; GNU Emacs is distributed in the hope that it will be useful,
18 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
19 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 ;; GNU General Public License for more details.
22 ;; You should have received a copy of the GNU General Public License
23 ;; along with GNU Emacs; see the file COPYING. If not, write to the
24 ;; Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
25 ;; Boston, MA 02110-1301, USA.
29 ;; Spam handling in MH-E.
35 (eval-when-compile (require 'mh-acros))
39 ;; Interactive functions callable from the folder buffer
41 (defun mh-junk-blacklist (range)
42 "Blacklist RANGE as spam.
44 This command trains the spam program in use (see the option
45 `mh-junk-program') with the content of RANGE and then handles the
46 message(s) as specified by the option `mh-junk-disposition'.
48 Check the documentation of `mh-interactive-range' to see how RANGE is
49 read in interactive use.
51 For more information about using your particular spam fighting
54 - `mh-spamassassin-blacklist'
55 - `mh-bogofilter-blacklist'
56 - `mh-spamprobe-blacklist'"
57 (interactive (list (mh-interactive-range "Blacklist")))
58 (let ((blacklist-func (nth 1 (assoc mh-junk-choice mh-junk-function-alist))))
59 (unless blacklist-func
60 (error "Customize `mh-junk-program' appropriately"))
61 (let ((dest (cond ((null mh-junk-disposition) nil)
62 ((equal mh-junk-disposition "") "+")
63 ((eq (aref mh-junk-disposition 0) ?+)
65 ((eq (aref mh-junk-disposition 0) ?@)
66 (concat mh-current-folder "/"
67 (substring mh-junk-disposition 1)))
68 (t (concat "+" mh-junk-disposition)))))
69 (mh-iterate-on-range msg range
70 (message "Blacklisting message %d..." msg)
71 (funcall (symbol-function blacklist-func) msg)
72 (message "Blacklisting message %d...done" msg)
73 (if (not (memq msg mh-seen-list))
74 (setq mh-seen-list (cons msg mh-seen-list)))
76 (mh-refile-a-msg nil (intern dest))
77 (mh-delete-a-msg nil)))
81 (defun mh-junk-whitelist (range)
82 "Whitelist RANGE as ham.
84 This command reclassifies the RANGE as ham if it were incorrectly
85 classified as spam (see the option `mh-junk-program'). It then
86 refiles the message into the \"+inbox\" folder.
88 Check the documentation of `mh-interactive-range' to see how
89 RANGE is read in interactive use."
90 (interactive (list (mh-interactive-range "Whitelist")))
91 (let ((whitelist-func (nth 2 (assoc mh-junk-choice mh-junk-function-alist))))
92 (unless whitelist-func
93 (error "Customize `mh-junk-program' appropriately"))
94 (mh-iterate-on-range msg range
95 (message "Whitelisting message %d..." msg)
96 (funcall (symbol-function whitelist-func) msg)
97 (message "Whitelisting message %d...done" msg)
98 (mh-refile-a-msg nil (intern mh-inbox)))
103 ;; Spamassassin Interface
105 (defvar mh-spamassassin-executable (executable-find "spamassassin"))
106 (defvar mh-sa-learn-executable (executable-find "sa-learn"))
108 (defun mh-spamassassin-blacklist (msg)
109 "Blacklist MSG with SpamAssassin.
111 SpamAssassin is one of the more popular spam filtering programs.
112 Get it from your local distribution or from
113 http://spamassassin.org/.
115 To use SpamAssassin, add the following recipes to
118 MAILDIR=$HOME/`mhparam Path`
120 # Fight spam with SpamAssassin.
124 # Anything with a spam level of 10 or more is junked immediately.
126 * ^X-Spam-Level: ..........
130 * ^X-Spam-Status: Yes
133 If you don't use \"spamc\", use \"spamassassin -P -a\".
135 Note that one of the recipes above throws away messages with a
136 score greater than or equal to 10. Here's how you can determine a
137 value that works best for you.
139 First, run \"spamassassin -t\" on every mail message in your
140 archive and use Gnumeric to verify that the average plus the
141 standard deviation of good mail is under 5, the SpamAssassin
142 default for \"spam\".
144 Using Gnumeric, sort the messages by score and view the messages
145 with the highest score. Determine the score which encompasses all
146 of your interesting messages and add a couple of points to be
147 conservative. Add that many dots to the \"X-Spam-Level:\" header
148 field above to send messages with that score down the drain.
150 In the example above, messages with a score of 5-9 are set aside
151 in the \"+spam\" folder for later review. The major weakness of
152 rules-based filters is a plethora of false positives so it is
155 If SpamAssassin classifies a message incorrectly, or is unsure,
156 you can use the MH-E commands \\[mh-junk-blacklist] and
157 \\[mh-junk-whitelist].
159 The command \\[mh-junk-blacklist] adds a \"blacklist_from\" entry
160 to \"~/spamassassin/user_prefs\", deletes the message, and sends
161 the message to the Razor, so that others might not see this spam.
162 If the \"sa-learn\" command is available, the message is also
163 recategorized as spam.
165 The command \\[mh-junk-whitelist] adds a \"whitelist_from\" rule
166 to the \"~/.spamassassin/user_prefs\" file. If the \"sa-learn\"
167 command is available, the message is also recategorized as ham.
169 Over time, you'll observe that the same host or domain occurs
170 repeatedly in the \"blacklist_from\" entries, so you might think
171 that you could avoid future spam by blacklisting all mail from a
172 particular domain. The utility function
173 `mh-spamassassin-identify-spammers' helps you do precisely that.
174 This function displays a frequency count of the hosts and domains
175 in the \"blacklist_from\" entries from the last blank line in
176 \"~/.spamassassin/user_prefs\" to the end of the file. This
177 information can be used so that you can replace multiple
178 \"blacklist_from\" entries with a single wildcard entry such as:
180 blacklist_from *@*amazingoffersdirect2u.com
182 In versions of SpamAssassin (2.50 and on) that support a Bayesian
183 classifier, \\[mh-junk-blacklist] uses the program \"sa-learn\"
184 to recategorize the message as spam. Neither MH-E, nor
185 SpamAssassin, rebuilds the database after adding words, so you
186 will need to run \"sa-learn --rebuild\" periodically. This can be
187 done by adding the following to your crontab:
189 0 * * * * sa-learn --rebuild > /dev/null 2>&1"
190 (unless mh-spamassassin-executable
191 (error "Unable to find the spamassassin executable"))
192 (let ((current-folder mh-current-folder)
193 (msg-file (mh-msg-filename msg mh-current-folder))
196 (message "Reporting message %d..." msg)
197 (mh-truncate-log-buffer)
198 (call-process mh-spamassassin-executable msg-file mh-log-buffer nil
199 ;;"--report" "--remove-from-whitelist"
200 "-r" "-R") ; spamassassin V2.20
201 (when mh-sa-learn-executable
202 (message "Recategorizing this message as spam...")
203 (call-process mh-sa-learn-executable msg-file mh-log-buffer nil
204 "--single" "--spam" "--local" "--no-rebuild"))
205 (message "Blacklisting message %d..." msg)
206 (set-buffer (get-buffer-create mh-temp-buffer))
208 (call-process (expand-file-name mh-scan-prog mh-progs)
209 nil mh-junk-background nil
210 (format "%s" msg) current-folder
211 "-format" "%<(mymbox{from})%|%(addr{from})%>")
212 (goto-char (point-min))
213 (if (search-forward-regexp "^\\(.+\\)$" nil t)
215 (setq sender (match-string 0))
216 (mh-spamassassin-add-rule "blacklist_from" sender)
217 (message "Blacklisting message %d...done" msg))
218 (message "Blacklisting message %d...not done (from my address)" msg)))))
220 (defun mh-spamassassin-whitelist (msg)
221 "Whitelist MSG with SpamAssassin.
223 The \\[mh-junk-whitelist] command adds a \"whitelist_from\" rule to
224 the \"~/.spamassassin/user_prefs\" file. If the \"sa-learn\" command
225 is available, the message is also recategorized as ham.
227 See `mh-spamassassin-blacklist' for more information."
228 (unless mh-spamassassin-executable
229 (error "Unable to find the spamassassin executable"))
230 (let ((msg-file (mh-msg-filename msg mh-current-folder))
231 (show-buffer (get-buffer mh-show-buffer))
234 (set-buffer (get-buffer-create mh-temp-buffer))
236 (message "Removing spamassassin markup from message...")
237 (call-process mh-spamassassin-executable msg-file mh-temp-buffer nil
239 "-d") ; spamassassin V2.20
241 (kill-buffer show-buffer))
242 (write-file msg-file)
243 (when mh-sa-learn-executable
244 (message "Recategorizing this message as ham...")
245 (call-process mh-sa-learn-executable msg-file mh-temp-buffer nil
246 "--single" "--ham" "--local --no-rebuild"))
247 (message "Whitelisting message %d..." msg)
249 (car (mh-funcall-if-exists
250 ietf-drums-parse-address (mh-get-header-field "From:"))))
252 (unless (or (null from) (equal from ""))
253 (mh-spamassassin-add-rule "whitelist_from" from))
254 (message "Whitelisting message %d...done" msg))))
256 (defun mh-spamassassin-add-rule (rule body)
257 "Add a new rule to \"~/.spamassassin/user_prefs\".
258 The name of the rule is RULE and its body is BODY."
259 (save-window-excursion
260 (let* ((line (format "%s\t%s\n" rule body))
262 (file (expand-file-name "~/.spamassassin/user_prefs"))
263 (buffer-exists (find-buffer-visiting file)))
265 (if (not (search-forward (format "\n%s" line) nil t))
267 (goto-char (point-max))
268 (insert (if (bolp) "" "\n") line)
270 (if (not buffer-exists)
271 (kill-buffer nil)))))
273 (defun mh-spamassassin-identify-spammers ()
274 "Identify spammers who are repeat offenders.
276 This function displays a frequency count of the hosts and domains
277 in the \"blacklist_from\" entries from the last blank line in
278 \"~/.spamassassin/user_prefs\" to the end of the file. This
279 information can be used so that you can replace multiple
280 \"blacklist_from\" entries with a single wildcard entry such as:
282 blacklist_from *@*amazingoffersdirect2u.com"
284 (let* ((file (expand-file-name "~/.spamassassin/user_prefs"))
285 (domains (make-hash-table :test 'equal)))
287 ;; Only consider entries between last blank line and end of file.
288 (goto-char (1- (point-max)))
289 (search-backward-regexp "^$")
290 ;; Perform frequency count.
292 (while (search-forward-regexp "^blacklist_from\\s-*\\(.*\\)@\\(.*\\)$"
294 (let ((host (match-string 2))
296 ;; Remove top-level-domain from hostname.
297 (setq host (cdr (reverse (split-string host "\\."))))
298 ;; Add counts for each host and domain part.
300 (setq value (gethash (car host) domains))
301 (setf (gethash (car host) domains) (1+ (if (not value) 0 value)))
302 (setq host (cdr host))))))
305 (delete-other-windows)
306 (pop-to-buffer (get-buffer-create "*MH-E Spammer Frequencies*"))
308 (maphash '(lambda (key value) ""
310 (insert (format "%s %s\n" key value))))
312 (sort-numeric-fields 2 (point-min) (point-max))
313 (reverse-region (point-min) (point-max))
314 (goto-char (point-min))))
318 ;; Bogofilter Interface
320 (defvar mh-bogofilter-executable (executable-find "bogofilter"))
322 (defun mh-bogofilter-blacklist (msg)
323 "Blacklist MSG with bogofilter.
325 Bogofilter is a Bayesian spam filtering program. Get it from your
326 local distribution or from http://bogofilter.sourceforge.net/.
328 Bogofilter is taught by running:
330 bogofilter -n < good-message
332 on every good message, and
334 bogofilter -s < spam-message
336 on every spam message. This is called a full training; three other
337 training methods are described in the FAQ that is distributed with
338 bogofilter. Note that most Bayesian filters need 1000 to 5000 of each
339 type of message to start doing a good job.
341 To use bogofilter, add the following recipes to \".procmailrc\":
343 MAILDIR=$HOME/`mhparam Path`
345 # Fight spam with bogofilter.
347 | bogofilter -3 -e -p
350 * ^X-Bogosity: Yes, tests=bogofilter
354 * ^X-Bogosity: Unsure, tests=bogofilter
357 If bogofilter classifies a message incorrectly, or is unsure, you can
358 use the MH-E commands \\[mh-junk-blacklist] and \\[mh-junk-whitelist]
359 to update bogofilter's training.
361 The \"Bogofilter FAQ\" suggests that you run the following
362 occasionally to shrink the database:
364 bogoutil -d wordlist.db | bogoutil -l wordlist.db.new
365 mv wordlist.db wordlist.db.prv
366 mv wordlist.db.new wordlist.db
368 The \"Bogofilter tuning HOWTO\" describes how you can fine-tune Bogofilter."
369 (unless mh-bogofilter-executable
370 (error "Unable to find the bogofilter executable"))
371 (let ((msg-file (mh-msg-filename msg mh-current-folder)))
372 (call-process mh-bogofilter-executable msg-file mh-junk-background
375 (defun mh-bogofilter-whitelist (msg)
376 "Whitelist MSG with bogofilter.
378 See `mh-bogofilter-blacklist' for more information."
379 (unless mh-bogofilter-executable
380 (error "Unable to find the bogofilter executable"))
381 (let ((msg-file (mh-msg-filename msg mh-current-folder)))
382 (call-process mh-bogofilter-executable msg-file mh-junk-background
387 ;; Spamprobe Interface
389 (defvar mh-spamprobe-executable (executable-find "spamprobe"))
391 (defun mh-spamprobe-blacklist (msg)
392 "Blacklist MSG with SpamProbe.
394 SpamProbe is a Bayesian spam filtering program. Get it from your local
395 distribution or from http://spamprobe.sourceforge.net.
397 To use SpamProbe, add the following recipes to \".procmailrc\":
399 MAILDIR=$HOME/`mhparam Path`
401 # Fight spam with SpamProbe.
403 SCORE=| spamprobe receive
406 | formail -I \"X-SpamProbe: $SCORE\"
412 If SpamProbe classifies a message incorrectly, you can use the
413 MH-E commands \\[mh-junk-blacklist] and \\[mh-junk-whitelist] to
414 update SpamProbe's training."
415 (unless mh-spamprobe-executable
416 (error "Unable to find the spamprobe executable"))
417 (let ((msg-file (mh-msg-filename msg mh-current-folder)))
418 (call-process mh-spamprobe-executable msg-file mh-junk-background
421 (defun mh-spamprobe-whitelist (msg)
422 "Whitelist MSG with SpamProbe.
424 See `mh-spamprobe-blacklist' for more information."
425 (unless mh-spamprobe-executable
426 (error "Unable to find the spamprobe executable"))
427 (let ((msg-file (mh-msg-filename msg mh-current-folder)))
428 (call-process mh-spamprobe-executable msg-file mh-junk-background
434 ;; indent-tabs-mode: nil
435 ;; sentence-end-double-space: nil
438 ;; arch-tag: 603335f1-77ff-4306-8828-5d3dad51abe1
439 ;;; mh-junk.el ends here