code.delx.au - gnu-emacs/blob - lisp/mh-e/mh-junk.el

   1 ;;; mh-junk.el --- MH-E interface to anti-spam measures
   2
   3 ;; Copyright (C) 2003, 2004, 2005, 2006 Free Software Foundation, Inc.
   4
   5 ;; Author: Satyaki Das <satyaki@theforce.stanford.edu>,
   6 ;;         Bill Wohler <wohler@newt.com>
   7 ;; Maintainer: Bill Wohler <wohler@newt.com>
   8 ;; Keywords:  mail, spam
   9
  10 ;; This file is part of GNU Emacs.
  11
  12 ;; GNU Emacs is free software; you can redistribute it and/or modify
  13 ;; it under the terms of the GNU General Public License as published by
  14 ;; the Free Software Foundation; either version 2, or (at your option)
  15 ;; any later version.
  16
  17 ;; GNU Emacs is distributed in the hope that it will be useful,
  18 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
  19 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20 ;; GNU General Public License for more details.
  21
  22 ;; You should have received a copy of the GNU General Public License
  23 ;; along with GNU Emacs; see the file COPYING.  If not, write to the
  24 ;; Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  25 ;; Boston, MA 02110-1301, USA.
  26
  27 ;;; Commentary:
  28
  29 ;; Spam handling in MH-E.
  30
  31 ;;; Change Log:
  32
  33 ;;; Code:
  34
  35 (require 'mh-e)
  36 (require 'mh-scan)
  37 (mh-require-cl)
  38
  39 ;;;###mh-autoload
  40 (defun mh-junk-blacklist (range)
  41   "Blacklist RANGE as spam.
  42
  43 This command trains the spam program in use (see the option
  44 `mh-junk-program') with the content of RANGE and then handles the
  45 message(s) as specified by the option `mh-junk-disposition'.
  46
  47 Check the documentation of `mh-interactive-range' to see how RANGE is
  48 read in interactive use.
  49
  50 For more information about using your particular spam fighting
  51 program, see:
  52
  53   - `mh-spamassassin-blacklist'
  54   - `mh-bogofilter-blacklist'
  55   - `mh-spamprobe-blacklist'"
  56   (interactive (list (mh-interactive-range "Blacklist")))
  57   (let ((blacklist-func (nth 1 (assoc mh-junk-choice mh-junk-function-alist))))
  58     (unless blacklist-func
  59       (error "Customize `mh-junk-program' appropriately"))
  60     (let ((dest (cond ((null mh-junk-disposition) nil)
  61                       ((equal mh-junk-disposition "") "+")
  62                       ((eq (aref mh-junk-disposition 0) ?+)
  63                        mh-junk-disposition)
  64                       ((eq (aref mh-junk-disposition 0) ?@)
  65                        (concat mh-current-folder "/"
  66                                (substring mh-junk-disposition 1)))
  67                       (t (concat "+" mh-junk-disposition)))))
  68       (mh-iterate-on-range msg range
  69         (message "Blacklisting message %d..." msg)
  70         (funcall (symbol-function blacklist-func) msg)
  71         (message "Blacklisting message %d...done" msg)
  72         (if (not (memq msg mh-seen-list))
  73             (setq mh-seen-list (cons msg mh-seen-list)))
  74         (if dest
  75             (mh-refile-a-msg nil (intern dest))
  76           (mh-delete-a-msg nil)))
  77       (mh-next-msg))))
  78
  79 ;;;###mh-autoload
  80 (defun mh-junk-whitelist (range)
  81   "Whitelist RANGE as ham.
  82
  83 This command reclassifies the RANGE as ham if it were incorrectly
  84 classified as spam (see the option `mh-junk-program'). It then
  85 refiles the message into the \"+inbox\" folder.
  86
  87 Check the documentation of `mh-interactive-range' to see how
  88 RANGE is read in interactive use."
  89   (interactive (list (mh-interactive-range "Whitelist")))
  90   (let ((whitelist-func (nth 2 (assoc mh-junk-choice mh-junk-function-alist))))
  91     (unless whitelist-func
  92       (error "Customize `mh-junk-program' appropriately"))
  93     (mh-iterate-on-range msg range
  94       (message "Whitelisting message %d..." msg)
  95       (funcall (symbol-function whitelist-func) msg)
  96       (message "Whitelisting message %d...done" msg)
  97       (mh-refile-a-msg nil (intern mh-inbox)))
  98     (mh-next-msg)))
  99
 100 \f
 101
 102 ;; Spamassassin Interface
 103
 104 (defvar mh-spamassassin-executable (executable-find "spamassassin"))
 105 (defvar mh-sa-learn-executable (executable-find "sa-learn"))
 106
 107 ;;;###mh-autoload
 108 (defun mh-spamassassin-blacklist (msg)
 109   "Blacklist MSG with SpamAssassin.
 110
 111 SpamAssassin is one of the more popular spam filtering programs.
 112 Get it from your local distribution or from
 113 http://spamassassin.org/.
 114
 115 To use SpamAssassin, add the following recipes to
 116 \".procmailrc\":
 117
 118     # Append to $PATH the location of mhparam in some distros.
 119     PATH=$PATH:/usr/bin/mh
 120
 121     MAILDIR=$HOME/`mhparam Path`
 122
 123     # Fight spam with SpamAssassin.
 124     :0fw
 125     | spamc
 126
 127     # Anything with a spam level of 10 or more is junked immediately.
 128     :0:
 129     * ^X-Spam-Level: ..........
 130     /dev/null
 131
 132     :0:
 133     * ^X-Spam-Status: Yes
 134     spam/.
 135
 136 If you don't use \"spamc\", use \"spamassassin -P -a\".
 137
 138 Note that one of the recipes above throws away messages with a
 139 score greater than or equal to 10. Here's how you can determine a
 140 value that works best for you.
 141
 142 First, run \"spamassassin -t\" on every mail message in your
 143 archive and use Gnumeric to verify that the average plus the
 144 standard deviation of good mail is under 5, the SpamAssassin
 145 default for \"spam\".
 146
 147 Using Gnumeric, sort the messages by score and view the messages
 148 with the highest score. Determine the score which encompasses all
 149 of your interesting messages and add a couple of points to be
 150 conservative. Add that many dots to the \"X-Spam-Level:\" header
 151 field above to send messages with that score down the drain.
 152
 153 In the example above, messages with a score of 5-9 are set aside
 154 in the \"+spam\" folder for later review. The major weakness of
 155 rules-based filters is a plethora of false positives so it is
 156 worthwhile to check.
 157
 158 If SpamAssassin classifies a message incorrectly, or is unsure,
 159 you can use the MH-E commands \\[mh-junk-blacklist] and
 160 \\[mh-junk-whitelist].
 161
 162 The command \\[mh-junk-blacklist] adds a \"blacklist_from\" entry
 163 to \"~/spamassassin/user_prefs\", deletes the message, and sends
 164 the message to the Razor, so that others might not see this spam.
 165 If the \"sa-learn\" command is available, the message is also
 166 recategorized as spam.
 167
 168 The command \\[mh-junk-whitelist] adds a \"whitelist_from\" rule
 169 to the \"~/.spamassassin/user_prefs\" file. If the \"sa-learn\"
 170 command is available, the message is also recategorized as ham.
 171
 172 Over time, you'll observe that the same host or domain occurs
 173 repeatedly in the \"blacklist_from\" entries, so you might think
 174 that you could avoid future spam by blacklisting all mail from a
 175 particular domain. The utility function
 176 `mh-spamassassin-identify-spammers' helps you do precisely that.
 177 This function displays a frequency count of the hosts and domains
 178 in the \"blacklist_from\" entries from the last blank line in
 179 \"~/.spamassassin/user_prefs\" to the end of the file. This
 180 information can be used so that you can replace multiple
 181 \"blacklist_from\" entries with a single wildcard entry such as:
 182
 183     blacklist_from *@*amazingoffersdirect2u.com
 184
 185 In versions of SpamAssassin (2.50 and on) that support a Bayesian
 186 classifier, \\[mh-junk-blacklist] uses the program \"sa-learn\"
 187 to recategorize the message as spam. Neither MH-E, nor
 188 SpamAssassin, rebuilds the database after adding words, so you
 189 will need to run \"sa-learn --rebuild\" periodically. This can be
 190 done by adding the following to your crontab:
 191
 192     0 * * * *   sa-learn --rebuild > /dev/null 2>&1"
 193   (unless mh-spamassassin-executable
 194     (error "Unable to find the spamassassin executable"))
 195   (let ((current-folder mh-current-folder)
 196         (msg-file (mh-msg-filename msg mh-current-folder))
 197         (sender))
 198     (save-excursion
 199       (message "Reporting message %d..." msg)
 200       (mh-truncate-log-buffer)
 201       (call-process mh-spamassassin-executable msg-file mh-log-buffer nil
 202                     ;;"--report" "--remove-from-whitelist"
 203                     "-r" "-R")          ; spamassassin V2.20
 204       (when mh-sa-learn-executable
 205           (message "Recategorizing this message as spam...")
 206           (call-process mh-sa-learn-executable msg-file mh-log-buffer nil
 207                         "--single" "--spam" "--local" "--no-rebuild"))
 208       (message "Blacklisting message %d..." msg)
 209       (set-buffer (get-buffer-create mh-temp-buffer))
 210       (erase-buffer)
 211       (call-process (expand-file-name mh-scan-prog mh-progs)
 212                     nil mh-junk-background nil
 213                     (format "%s" msg) current-folder
 214                     "-format" "%<(mymbox{from})%|%(addr{from})%>")
 215       (goto-char (point-min))
 216       (if (search-forward-regexp "^\\(.+\\)$" nil t)
 217           (progn
 218             (setq sender (match-string 0))
 219             (mh-spamassassin-add-rule "blacklist_from" sender)
 220             (message "Blacklisting message %d...done" msg))
 221         (message "Blacklisting message %d...not done (from my address)" msg)))))
 222
 223 ;;;###mh-autoload
 224 (defun mh-spamassassin-whitelist (msg)
 225   "Whitelist MSG with SpamAssassin.
 226
 227 The \\[mh-junk-whitelist] command adds a \"whitelist_from\" rule to
 228 the \"~/.spamassassin/user_prefs\" file. If the \"sa-learn\" command
 229 is available, the message is also recategorized as ham.
 230
 231 See `mh-spamassassin-blacklist' for more information."
 232   (unless mh-spamassassin-executable
 233     (error "Unable to find the spamassassin executable"))
 234   (let ((msg-file (mh-msg-filename msg mh-current-folder))
 235         (show-buffer (get-buffer mh-show-buffer))
 236         from)
 237     (save-excursion
 238       (set-buffer (get-buffer-create mh-temp-buffer))
 239       (erase-buffer)
 240       (message "Removing spamassassin markup from message...")
 241       (call-process mh-spamassassin-executable msg-file mh-temp-buffer nil
 242                     ;; "--remove-markup"
 243                     "-d")               ; spamassassin V2.20
 244       (if show-buffer
 245           (kill-buffer show-buffer))
 246       (write-file msg-file)
 247       (when mh-sa-learn-executable
 248         (message "Recategorizing this message as ham...")
 249         (call-process mh-sa-learn-executable msg-file mh-temp-buffer nil
 250                       "--single" "--ham" "--local" "--no-rebuild"))
 251       (message "Whitelisting message %d..." msg)
 252       (setq from
 253             (car (mh-funcall-if-exists
 254                   ietf-drums-parse-address (mh-get-header-field "From:"))))
 255       (kill-buffer nil)
 256       (unless (or (null from) (equal from ""))
 257         (mh-spamassassin-add-rule "whitelist_from" from))
 258       (message "Whitelisting message %d...done" msg))))
 259
 260 (defun mh-spamassassin-add-rule (rule body)
 261   "Add a new rule to \"~/.spamassassin/user_prefs\".
 262 The name of the rule is RULE and its body is BODY."
 263   (save-window-excursion
 264     (let* ((line (format "%s\t%s\n" rule body))
 265            (case-fold-search t)
 266            (file (expand-file-name "~/.spamassassin/user_prefs"))
 267            (buffer-exists (find-buffer-visiting file)))
 268       (find-file file)
 269       (if (not (search-forward (format "\n%s" line) nil t))
 270           (progn
 271             (goto-char (point-max))
 272             (insert (if (bolp) "" "\n") line)
 273             (save-buffer)))
 274       (if (not buffer-exists)
 275           (kill-buffer nil)))))
 276
 277 ;;;###mh-autoload
 278 (defun mh-spamassassin-identify-spammers ()
 279   "Identify spammers who are repeat offenders.
 280
 281 This function displays a frequency count of the hosts and domains
 282 in the \"blacklist_from\" entries from the last blank line in
 283 \"~/.spamassassin/user_prefs\" to the end of the file. This
 284 information can be used so that you can replace multiple
 285 \"blacklist_from\" entries with a single wildcard entry such as:
 286
 287     blacklist_from *@*amazingoffersdirect2u.com"
 288   (interactive)
 289   (let* ((file (expand-file-name "~/.spamassassin/user_prefs"))
 290          (domains (make-hash-table :test 'equal)))
 291     (find-file file)
 292     ;; Only consider entries between last blank line and end of file.
 293     (goto-char (1- (point-max)))
 294     (search-backward-regexp "^$")
 295     ;; Perform frequency count.
 296     (save-excursion
 297       (while (search-forward-regexp "^blacklist_from\\s-*\\(.*\\)@\\(.*\\)$"
 298                                     nil t)
 299         (let ((host (match-string 2))
 300               value)
 301           ;; Remove top-level-domain from hostname.
 302           (setq host (cdr (reverse (split-string host "\\."))))
 303           ;; Add counts for each host and domain part.
 304           (while host
 305             (setq value (gethash (car host) domains))
 306             (setf (gethash (car host) domains) (1+ (if (not value) 0 value)))
 307             (setq host (cdr host))))))
 308
 309     ;; Output
 310     (delete-other-windows)
 311     (pop-to-buffer (get-buffer-create "*MH-E Spammer Frequencies*"))
 312     (erase-buffer)
 313     (maphash '(lambda (key value) ""
 314                 (if (> value 2)
 315                     (insert (format "%s %s\n" key value))))
 316              domains)
 317     (sort-numeric-fields 2 (point-min) (point-max))
 318     (reverse-region (point-min) (point-max))
 319     (goto-char (point-min))))
 320
 321 \f
 322
 323 ;; Bogofilter Interface
 324
 325 (defvar mh-bogofilter-executable (executable-find "bogofilter"))
 326
 327 ;;;###mh-autoload
 328 (defun mh-bogofilter-blacklist (msg)
 329   "Blacklist MSG with bogofilter.
 330
 331 Bogofilter is a Bayesian spam filtering program. Get it from your
 332 local distribution or from http://bogofilter.sourceforge.net/.
 333
 334 Bogofilter is taught by running:
 335
 336     bogofilter -n < good-message
 337
 338 on every good message, and
 339
 340     bogofilter -s < spam-message
 341
 342 on every spam message. This is called a full training; three other
 343 training methods are described in the FAQ that is distributed with
 344 bogofilter. Note that most Bayesian filters need 1000 to 5000 of each
 345 type of message to start doing a good job.
 346
 347 To use bogofilter, add the following recipes to \".procmailrc\":
 348
 349     MAILDIR=$HOME/`mhparam Path`
 350
 351     # Fight spam with bogofilter.
 352     :0fw
 353     | bogofilter -3 -e -p
 354
 355     :0:
 356     * ^X-Bogosity: Yes, tests=bogofilter
 357     spam/.
 358
 359     :0:
 360     * ^X-Bogosity: Unsure, tests=bogofilter
 361     spam/unsure/.
 362
 363 If bogofilter classifies a message incorrectly, or is unsure, you can
 364 use the MH-E commands \\[mh-junk-blacklist] and \\[mh-junk-whitelist]
 365 to update bogofilter's training.
 366
 367 The \"Bogofilter FAQ\" suggests that you run the following
 368 occasionally to shrink the database:
 369
 370     bogoutil -d wordlist.db | bogoutil -l wordlist.db.new
 371     mv wordlist.db wordlist.db.prv
 372     mv wordlist.db.new wordlist.db
 373
 374 The \"Bogofilter tuning HOWTO\" describes how you can fine-tune Bogofilter."
 375   (unless mh-bogofilter-executable
 376     (error "Unable to find the bogofilter executable"))
 377   (let ((msg-file (mh-msg-filename msg mh-current-folder)))
 378     (call-process mh-bogofilter-executable msg-file mh-junk-background
 379                   nil "-s")))
 380
 381 ;;;###mh-autoload
 382 (defun mh-bogofilter-whitelist (msg)
 383   "Whitelist MSG with bogofilter.
 384
 385 See `mh-bogofilter-blacklist' for more information."
 386   (unless mh-bogofilter-executable
 387     (error "Unable to find the bogofilter executable"))
 388   (let ((msg-file (mh-msg-filename msg mh-current-folder)))
 389     (call-process mh-bogofilter-executable msg-file mh-junk-background
 390                   nil "-n")))
 391
 392 \f
 393
 394 ;; Spamprobe Interface
 395
 396 (defvar mh-spamprobe-executable (executable-find "spamprobe"))
 397
 398 ;;;###mh-autoload
 399 (defun mh-spamprobe-blacklist (msg)
 400   "Blacklist MSG with SpamProbe.
 401
 402 SpamProbe is a Bayesian spam filtering program. Get it from your local
 403 distribution or from http://spamprobe.sourceforge.net.
 404
 405 To use SpamProbe, add the following recipes to \".procmailrc\":
 406
 407     MAILDIR=$HOME/`mhparam Path`
 408
 409     # Fight spam with SpamProbe.
 410     :0
 411     SCORE=| spamprobe receive
 412
 413     :0 wf
 414     | formail -I \"X-SpamProbe: $SCORE\"
 415
 416     :0:
 417     *^X-SpamProbe: SPAM
 418     spam/.
 419
 420 If SpamProbe classifies a message incorrectly, you can use the
 421 MH-E commands \\[mh-junk-blacklist] and \\[mh-junk-whitelist] to
 422 update SpamProbe's training."
 423   (unless mh-spamprobe-executable
 424     (error "Unable to find the spamprobe executable"))
 425   (let ((msg-file (mh-msg-filename msg mh-current-folder)))
 426     (call-process mh-spamprobe-executable msg-file mh-junk-background
 427                   nil "spam")))
 428
 429 ;;;###mh-autoload
 430 (defun mh-spamprobe-whitelist (msg)
 431   "Whitelist MSG with SpamProbe.
 432
 433 See `mh-spamprobe-blacklist' for more information."
 434   (unless mh-spamprobe-executable
 435     (error "Unable to find the spamprobe executable"))
 436   (let ((msg-file (mh-msg-filename msg mh-current-folder)))
 437     (call-process mh-spamprobe-executable msg-file mh-junk-background
 438                   nil "good")))
 439
 440 (provide 'mh-junk)
 441
 442 ;; Local Variables:
 443 ;; indent-tabs-mode: nil
 444 ;; sentence-end-double-space: nil
 445 ;; End:
 446
 447 ;; arch-tag: 603335f1-77ff-4306-8828-5d3dad51abe1
 448 ;;; mh-junk.el ends here