code.delx.au - gnu-emacs/blob - lisp/gnus/deuglify.el

   1 ;;; deuglify.el --- deuglify broken Outlook (Express) articles
   2
   3 ;; Copyright (C) 2002, 2003, 2004, 2005 Free Software Foundation, Inc.
   4 ;; Copyright (C) 2001, 2002 Raymond Scholz
   5
   6 ;; Author: Raymond Scholz <rscholz@zonix.de>
   7 ;;         Thomas Steffen (unwrapping algorithm,
   8 ;;                         based on an idea of Stefan Monnier)
   9 ;; Keywords: mail, news
  10
  11 ;; This file is part of GNU Emacs.
  12
  13 ;; GNU Emacs is free software; you can redistribute it and/or modify
  14 ;; it under the terms of the GNU General Public License as published by
  15 ;; the Free Software Foundation; either version 2, or (at your option)
  16 ;; any later version.
  17
  18 ;; GNU Emacs is distributed in the hope that it will be useful,
  19 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
  20 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21 ;; GNU General Public License for more details.
  22
  23 ;; You should have received a copy of the GNU General Public License
  24 ;; along with GNU Emacs; see the file COPYING.  If not, write to the
  25 ;; Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  26 ;; Boston, MA 02111-1307, USA.
  27
  28 ;;; Commentary:
  29
  30 ;; This file enables Gnus to repair broken citations produced by
  31 ;; common user agents like MS Outlook (Express).  It may repair
  32 ;; articles of other user agents too.
  33 ;;
  34 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  35
  36 ;;
  37 ;; Outlook sometimes wraps cited lines before sending a message as
  38 ;; seen in this example:
  39 ;;
  40 ;; Example #1
  41 ;; ----------
  42 ;;
  43 ;; John Doe wrote:
  44 ;;
  45 ;; > This sentence no verb.  This sentence no verb.  This sentence
  46 ;; no
  47 ;; > verb.  This sentence no verb.  This sentence no verb.  This
  48 ;; > sentence no verb.
  49 ;;
  50 ;; The function `gnus-article-outlook-unwrap-lines' tries to recognize those
  51 ;; erroneously wrapped lines and will unwrap them.  I.e. putting the
  52 ;; wrapped parts ("no" in this example) back where they belong (at the
  53 ;; end of the cited line above).
  54 ;;
  55 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  56 ;;
  57 ;; Note that some people not only use broken user agents but also
  58 ;; practice a bad citation style by omitting blank lines between the
  59 ;; cited text and their own text.
  60 ;:
  61 ;; Example #2
  62 ;; ----------
  63 ;;
  64 ;; John Doe wrote:
  65 ;;
  66 ;; > This sentence no verb.  This sentence no verb.  This sentence no
  67 ;; You forgot in all your sentences.
  68 ;; > verb.  This sentence no verb.  This sentence no verb.  This
  69 ;; > sentence no verb.
  70 ;;
  71 ;; Unwrapping "You forgot in all your sentences." would be illegal as
  72 ;; this part wasn't intended to be cited text.
  73 ;; `gnus-article-outlook-unwrap-lines' will only unwrap lines if the resulting
  74 ;; citation line will be of a certain maximum length.  You can control
  75 ;; this by adjusting `gnus-outlook-deuglify-unwrap-max'.  Also
  76 ;; unwrapping will only be done if the line above the (possibly)
  77 ;; wrapped line has a minimum length of `gnus-outlook-deuglify-unwrap-min'.
  78 ;;
  79 ;; Furthermore no unwrapping will be undertaken if the last character
  80 ;; is one of the chars specified in
  81 ;; `gnus-outlook-deuglify-unwrap-stop-chars'.  Setting this to ".?!"
  82 ;; inhibits unwrapping if the cited line ends with a full stop,
  83 ;; question mark or exclamation mark.  Note that this variable
  84 ;; defaults to `nil', triggering a few false positives but generally
  85 ;; giving you better results.
  86 ;;
  87 ;; Unwrapping works on every level of citation.  Thus you will be able
  88 ;; repair broken citations of broken user agents citing broken
  89 ;; citations of broken user agents citing broken citations...
  90 ;;
  91 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  92 ;;
  93 ;; Citations are commonly introduced with an attribution line
  94 ;; indicating who wrote the cited text.  Outlook adds superfluous
  95 ;; information that can be found in the header of the message to this
  96 ;; line and often wraps it.
  97 ;;
  98 ;; If that weren't enough, lots of people write their own text above
  99 ;; the cited text and cite the complete original article below.
 100 ;;
 101 ;; Example #3
 102 ;; ----------
 103 ;;
 104 ;; Hey, John.  There's no in all your sentences!
 105 ;;
 106 ;; John Doe <john.doe@some.domain> wrote in message
 107 ;; news:a87usw8$dklsssa$2@some.news.server...
 108 ;; > This sentence no verb.  This sentence no verb.  This sentence
 109 ;; no
 110 ;; > verb.  This sentence no verb.  This sentence no verb.  This
 111 ;; > sentence no verb.
 112 ;; >
 113 ;; > Bye, John
 114 ;;
 115 ;; Repairing the attribution line will be done by function
 116 ;; `gnus-article-outlook-repair-attribution which calls other function that
 117 ;; try to recognize and repair broken attribution lines.  See variable
 118 ;; `gnus-outlook-deuglify-attrib-cut-regexp' for stuff that should be
 119 ;; cut off from the beginning of an attribution line and variable
 120 ;; `gnus-outlook-deuglify-attrib-verb-regexp' for the verbs that are
 121 ;; required to be found in an attribution line.  These function return
 122 ;; the point where the repaired attribution line starts.
 123 ;;
 124 ;; Rearranging the article so that the cited text appears above the
 125 ;; new text will be done by function
 126 ;; `gnus-article-outlook-rearrange-citation'.  This function calls
 127 ;; `gnus-article-outlook-repair-attribution to find and repair an attribution
 128 ;; line.
 129 ;;
 130 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 131 ;;
 132 ;; Well, and that's what the message will look like after applying
 133 ;; deuglification:
 134 ;;
 135 ;; Example #3 (deuglified)
 136 ;; -----------------------
 137 ;;
 138 ;; John Doe <john.doe@some.domain> wrote:
 139 ;;
 140 ;; > This sentence no verb.  This sentence no verb.  This sentence no
 141 ;; > verb.  This sentence no verb.  This sentence no verb.  This
 142 ;; > sentence no verb.
 143 ;; >
 144 ;; > Bye, John
 145 ;;
 146 ;; Hey, John.  There's no in all your sentences!
 147 ;;
 148 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 149 ;;
 150 ;; Usage
 151 ;; -----
 152 ;;
 153 ;; Press `W k' in the Summary Buffer.
 154 ;;
 155 ;; Non recommended usage :-)
 156 ;; ---------------------
 157 ;;
 158 ;; To automatically invoke deuglification on every article you read,
 159 ;; put something like that in your .gnus:
 160 ;;
 161 ;; (add-hook 'gnus-article-decode-hook 'gnus-article-outlook-unwrap-lines)
 162 ;;
 163 ;; or _one_ of the following lines:
 164 ;;
 165 ;; ;; repair broken attribution lines
 166 ;; (add-hook 'gnus-article-decode-hook 'gnus-article-outlook-repair-attribution)
 167 ;;
 168 ;; ;; repair broken attribution lines and citations
 169 ;; (add-hook 'gnus-article-decode-hook 'gnus-article-outlook-rearrange-citation)
 170 ;;
 171 ;; Note that there always may be some false positives, so I suggest
 172 ;; using the manual invocation.  After deuglification you may want to
 173 ;; refill the whole article using `W w'.
 174 ;;
 175 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 176 ;;
 177 ;; Limitations
 178 ;; -----------
 179 ;;
 180 ;; As I said before there may (or will) be a few false positives on
 181 ;; unwrapping cited lines with `gnus-article-outlook-unwrap-lines'.
 182 ;;
 183 ;; `gnus-article-outlook-repair-attribution will only fix the first
 184 ;; attribution line found in the article.  Furthermore it fixed to
 185 ;; certain kinds of attributions.  And there may be horribly many
 186 ;; false positives, vanishing lines and so on -- so don't trust your
 187 ;; eyes.  Again I recommend manual invocation.
 188 ;;
 189 ;; `gnus-article-outlook-rearrange-citation' carries all the limitations of
 190 ;; `gnus-article-outlook-repair-attribution.
 191 ;;
 192 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 193 ;;
 194 ;; See ChangeLog for other changes.
 195 ;;
 196 ;; Revision 1.5  2002/01/27 14:39:17  rscholz
 197 ;; * New variable `gnus-outlook-deuglify-no-wrap-chars' to inhibit
 198 ;;   unwrapping if one these chars is first in the possibly wrapped line.
 199 ;; * Improved rearranging of the article.
 200 ;; * New function `gnus-outlook-repair-attribution-block' for repairing
 201 ;;   those big "Original Message (following some headers)" attributions.
 202 ;;
 203 ;; Revision 1.4  2002/01/03 14:05:00  rscholz
 204 ;; Renamed `gnus-outlook-deuglify-article' to
 205 ;; `gnus-article-outlook-deuglify-article'.
 206 ;; Made it easier to deuglify the article while being in Gnus' Article
 207 ;; Edit Mode.  (suggested by Phil Nitschke)
 208 ;;
 209 ;;
 210 ;; Revision 1.3  2002/01/02 23:35:54  rscholz
 211 ;; Fix a bug that caused succeeding long attribution lines to be
 212 ;; unwrapped.  Minor doc fixes and regular expression tuning.
 213 ;;
 214 ;; Revision 1.2  2001/12/30 20:14:34  rscholz
 215 ;; Clean up source.
 216 ;;
 217 ;; Revision 1.1  2001/12/30 20:13:32  rscholz
 218 ;; Initial revision
 219 ;;
 220 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 221
 222 ;;; Code:
 223
 224 (require 'gnus-art)
 225 (require 'gnus-sum)
 226
 227 (defconst gnus-outlook-deuglify-version "1.5 Gnus version"
 228   "Version of gnus-outlook-deuglify.")
 229
 230 ;;; User Customizable Variables:
 231
 232 (defgroup gnus-outlook-deuglify nil
 233   "Deuglify articles generated by broken user agents like MS Outlook (Express)."
 234   :version "22.1"
 235   :group 'gnus)
 236
 237 ;;;###autoload
 238 (defcustom gnus-outlook-deuglify-unwrap-min 45
 239   "Minimum length of the cited line above the (possibly) wrapped line."
 240   :version "22.1"
 241   :type 'integer
 242   :group 'gnus-outlook-deuglify)
 243
 244 ;;;###autoload
 245 (defcustom gnus-outlook-deuglify-unwrap-max 95
 246   "Maximum length of the cited line after unwrapping."
 247   :version "22.1"
 248   :type 'integer
 249   :group 'gnus-outlook-deuglify)
 250
 251 (defcustom gnus-outlook-deuglify-cite-marks ">|#%"
 252   "Characters that indicate cited lines."
 253   :version "22.1"
 254   :type 'string
 255   :group 'gnus-outlook-deuglify)
 256
 257 (defcustom gnus-outlook-deuglify-unwrap-stop-chars nil ;; ".?!" or nil
 258   "Characters that inhibit unwrapping if they are the last one on the cited line above the possible wrapped line."
 259   :version "22.1"
 260   :type '(radio (const :format "None  " nil)
 261                 (string :value ".?!"))
 262   :group 'gnus-outlook-deuglify)
 263
 264 (defcustom gnus-outlook-deuglify-no-wrap-chars "`"
 265   "Characters that inhibit unwrapping if they are the first one in the possibly wrapped line."
 266   :version "22.1"
 267   :type 'string
 268   :group 'gnus-outlook-deuglify)
 269
 270 (defcustom  gnus-outlook-deuglify-attrib-cut-regexp
 271   "\\(On \\|Am \\)?\\(Mon\\|Tue\\|Wed\\|Thu\\|Fri\\|Sat\\|Sun\\),[^,]+, "
 272   "Regular expression matching the beginning of an attribution line that should be cut off."
 273   :version "22.1"
 274   :type 'string
 275   :group 'gnus-outlook-deuglify)
 276
 277 (defcustom gnus-outlook-deuglify-attrib-verb-regexp
 278   "wrote\\|writes\\|says\\|schrieb\\|schreibt\\|meinte\\|skrev\\|a écrit\\|schreef\\|escribió"
 279   "Regular expression matching the verb used in an attribution line."
 280   :version "22.1"
 281   :type 'string
 282   :group 'gnus-outlook-deuglify)
 283
 284 (defcustom  gnus-outlook-deuglify-attrib-end-regexp
 285   ": *\\|\\.\\.\\."
 286   "Regular expression matching the end of an attribution line."
 287   :version "22.1"
 288   :type 'string
 289   :group 'gnus-outlook-deuglify)
 290
 291 ;;;###autoload
 292 (defcustom gnus-outlook-display-hook nil
 293   "A hook called after an deuglified article has been prepared.
 294 It is run after `gnus-article-prepare-hook'."
 295   :version "22.1"
 296   :type 'hook
 297   :group 'gnus-outlook-deuglify)
 298
 299 ;; Functions
 300
 301 (defun gnus-outlook-display-article-buffer ()
 302   "Redisplay current buffer or article buffer."
 303   (with-current-buffer (or gnus-article-buffer (current-buffer))
 304     ;; "Emulate" `gnus-article-prepare-display' without calling
 305     ;; it. Calling `gnus-article-prepare-display' on an already
 306     ;; prepared article removes all MIME parts.  I'm unsure whether
 307     ;; this is a bug or not.
 308     (gnus-article-highlight t)
 309     (gnus-treat-article nil)
 310     (gnus-run-hooks 'gnus-article-prepare-hook
 311                     'gnus-outlook-display-hook)))
 312
 313 ;;;###autoload
 314 (defun gnus-article-outlook-unwrap-lines (&optional nodisplay)
 315   "Unwrap lines that appear to be wrapped citation lines.
 316 You can control what lines will be unwrapped by frobbing
 317 `gnus-outlook-deuglify-unwrap-min' and `gnus-outlook-deuglify-unwrap-max',
 318 indicating the minimum and maximum length of an unwrapped citation line.  If
 319 NODISPLAY is non-nil, don't redisplay the article buffer."
 320   (interactive "P")
 321   (save-excursion
 322     (let ((case-fold-search nil)
 323           (inhibit-read-only t)
 324           (cite-marks gnus-outlook-deuglify-cite-marks)
 325           (no-wrap gnus-outlook-deuglify-no-wrap-chars)
 326           (stop-chars gnus-outlook-deuglify-unwrap-stop-chars))
 327       (gnus-with-article-buffer
 328         (article-goto-body)
 329         (while (re-search-forward
 330                 (concat
 331                  "^\\([ \t" cite-marks "]*\\)"
 332                  "\\([" cite-marks "].*[^\n " stop-chars "]\\)[ \t]?\n"
 333                  "\\1\\([^\n " cite-marks no-wrap "]+.*\\)$")
 334               nil t)
 335           (let ((len12 (- (match-end 2) (match-beginning 1)))
 336               (len3 (- (match-end 3) (match-beginning 3))))
 337             (if (and (> len12 gnus-outlook-deuglify-unwrap-min)
 338                      (< (+ len12 len3) gnus-outlook-deuglify-unwrap-max))
 339                 (progn
 340                   (replace-match "\\1\\2 \\3")
 341                   (goto-char (match-beginning 0)))))))))
 342   (unless nodisplay (gnus-outlook-display-article-buffer)))
 343
 344 (defun gnus-outlook-rearrange-article (attr-start)
 345   "Put the text from ATTR-START to the end of buffer at the top of the article buffer."
 346   (save-excursion
 347     (let ((inhibit-read-only t)
 348           (cite-marks gnus-outlook-deuglify-cite-marks))
 349       (gnus-with-article-buffer
 350         (article-goto-body)
 351         ;; article does not start with attribution
 352         (unless (= (point) attr-start)
 353           (gnus-kill-all-overlays)
 354           (let ((cur (point))
 355                 ;; before signature or end of buffer
 356                 (to (if (gnus-article-search-signature)
 357                         (point)
 358                       (point-max))))
 359             ;; handle the case where the full quote is below the
 360             ;; signature
 361             (if (< to attr-start)
 362                 (setq to (point-max)))
 363             (transpose-regions cur attr-start attr-start to)))))))
 364
 365 ;; John Doe <john.doe@some.domain> wrote in message
 366 ;; news:a87usw8$dklsssa$2@some.news.server...
 367
 368 (defun gnus-outlook-repair-attribution-outlook ()
 369   "Repair a broken attribution line (Outlook)."
 370   (save-excursion
 371     (let ((case-fold-search nil)
 372           (inhibit-read-only t)
 373           (cite-marks gnus-outlook-deuglify-cite-marks))
 374       (gnus-with-article-buffer
 375         (article-goto-body)
 376         (if (re-search-forward
 377              (concat "^\\([^" cite-marks "].+\\)"
 378                      "\\(" gnus-outlook-deuglify-attrib-verb-regexp "\\)"
 379                      "\\(.*\n?[^\n" cite-marks "].*\\)?"
 380                      "\\(" gnus-outlook-deuglify-attrib-end-regexp "\\)$")
 381              nil t)
 382             (progn
 383               (gnus-kill-all-overlays)
 384               (replace-match "\\1\\2\\4")
 385               (match-beginning 0)))))))
 386
 387
 388 ;; ----- Original Message -----
 389 ;; From: "John Doe" <john.doe@some.domain>
 390 ;; To: "Doe Foundation" <info@doefnd.org>
 391 ;; Sent: Monday, November 19, 2001 12:13 PM
 392 ;; Subject: More Doenuts
 393
 394 (defun gnus-outlook-repair-attribution-block ()
 395   "Repair a big broken attribution block."
 396   (save-excursion
 397     (let ((case-fold-search nil)
 398           (inhibit-read-only t)
 399           (cite-marks gnus-outlook-deuglify-cite-marks))
 400       (gnus-with-article-buffer
 401         (article-goto-body)
 402         (if (re-search-forward
 403             (concat "^[" cite-marks " \t]*--* ?[^-]+ [^-]+ ?--*\\s *\n"
 404                      "[^\n:]+:[ \t]*\\([^\n]+\\)\n"
 405                      "\\([^\n:]+:[ \t]*[^\n]+\n\\)+")
 406              nil t)
 407             (progn
 408               (gnus-kill-all-overlays)
 409               (replace-match "\\1 wrote:\n")
 410               (match-beginning 0)))))))
 411
 412 ;; On Wed, 16 Jan 2002 23:23:30 +0100, John Doe <john.doe@some.domain> wrote:
 413
 414 (defun gnus-outlook-repair-attribution-other ()
 415   "Repair a broken attribution line (other user agents than Outlook)."
 416   (save-excursion
 417     (let ((case-fold-search nil)
 418           (inhibit-read-only t)
 419           (cite-marks gnus-outlook-deuglify-cite-marks))
 420       (gnus-with-article-buffer
 421         (article-goto-body)
 422         (if (re-search-forward
 423              (concat "^\\("gnus-outlook-deuglify-attrib-cut-regexp"\\)?"
 424                      "\\([^" cite-marks "].+\\)\n\\([^\n" cite-marks "].*\\)?"
 425                      "\\(" gnus-outlook-deuglify-attrib-verb-regexp "\\).*"
 426                      "\\(" gnus-outlook-deuglify-attrib-end-regexp "\\)$")
 427              nil t)
 428             (progn
 429               (gnus-kill-all-overlays)
 430               (replace-match "\\4 \\5\\6\\7")
 431               (match-beginning 0)))))))
 432
 433 ;;;###autoload
 434 (defun gnus-article-outlook-repair-attribution (&optional nodisplay)
 435   "Repair a broken attribution line.
 436 If NODISPLAY is non-nil, don't redisplay the article buffer."
 437   (interactive "P")
 438   (let ((attrib-start
 439          (or
 440           (gnus-outlook-repair-attribution-other)
 441           (gnus-outlook-repair-attribution-block)
 442           (gnus-outlook-repair-attribution-outlook))))
 443     (unless nodisplay (gnus-outlook-display-article-buffer))
 444     attrib-start))
 445
 446 (defun gnus-article-outlook-rearrange-citation (&optional nodisplay)
 447   "Repair broken citations.
 448 If NODISPLAY is non-nil, don't redisplay the article buffer."
 449   (interactive "P")
 450   (let ((attrib-start (gnus-article-outlook-repair-attribution 'nodisplay)))
 451     ;; rearrange citations if an attribution line has been recognized
 452     (if attrib-start
 453         (gnus-outlook-rearrange-article attrib-start)))
 454   (unless nodisplay (gnus-outlook-display-article-buffer)))
 455
 456 ;;;###autoload
 457 (defun gnus-outlook-deuglify-article (&optional nodisplay)
 458   "Full deuglify of broken Outlook (Express) articles.
 459 Treat dumbquotes, unwrap lines, repair attribution and rearrange citation.  If
 460 NODISPLAY is non-nil, don't redisplay the article buffer."
 461   (interactive "P")
 462   ;; apply treatment of dumb quotes
 463   (gnus-article-treat-dumbquotes)
 464   ;; repair wrapped cited lines
 465   (gnus-article-outlook-unwrap-lines 'nodisplay)
 466   ;; repair attribution line and rearrange citation.
 467   (gnus-article-outlook-rearrange-citation 'nodisplay)
 468   (unless nodisplay (gnus-outlook-display-article-buffer)))
 469
 470 ;;;###autoload
 471 (defun gnus-article-outlook-deuglify-article ()
 472   "Deuglify broken Outlook (Express) articles and redisplay."
 473   (interactive)
 474   (gnus-outlook-deuglify-article nil))
 475
 476 (provide 'deuglify)
 477
 478 ;; Local Variables:
 479 ;; coding: iso-8859-1
 480 ;; End:
 481
 482 ;;; arch-tag: 5f895cc9-51a9-487c-b42e-28844d79eb73
 483 ;;; deuglify.el ends here