code.delx.au - offlineimap/blob - offlineimap/folder/IMAP.py

   1 # IMAP folder support
   2 # Copyright (C) 2002-2007 John Goerzen
   3 # <jgoerzen@complete.org>
   4 #
   5 #    This program is free software; you can redistribute it and/or modify
   6 #    it under the terms of the GNU General Public License as published by
   7 #    the Free Software Foundation; either version 2 of the License, or
   8 #    (at your option) any later version.
   9 #
  10 #    This program is distributed in the hope that it will be useful,
  11 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 #    GNU General Public License for more details.
  14 #
  15 #    You should have received a copy of the GNU General Public License
  16 #    along with this program; if not, write to the Free Software
  17 #    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  18
  19 from Base import BaseFolder
  20 import imaplib
  21 from offlineimap import imaputil, imaplibutil
  22 from offlineimap.ui import UIBase
  23 from offlineimap.version import versionstr
  24 import rfc822, time, string, random, binascii, re
  25 from StringIO import StringIO
  26 from copy import copy
  27
  28
  29 class IMAPFolder(BaseFolder):
  30     def __init__(self, imapserver, name, visiblename, accountname, repository):
  31         self.config = imapserver.config
  32         self.expunge = repository.getexpunge()
  33         self.name = imaputil.dequote(name)
  34         self.root = None # imapserver.root
  35         self.sep = imapserver.delim
  36         self.imapserver = imapserver
  37         self.messagelist = None
  38         self.visiblename = visiblename
  39         self.accountname = accountname
  40         self.repository = repository
  41         self.randomgenerator = random.Random()
  42         BaseFolder.__init__(self)
  43
  44     def getaccountname(self):
  45         return self.accountname
  46
  47     def suggeststhreads(self):
  48         return 1
  49
  50     def waitforthread(self):
  51         self.imapserver.connectionwait()
  52
  53     def getcopyinstancelimit(self):
  54         return 'MSGCOPY_' + self.repository.getname()
  55
  56     def getvisiblename(self):
  57         return self.visiblename
  58
  59     def getuidvalidity(self):
  60         imapobj = self.imapserver.acquireconnection()
  61         try:
  62             # Primes untagged_responses
  63             imapobj.select(self.getfullname(), readonly = 1)
  64             return long(imapobj.untagged_responses['UIDVALIDITY'][0])
  65         finally:
  66             self.imapserver.releaseconnection(imapobj)
  67
  68     def cachemessagelist(self):
  69         imapobj = self.imapserver.acquireconnection()
  70         self.messagelist = {}
  71
  72         try:
  73             # Primes untagged_responses
  74             imapobj.select(self.getfullname(), readonly = 1, force = 1)
  75             try:
  76                 # Some mail servers do not return an EXISTS response if
  77                 # the folder is empty.
  78                 maxmsgid = long(imapobj.untagged_responses['EXISTS'][0])
  79             except KeyError:
  80                 return
  81             if maxmsgid < 1:
  82                 # No messages; return
  83                 return
  84
  85             # Now, get the flags and UIDs for these.
  86             # We could conceivably get rid of maxmsgid and just say
  87             # '1:*' here.
  88             response = imapobj.fetch('1:%d' % maxmsgid, '(FLAGS UID INTERNALDATE)')[1]
  89         finally:
  90             self.imapserver.releaseconnection(imapobj)
  91         for messagestr in response:
  92             # Discard the message number.
  93             messagestr = string.split(messagestr, maxsplit = 1)[1]
  94             options = imaputil.flags2hash(messagestr)
  95             if not options.has_key('UID'):
  96                 UIBase.getglobalui().warn('No UID in message with options %s' %\
  97                                           str(options),
  98                                           minor = 1)
  99             else:
 100                 uid = long(options['UID'])
 101                 flags = imaputil.flagsimap2maildir(options['FLAGS'])
 102                 rtime = imaplibutil.Internaldate2epoch(messagestr)
 103                 self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime}
 104
 105     def getmessagelist(self):
 106         return self.messagelist
 107
 108     def getmessage(self, uid):
 109         ui = UIBase.getglobalui()
 110         imapobj = self.imapserver.acquireconnection()
 111         try:
 112             imapobj.select(self.getfullname(), readonly = 1)
 113             initialresult = imapobj.uid('fetch', '%d' % uid, '(BODY.PEEK[])')
 114             ui.debug('imap', 'Returned object from fetching %d: %s' % \
 115                      (uid, str(initialresult)))
 116             return initialresult[1][0][1].replace("\r\n", "\n")
 117
 118         finally:
 119             self.imapserver.releaseconnection(imapobj)
 120
 121     def getmessagetime(self, uid):
 122         return self.messagelist[uid]['time']
 123
 124     def getmessageflags(self, uid):
 125         return self.messagelist[uid]['flags']
 126
 127     def savemessage_getnewheader(self, content):
 128         headername = 'X-OfflineIMAP-%s-' % str(binascii.crc32(content)).replace('-', 'x')
 129         headername += binascii.hexlify(self.repository.getname()) + '-'
 130         headername += binascii.hexlify(self.getname())
 131         headervalue= '%d-' % long(time.time())
 132         headervalue += str(self.randomgenerator.random()).replace('.', '')
 133         headervalue += '-v' + versionstr
 134         return (headername, headervalue)
 135
 136     def savemessage_addheader(self, content, headername, headervalue):
 137         ui = UIBase.getglobalui()
 138         ui.debug('imap',
 139                  'savemessage_addheader: called to add %s: %s' % (headername,
 140                                                                   headervalue))
 141         insertionpoint = content.find("\r\n")
 142         ui.debug('imap', 'savemessage_addheader: insertionpoint = %d' % insertionpoint)
 143         leader = content[0:insertionpoint]
 144         ui.debug('imap', 'savemessage_addheader: leader = %s' % repr(leader))
 145         if insertionpoint == 0 or insertionpoint == -1:
 146             newline = ''
 147             insertionpoint = 0
 148         else:
 149             newline = "\r\n"
 150         newline += "%s: %s" % (headername, headervalue)
 151         ui.debug('imap', 'savemessage_addheader: newline = ' + repr(newline))
 152         trailer = content[insertionpoint:]
 153         ui.debug('imap', 'savemessage_addheader: trailer = ' + repr(trailer))
 154         return leader + newline + trailer
 155
 156     def savemessage_searchforheader(self, imapobj, headername, headervalue):
 157         if imapobj.untagged_responses.has_key('APPENDUID'):
 158             return long(imapobj.untagged_responses['APPENDUID'][-1].split(' ')[1])
 159
 160         ui = UIBase.getglobalui()
 161         ui.debug('imap', 'savemessage_searchforheader called for %s: %s' % \
 162                  (headername, headervalue))
 163         # Now find the UID it got.
 164         headervalue = imapobj._quote(headervalue)
 165         try:
 166             matchinguids = imapobj.uid('search', 'HEADER', headername, headervalue)[1][0]
 167         except imapobj.error, err:
 168             # IMAP server doesn't implement search or had a problem.
 169             ui.debug('imap', "savemessage_searchforheader: got IMAP error '%s' while attempting to UID SEARCH for message with header %s" % (err, headername))
 170             return 0
 171         ui.debug('imap', 'savemessage_searchforheader got initial matchinguids: ' + repr(matchinguids))
 172
 173         if matchinguids == '':
 174             ui.debug('imap', "savemessage_searchforheader: UID SEARCH for message with header %s yielded no results" % headername)
 175             return 0
 176
 177         matchinguids = matchinguids.split(' ')
 178         ui.debug('imap', 'savemessage_searchforheader: matchinguids now ' + \
 179                  repr(matchinguids))
 180         if len(matchinguids) != 1 or matchinguids[0] == None:
 181             raise ValueError, "While attempting to find UID for message with header %s, got wrong-sized matchinguids of %s" % (headername, str(matchinguids))
 182         matchinguids.sort()
 183         return long(matchinguids[0])
 184
 185     def savemessage(self, uid, content, flags, rtime):
 186         imapobj = self.imapserver.acquireconnection()
 187         ui = UIBase.getglobalui()
 188         ui.debug('imap', 'savemessage: called')
 189         try:
 190             try:
 191                 imapobj.select(self.getfullname()) # Needed for search
 192             except imapobj.readonly:
 193                 ui.msgtoreadonly(self, uid, content, flags)
 194                 # Return indicating message taken, but no UID assigned.
 195                 # Fudge it.
 196                 return 0
 197
 198             # This backend always assigns a new uid, so the uid arg is ignored.
 199             # In order to get the new uid, we need to save off the message ID.
 200
 201             message = rfc822.Message(StringIO(content))
 202             datetuple_msg = rfc822.parsedate(message.getheader('Date'))
 203             # Will be None if missing or not in a valid format.
 204
 205             # If time isn't known
 206             if rtime == None and datetuple_msg == None:
 207                 datetuple = time.localtime()
 208             elif rtime == None:
 209                 datetuple = datetuple_msg
 210             else:
 211                 datetuple = time.localtime(rtime)
 212
 213             try:
 214                 if datetuple[0] < 1981:
 215                     raise ValueError
 216
 217                 # Check for invalid date
 218                 datetuple_check = time.localtime(time.mktime(datetuple))
 219                 if datetuple[:2] != datetuple_check[:2]:
 220                     raise ValueError
 221
 222                 # This could raise a value error if it's not a valid format.
 223                 date = imaplib.Time2Internaldate(datetuple)
 224             except (ValueError, OverflowError):
 225                 # Argh, sometimes it's a valid format but year is 0102
 226                 # or something.  Argh.  It seems that Time2Internaldate
 227                 # will rause a ValueError if the year is 0102 but not 1902,
 228                 # but some IMAP servers nonetheless choke on 1902.
 229                 date = imaplib.Time2Internaldate(time.localtime())
 230
 231             ui.debug('imap', 'savemessage: using date ' + str(date))
 232             content = re.sub("(?<!\r)\n", "\r\n", content)
 233             ui.debug('imap', 'savemessage: initial content is: ' + repr(content))
 234
 235             (headername, headervalue) = self.savemessage_getnewheader(content)
 236             ui.debug('imap', 'savemessage: new headers are: %s: %s' % \
 237                      (headername, headervalue))
 238             content = self.savemessage_addheader(content, headername,
 239                                                  headervalue)
 240             ui.debug('imap', 'savemessage: new content is: ' + repr(content))
 241             ui.debug('imap', 'savemessage: new content length is ' + \
 242                      str(len(content)))
 243
 244             assert(imapobj.append(self.getfullname(),
 245                                        imaputil.flagsmaildir2imap(flags),
 246                                        date, content)[0] == 'OK')
 247
 248             # Checkpoint.  Let it write out the messages, etc.
 249             assert(imapobj.check()[0] == 'OK')
 250
 251             # Keep trying until we get the UID.
 252             ui.debug('imap', 'savemessage: first attempt to get new UID')
 253             uid = self.savemessage_searchforheader(imapobj, headername,
 254                                                    headervalue)
 255             # See docs for savemessage in Base.py for explanation of this and other return values
 256             if uid <= 0:
 257                 ui.debug('imap', 'savemessage: first attempt to get new UID failed.  Going to run a NOOP and try again.')
 258                 assert(imapobj.noop()[0] == 'OK')
 259                 uid = self.savemessage_searchforheader(imapobj, headername,
 260                                                        headervalue)
 261         finally:
 262             self.imapserver.releaseconnection(imapobj)
 263
 264         if uid: # avoid UID FETCH 0 crash happening later on
 265             self.messagelist[uid] = {'uid': uid, 'flags': flags}
 266
 267         ui.debug('imap', 'savemessage: returning %d' % uid)
 268         return uid
 269
 270     def savemessageflags(self, uid, flags):
 271         imapobj = self.imapserver.acquireconnection()
 272         try:
 273             try:
 274                 imapobj.select(self.getfullname())
 275             except imapobj.readonly:
 276                 UIBase.getglobalui().flagstoreadonly(self, [uid], flags)
 277                 return
 278             result = imapobj.uid('store', '%d' % uid, 'FLAGS',
 279                                  imaputil.flagsmaildir2imap(flags))
 280             assert result[0] == 'OK', 'Error with store: ' + '. '.join(r[1])
 281         finally:
 282             self.imapserver.releaseconnection(imapobj)
 283         result = result[1][0]
 284         if not result:
 285             self.messagelist[uid]['flags'] = flags
 286         else:
 287             flags = imaputil.flags2hash(imaputil.imapsplit(result)[1])['FLAGS']
 288             self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags)
 289
 290     def addmessageflags(self, uid, flags):
 291         self.addmessagesflags([uid], flags)
 292
 293     def addmessagesflags_noconvert(self, uidlist, flags):
 294         self.processmessagesflags('+', uidlist, flags)
 295
 296     def addmessagesflags(self, uidlist, flags):
 297         """This is here for the sake of UIDMaps.py -- deletemessages must
 298         add flags and get a converted UID, and if we don't have noconvert,
 299         then UIDMaps will try to convert it twice."""
 300         self.addmessagesflags_noconvert(uidlist, flags)
 301
 302     def deletemessageflags(self, uid, flags):
 303         self.deletemessagesflags([uid], flags)
 304
 305     def deletemessagesflags(self, uidlist, flags):
 306         self.processmessagesflags('-', uidlist, flags)
 307
 308     def processmessagesflags(self, operation, uidlist, flags):
 309         if len(uidlist) > 101:
 310             # Hack for those IMAP ervers with a limited line length
 311             self.processmessagesflags(operation, uidlist[:100], flags)
 312             self.processmessagesflags(operation, uidlist[100:], flags)
 313             return
 314
 315         imapobj = self.imapserver.acquireconnection()
 316         try:
 317             try:
 318                 imapobj.select(self.getfullname())
 319             except imapobj.readonly:
 320                 UIBase.getglobalui().flagstoreadonly(self, uidlist, flags)
 321                 return
 322             r = imapobj.uid('store',
 323                             imaputil.listjoin(uidlist),
 324                             operation + 'FLAGS',
 325                             imaputil.flagsmaildir2imap(flags))
 326             assert r[0] == 'OK', 'Error with store: ' + '. '.join(r[1])
 327             r = r[1]
 328         finally:
 329             self.imapserver.releaseconnection(imapobj)
 330         # Some IMAP servers do not always return a result.  Therefore,
 331         # only update the ones that it talks about, and manually fix
 332         # the others.
 333         needupdate = copy(uidlist)
 334         for result in r:
 335             if result == None:
 336                 # Compensate for servers that don't return anything from
 337                 # STORE.
 338                 continue
 339             attributehash = imaputil.flags2hash(imaputil.imapsplit(result)[1])
 340             if not ('UID' in attributehash and 'FLAGS' in attributehash):
 341                 # Compensate for servers that don't return a UID attribute.
 342                 continue
 343             flags = attributehash['FLAGS']
 344             uid = long(attributehash['UID'])
 345             self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags)
 346             try:
 347                 needupdate.remove(uid)
 348             except ValueError:          # Let it slide if it's not in the list
 349                 pass
 350         for uid in needupdate:
 351             if operation == '+':
 352                 for flag in flags:
 353                     if not flag in self.messagelist[uid]['flags']:
 354                         self.messagelist[uid]['flags'].append(flag)
 355                     self.messagelist[uid]['flags'].sort()
 356             elif operation == '-':
 357                 for flag in flags:
 358                     if flag in self.messagelist[uid]['flags']:
 359                         self.messagelist[uid]['flags'].remove(flag)
 360
 361     def deletemessage(self, uid):
 362         self.deletemessages_noconvert([uid])
 363
 364     def deletemessages(self, uidlist):
 365         self.deletemessages_noconvert(uidlist)
 366
 367     def deletemessages_noconvert(self, uidlist):
 368         # Weed out ones not in self.messagelist
 369         uidlist = [uid for uid in uidlist if uid in self.messagelist]
 370         if not len(uidlist):
 371             return
 372
 373         self.addmessagesflags_noconvert(uidlist, ['T'])
 374         imapobj = self.imapserver.acquireconnection()
 375         try:
 376             try:
 377                 imapobj.select(self.getfullname())
 378             except imapobj.readonly:
 379                 UIBase.getglobalui().deletereadonly(self, uidlist)
 380                 return
 381             if self.expunge:
 382                 assert(imapobj.expunge()[0] == 'OK')
 383         finally:
 384             self.imapserver.releaseconnection(imapobj)
 385         for uid in uidlist:
 386             del self.messagelist[uid]
 387
 388