code.delx.au - offlineimap/blob - offlineimap/imapserver.py

   1 # IMAP server support
   2 # Copyright (C) 2002 - 2007 John Goerzen
   3 # <jgoerzen@complete.org>
   4 #
   5 #    This program is free software; you can redistribute it and/or modify
   6 #    it under the terms of the GNU General Public License as published by
   7 #    the Free Software Foundation; either version 2 of the License, or
   8 #    (at your option) any later version.
   9 #
  10 #    This program is distributed in the hope that it will be useful,
  11 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 #    GNU General Public License for more details.
  14 #
  15 #    You should have received a copy of the GNU General Public License
  16 #    along with this program; if not, write to the Free Software
  17 #    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  18
  19 import imaplib
  20 from offlineimap import imaplibutil, imaputil, threadutil
  21 from offlineimap.ui import UIBase
  22 from threading import *
  23 import thread, hmac, os
  24 import base64
  25
  26 from StringIO import StringIO
  27 from platform import system
  28
  29 try:
  30     # do we have a recent pykerberos?
  31     have_gss = False
  32     import kerberos
  33     if 'authGSSClientWrap' in dir(kerberos):
  34         have_gss = True
  35 except ImportError:
  36     pass
  37
  38 class UsefulIMAPMixIn:
  39     def getstate(self):
  40         return self.state
  41     def getselectedfolder(self):
  42         if self.getstate() == 'SELECTED':
  43             return self.selectedfolder
  44         return None
  45
  46     def select(self, mailbox='INBOX', readonly=None, force = 0):
  47         if (not force) and self.getselectedfolder() == mailbox \
  48            and self.is_readonly == readonly:
  49             # No change; return.
  50             return
  51         result = self.__class__.__bases__[1].select(self, mailbox, readonly)
  52         if result[0] != 'OK':
  53             raise ValueError, "Error from select: %s" % str(result)
  54         if self.getstate() == 'SELECTED':
  55             self.selectedfolder = mailbox
  56         else:
  57             self.selectedfolder = None
  58
  59     def _mesg(self, s, secs=None):
  60         imaplibutil.new_mesg(self, s, secs)
  61
  62 class UsefulIMAP4(UsefulIMAPMixIn, imaplib.IMAP4):
  63     def open(self, host = '', port = imaplib.IMAP4_PORT):
  64         imaplibutil.new_open(self, host, port)
  65
  66     # This is a hack around Darwin's implementation of realloc() (which
  67     # Python uses inside the socket code). On Darwin, we split the
  68     # message into 100k chunks, which should be small enough - smaller
  69     # might start seriously hurting performance ...
  70
  71     def read(self, size):
  72         if system() == 'Darwin':
  73             read = 0
  74             io = StringIO()
  75             while read < size:
  76                 data = self.file.read(min(size-read,100000))
  77                 read += len(data)
  78                 io.write(data)
  79             return io.getvalue()
  80         else:
  81             return self.file.read(size)
  82
  83 class UsefulIMAP4_SSL(UsefulIMAPMixIn, imaplibutil.WrappedIMAP4_SSL):
  84     def open(self, host = '', port = imaplib.IMAP4_SSL_PORT):
  85         imaplibutil.new_open_ssl(self, host, port)
  86
  87     # This is the same hack as above, to be used in the case of an SSL
  88     # connexion.
  89
  90     def read(self, size):
  91         if system() == 'Darwin':
  92             read = 0
  93             io = StringIO()
  94             while read < size:
  95                 data = self.sslobj.read(min(size-read,100000))
  96                 read += len(data)
  97                 io.write(data)
  98             return io.getvalue()
  99         else:
 100             return self.sslobj.read(size)
 101
 102 class UsefulIMAP4_Tunnel(UsefulIMAPMixIn, imaplibutil.IMAP4_Tunnel): pass
 103
 104 class IMAPServer:
 105     GSS_STATE_STEP = 0
 106     GSS_STATE_WRAP = 1
 107     def __init__(self, config, reposname,
 108                  username = None, password = None, hostname = None,
 109                  port = None, ssl = 1, maxconnections = 1, tunnel = None,
 110                  reference = '""'):
 111         self.reposname = reposname
 112         self.config = config
 113         self.username = username
 114         self.password = password
 115         self.passworderror = None
 116         self.goodpassword = None
 117         self.hostname = hostname
 118         self.tunnel = tunnel
 119         self.port = port
 120         self.usessl = ssl
 121         self.delim = None
 122         self.root = None
 123         if port == None:
 124             if ssl:
 125                 self.port = 993
 126             else:
 127                 self.port = 143
 128         self.maxconnections = maxconnections
 129         self.availableconnections = []
 130         self.assignedconnections = []
 131         self.lastowner = {}
 132         self.semaphore = BoundedSemaphore(self.maxconnections)
 133         self.connectionlock = Lock()
 134         self.reference = reference
 135         self.gss_step = self.GSS_STATE_STEP
 136         self.gss_vc = None
 137         self.gssapi = False
 138
 139     def getpassword(self):
 140         if self.goodpassword != None:
 141             return self.goodpassword
 142
 143         if self.password != None and self.passworderror == None:
 144             return self.password
 145
 146         self.password = UIBase.getglobalui().getpass(self.reposname,
 147                                                      self.config,
 148                                                      self.passworderror)
 149         self.passworderror = None
 150
 151         return self.password
 152
 153     def getdelim(self):
 154         """Returns this server's folder delimiter.  Can only be called
 155         after one or more calls to acquireconnection."""
 156         return self.delim
 157
 158     def getroot(self):
 159         """Returns this server's folder root.  Can only be called after one
 160         or more calls to acquireconnection."""
 161         return self.root
 162
 163
 164     def releaseconnection(self, connection):
 165         """Releases a connection, returning it to the pool."""
 166         self.connectionlock.acquire()
 167         self.assignedconnections.remove(connection)
 168         self.availableconnections.append(connection)
 169         self.connectionlock.release()
 170         self.semaphore.release()
 171
 172     def md5handler(self, response):
 173         ui = UIBase.getglobalui()
 174         challenge = response.strip()
 175         ui.debug('imap', 'md5handler: got challenge %s' % challenge)
 176
 177         passwd = self.getpassword()
 178         retval = self.username + ' ' + hmac.new(passwd, challenge).hexdigest()
 179         ui.debug('imap', 'md5handler: returning %s' % retval)
 180         return retval
 181
 182     def plainauth(self, imapobj):
 183         UIBase.getglobalui().debug('imap',
 184                                    'Attempting plain authentication')
 185         imapobj.login(self.username, self.getpassword())
 186
 187     def gssauth(self, response):
 188         data = base64.b64encode(response)
 189         try:
 190             if self.gss_step == self.GSS_STATE_STEP:
 191                 if not self.gss_vc:
 192                     rc, self.gss_vc = kerberos.authGSSClientInit('imap@' +
 193                                                                  self.hostname)
 194                     response = kerberos.authGSSClientResponse(self.gss_vc)
 195                 rc = kerberos.authGSSClientStep(self.gss_vc, data)
 196                 if rc != kerberos.AUTH_GSS_CONTINUE:
 197                    self.gss_step = self.GSS_STATE_WRAP
 198             elif self.gss_step == self.GSS_STATE_WRAP:
 199                 rc = kerberos.authGSSClientUnwrap(self.gss_vc, data)
 200                 response = kerberos.authGSSClientResponse(self.gss_vc)
 201                 rc = kerberos.authGSSClientWrap(self.gss_vc, response,
 202                                                 self.username)
 203             response = kerberos.authGSSClientResponse(self.gss_vc)
 204         except kerberos.GSSError, err:
 205             # Kerberos errored out on us, respond with None to cancel the
 206             # authentication
 207             UIBase.getglobalui().debug('imap',
 208                                        '%s: %s' % (err[0][0], err[1][0]))
 209             return None
 210
 211         if not response:
 212             response = ''
 213         return base64.b64decode(response)
 214
 215     def acquireconnection(self):
 216         """Fetches a connection from the pool, making sure to create a new one
 217         if needed, to obey the maximum connection limits, etc.
 218         Opens a connection to the server and returns an appropriate
 219         object."""
 220
 221         self.semaphore.acquire()
 222         self.connectionlock.acquire()
 223         imapobj = None
 224
 225         if len(self.availableconnections): # One is available.
 226             # Try to find one that previously belonged to this thread
 227             # as an optimization.  Start from the back since that's where
 228             # they're popped on.
 229             threadid = thread.get_ident()
 230             imapobj = None
 231             for i in range(len(self.availableconnections) - 1, -1, -1):
 232                 tryobj = self.availableconnections[i]
 233                 if self.lastowner[tryobj] == threadid:
 234                     imapobj = tryobj
 235                     del(self.availableconnections[i])
 236                     break
 237             if not imapobj:
 238                 imapobj = self.availableconnections[0]
 239                 del(self.availableconnections[0])
 240             self.assignedconnections.append(imapobj)
 241             self.lastowner[imapobj] = thread.get_ident()
 242             self.connectionlock.release()
 243             return imapobj
 244
 245         self.connectionlock.release()   # Release until need to modify data
 246
 247         success = 0
 248         while not success:
 249             # Generate a new connection.
 250             if self.tunnel:
 251                 UIBase.getglobalui().connecting('tunnel', self.tunnel)
 252                 imapobj = UsefulIMAP4_Tunnel(self.tunnel)
 253                 success = 1
 254             elif self.usessl:
 255                 UIBase.getglobalui().connecting(self.hostname, self.port)
 256                 imapobj = UsefulIMAP4_SSL(self.hostname, self.port)
 257             else:
 258                 UIBase.getglobalui().connecting(self.hostname, self.port)
 259                 imapobj = UsefulIMAP4(self.hostname, self.port)
 260
 261             imapobj.mustquote = imaplibutil.mustquote
 262
 263             if not self.tunnel:
 264                 try:
 265                     # Try GSSAPI and continue if it fails
 266                     if 'AUTH=GSSAPI' in imapobj.capabilities and have_gss:
 267                         UIBase.getglobalui().debug('imap',
 268                             'Attempting GSSAPI authentication')
 269                         try:
 270                             imapobj.authenticate('GSSAPI', self.gssauth)
 271                         except imapobj.error, val:
 272                             UIBase.getglobalui().debug('imap',
 273                                 'GSSAPI Authentication failed')
 274                         else:
 275                             self.gssapi = True
 276                             self.password = None
 277
 278                     if not self.gssapi:
 279                         if 'AUTH=CRAM-MD5' in imapobj.capabilities:
 280                             UIBase.getglobalui().debug('imap',
 281                                                    'Attempting CRAM-MD5 authentication')
 282                             try:
 283                                 imapobj.authenticate('CRAM-MD5', self.md5handler)
 284                             except imapobj.error, val:
 285                                 self.plainauth(imapobj)
 286                         else:
 287                             self.plainauth(imapobj)
 288                     # Would bail by here if there was a failure.
 289                     success = 1
 290                     self.goodpassword = self.password
 291                 except imapobj.error, val:
 292                     self.passworderror = str(val)
 293                     self.password = None
 294
 295         if self.delim == None:
 296             listres = imapobj.list(self.reference, '""')[1]
 297             if listres == [None] or listres == None:
 298                 # Some buggy IMAP servers do not respond well to LIST "" ""
 299                 # Work around them.
 300                 listres = imapobj.list(self.reference, '"*"')[1]
 301             self.delim, self.root = \
 302                         imaputil.imapsplit(listres[0])[1:]
 303             self.delim = imaputil.dequote(self.delim)
 304             self.root = imaputil.dequote(self.root)
 305
 306         self.connectionlock.acquire()
 307         self.assignedconnections.append(imapobj)
 308         self.lastowner[imapobj] = thread.get_ident()
 309         self.connectionlock.release()
 310         return imapobj
 311
 312     def connectionwait(self):
 313         """Waits until there is a connection available.  Note that between
 314         the time that a connection becomes available and the time it is
 315         requested, another thread may have grabbed it.  This function is
 316         mainly present as a way to avoid spawning thousands of threads
 317         to copy messages, then have them all wait for 3 available connections.
 318         It's OK if we have maxconnections + 1 or 2 threads, which is what
 319         this will help us do."""
 320         threadutil.semaphorewait(self.semaphore)
 321
 322     def close(self):
 323         # Make sure I own all the semaphores.  Let the threads finish
 324         # their stuff.  This is a blocking method.
 325         self.connectionlock.acquire()
 326         threadutil.semaphorereset(self.semaphore, self.maxconnections)
 327         for imapobj in self.assignedconnections + self.availableconnections:
 328             imapobj.logout()
 329         self.assignedconnections = []
 330         self.availableconnections = []
 331         self.lastowner = {}
 332         self.connectionlock.release()
 333
 334     def keepalive(self, timeout, event):
 335         """Sends a NOOP to each connection recorded.   It will wait a maximum
 336         of timeout seconds between doing this, and will continue to do so
 337         until the Event object as passed is true.  This method is expected
 338         to be invoked in a separate thread, which should be join()'d after
 339         the event is set."""
 340         ui = UIBase.getglobalui()
 341         ui.debug('imap', 'keepalive thread started')
 342         while 1:
 343             ui.debug('imap', 'keepalive: top of loop')
 344             event.wait(timeout)
 345             ui.debug('imap', 'keepalive: after wait')
 346             if event.isSet():
 347                 ui.debug('imap', 'keepalive: event is set; exiting')
 348                 return
 349             ui.debug('imap', 'keepalive: acquiring connectionlock')
 350             self.connectionlock.acquire()
 351             numconnections = len(self.assignedconnections) + \
 352                              len(self.availableconnections)
 353             self.connectionlock.release()
 354             ui.debug('imap', 'keepalive: connectionlock released')
 355             threads = []
 356             imapobjs = []
 357
 358             for i in range(numconnections):
 359                 ui.debug('imap', 'keepalive: processing connection %d of %d' % (i, numconnections))
 360                 imapobj = self.acquireconnection()
 361                 ui.debug('imap', 'keepalive: connection %d acquired' % i)
 362                 imapobjs.append(imapobj)
 363                 thr = threadutil.ExitNotifyThread(target = imapobj.noop)
 364                 thr.setDaemon(1)
 365                 thr.start()
 366                 threads.append(thr)
 367                 ui.debug('imap', 'keepalive: thread started')
 368
 369             ui.debug('imap', 'keepalive: joining threads')
 370
 371             for thr in threads:
 372                 # Make sure all the commands have completed.
 373                 thr.join()
 374
 375             ui.debug('imap', 'keepalive: releasing connections')
 376
 377             for imapobj in imapobjs:
 378                 self.releaseconnection(imapobj)
 379
 380             ui.debug('imap', 'keepalive: bottom of loop')
 381
 382 class ConfigedIMAPServer(IMAPServer):
 383     """This class is designed for easier initialization given a ConfigParser
 384     object and an account name.  The passwordhash is used if
 385     passwords for certain accounts are known.  If the password for this
 386     account is listed, it will be obtained from there."""
 387     def __init__(self, repository, passwordhash = {}):
 388         """Initialize the object.  If the account is not a tunnel,
 389         the password is required."""
 390         self.repos = repository
 391         self.config = self.repos.getconfig()
 392         usetunnel = self.repos.getpreauthtunnel()
 393         if not usetunnel:
 394             host = self.repos.gethost()
 395             user = self.repos.getuser()
 396             port = self.repos.getport()
 397             ssl = self.repos.getssl()
 398         reference = self.repos.getreference()
 399         server = None
 400         password = None
 401
 402         if repository.getname() in passwordhash:
 403             password = passwordhash[repository.getname()]
 404
 405         # Connect to the remote server.
 406         if usetunnel:
 407             IMAPServer.__init__(self, self.config, self.repos.getname(),
 408                                 tunnel = usetunnel,
 409                                 reference = reference,
 410                                 maxconnections = self.repos.getmaxconnections())
 411         else:
 412             if not password:
 413                 password = self.repos.getpassword()
 414             IMAPServer.__init__(self, self.config, self.repos.getname(),
 415                                 user, password, host, port, ssl,
 416                                 self.repos.getmaxconnections(),
 417                                 reference = reference)