# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-import __main__
+from threading import *
+from offlineimap import threadutil
+from offlineimap.threadutil import InstanceLimitedThread
+from offlineimap.ui import UIBase
+import os.path, re
class BaseFolder:
+ def __init__(self):
+ self.uidlock = Lock()
+
def getname(self):
"""Returns name"""
return self.name
+ def suggeststhreads(self):
+ """Returns true if this folder suggests using threads for actions;
+ false otherwise. Probably only IMAP will return true."""
+ return 0
+
+ def waitforthread(self):
+ """For threading folders, waits until there is a resource available
+ before firing off a thread. For all others, returns immediately."""
+ pass
+
+ def getcopyinstancelimit(self):
+ """For threading folders, returns the instancelimitname for
+ InstanceLimitedThreads."""
+ raise NotImplementedException
+
+ def storesmessages(self):
+ """Should be true for any backend that actually saves message bodies.
+ (Almost all of them). False for the LocalStatus backend. Saves
+ us from having to slurp up messages just for localstatus purposes."""
+ return 1
+
def getvisiblename(self):
return self.name
+ def getrepository(self):
+ """Returns the repository object that this folder is within."""
+ return self.repository
+
def getroot(self):
"""Returns the root of the folder, in a folder-specific fashion."""
return self.root
else:
return self.getname()
- def isuidvalidityok(self, remotefolder):
- raise NotImplementedException
+ def getfolderbasename(self):
+ foldername = self.getname()
+ foldername = foldername.replace(self.repository.getsep(), '.')
+ foldername = re.sub('/\.$', '/dot', foldername)
+ foldername = re.sub('^\.$', 'dot', foldername)
+ return foldername
+
+ def isuidvalidityok(self):
+ if self.getsaveduidvalidity() != None:
+ return self.getsaveduidvalidity() == self.getuidvalidity()
+ else:
+ self.saveuidvalidity()
+ return 1
- def getuidvalidity(self):
- raise NotImplementedException
+ def _getuidfilename(self):
+ return os.path.join(self.repository.getuiddir(),
+ self.getfolderbasename())
+
+ def getsaveduidvalidity(self):
+ if hasattr(self, '_base_saved_uidvalidity'):
+ return self._base_saved_uidvalidity
+ uidfilename = self._getuidfilename()
+ if not os.path.exists(uidfilename):
+ self._base_saved_uidvalidity = None
+ else:
+ file = open(uidfilename, "rt")
+ self._base_saved_uidvalidity = long(file.readline().strip())
+ file.close()
+ return self._base_saved_uidvalidity
+
+ def saveuidvalidity(self):
+ newval = self.getuidvalidity()
+ uidfilename = self._getuidfilename()
+ self.uidlock.acquire()
+ try:
+ file = open(uidfilename + ".tmp", "wt")
+ file.write("%d\n" % newval)
+ file.close()
+ os.rename(uidfilename + ".tmp", uidfilename)
+ self._base_saved_uidvalidity = newval
+ finally:
+ self.uidlock.release()
- def saveuidvalidity(self, newval):
+ def getuidvalidity(self):
raise NotImplementedException
def cachemessagelist(self):
If the backend cannot assign a new uid, it returns the uid passed in
WITHOUT saving the message.
+
+ If the backend CAN assign a new uid, but cannot find out what this UID
+ is (as is the case with many IMAP servers), it returns 0 but DOES save
+ the message.
IMAP backend should be the only one that can assign a new uid.
def addmessagesflags(self, uidlist, flags):
for uid in uidlist:
- self.addmessageflags(uid)
+ self.addmessageflags(uid, flags)
def deletemessageflags(self, uid, flags):
"""Removes each flag given from the message's flag set. If a given
newflags.sort()
self.savemessageflags(uid, newflags)
+ def deletemessagesflags(self, uidlist, flags):
+ for uid in uidlist:
+ self.deletemessageflags(uid, flags)
+
def deletemessage(self, uid):
raise NotImplementedException
for uid in uidlist:
self.deletemessage(uid)
+ def syncmessagesto_neguid_msg(self, uid, dest, applyto, register = 1):
+ if register:
+ UIBase.getglobalui().registerthread(self.getaccountname())
+ UIBase.getglobalui().copyingmessage(uid, self, applyto)
+ successobject = None
+ successuid = None
+ message = self.getmessage(uid)
+ flags = self.getmessageflags(uid)
+ for tryappend in applyto:
+ successuid = tryappend.savemessage(uid, message, flags)
+ if successuid >= 0:
+ successobject = tryappend
+ break
+ # Did we succeed?
+ if successobject != None:
+ if successuid: # Only if IMAP actually assigned a UID
+ # Copy the message to the other remote servers.
+ for appendserver in \
+ [x for x in applyto if x != successobject]:
+ appendserver.savemessage(successuid, message, flags)
+ # Copy to its new name on the local server and delete
+ # the one without a UID.
+ self.savemessage(successuid, message, flags)
+ self.deletemessage(uid) # It'll be re-downloaded.
+ else:
+ # Did not find any server to take this message. Ignore.
+ pass
+
+
def syncmessagesto_neguid(self, dest, applyto):
"""Pass 1 of folder synchronization.
and once that succeeds, get the UID, add it to the others for real,
add it to local for real, and delete the fake one."""
- for uid in self.getmessagelist().keys():
- if uid >= 0:
- continue
- __main__.ui.copyingmessage(uid, self, applyto)
- successobject = None
- successuid = None
- message = self.getmessage(uid)
- flags = self.getmessageflags(uid)
- for tryappend in applyto:
- successuid = tryappend.savemessage(uid, message, flags)
- if successuid > 0:
- successobject = tryappend
- break
- # Did we succeed?
- if successobject != None:
- # Copy the message to the other remote servers.
- for appendserver in [x for x in applyto if x != successobject]:
- appendserver.savemessage(successuid, message, flags)
- # Copy it to its new name on the local server and delete
- # the one without a UID.
- self.savemessage(successuid, message, flags)
- self.deletemessage(uid)
+ uidlist = [uid for uid in self.getmessagelist().keys() if uid < 0]
+ threads = []
+
+ usethread = None
+ if applyto != None:
+ usethread = applyto[0]
+
+ for uid in uidlist:
+ if usethread and usethread.suggeststhreads():
+ usethread.waitforthread()
+ thread = InstanceLimitedThread(\
+ usethread.getcopyinstancelimit(),
+ target = self.syncmessagesto_neguid_msg,
+ name = "New msg sync from %s" % self.getvisiblename(),
+ args = (uid, dest, applyto))
+ thread.setDaemon(1)
+ thread.start()
+ threads.append(thread)
else:
- # Did not find any server to take this message. Ignore.
- pass
+ self.syncmessagesto_neguid_msg(uid, dest, applyto, register = 0)
+ for thread in threads:
+ thread.join()
+
+ def copymessageto(self, uid, applyto, register = 1):
+ # Sometimes, it could be the case that if a sync takes awhile,
+ # a message might be deleted from the maildir before it can be
+ # synced to the status cache. This is only a problem with
+ # self.getmessage(). So, don't call self.getmessage unless
+ # really needed.
+ if register:
+ UIBase.getglobalui().registerthread(self.getaccountname())
+ UIBase.getglobalui().copyingmessage(uid, self, applyto)
+ message = ''
+ # If any of the destinations actually stores the message body,
+ # load it up.
+ for object in applyto:
+ if object.storesmessages():
+ message = self.getmessage(uid)
+ break
+ flags = self.getmessageflags(uid)
+ for object in applyto:
+ newuid = object.savemessage(uid, message, flags)
+ if newuid > 0 and newuid != uid:
+ # Change the local uid.
+ self.savemessage(newuid, message, flags)
+ self.deletemessage(uid)
+ uid = newuid
+
def syncmessagesto_copy(self, dest, applyto):
"""Pass 2 of folder synchronization.
Look for messages present in self but not in dest. If any, add
them to dest."""
+ threads = []
for uid in self.getmessagelist().keys():
if uid < 0: # Ignore messages that pass 1 missed.
continue
if not uid in dest.getmessagelist():
- __main__.ui.copyingmessage(uid, self, applyto)
- message = self.getmessage(uid)
- flags = self.getmessageflags(uid)
- for object in applyto:
- newuid = object.savemessage(uid, message, flags)
- if newuid > 0 and newuid != uid:
- # Change the local uid.
- self.savemessage(newuid, message, flags)
- self.deletemessage(uid)
- uid = newuid
+ if self.suggeststhreads():
+ self.waitforthread()
+ thread = InstanceLimitedThread(\
+ self.getcopyinstancelimit(),
+ target = self.copymessageto,
+ name = "Copy message %d from %s" % (uid,
+ self.getvisiblename()),
+ args = (uid, applyto))
+ thread.setDaemon(1)
+ thread.start()
+ threads.append(thread)
+ else:
+ self.copymessageto(uid, applyto, register = 0)
+ for thread in threads:
+ thread.join()
def syncmessagesto_delete(self, dest, applyto):
"""Pass 3 of folder synchronization.
if not uid in self.getmessagelist():
deletelist.append(uid)
if len(deletelist):
- __main__.ui.deletingmessages(uidlist, applyto)
+ UIBase.getglobalui().deletingmessages(deletelist, applyto)
for object in applyto:
- object.deletemessages(uidlist)
+ object.deletemessages(deletelist)
def syncmessagesto_flags(self, dest, applyto):
"""Pass 4 of folder synchronization.
Look for any flag matching issues -- set dest message to have the
same flags that we have."""
+
+ # As an optimization over previous versions, we store up which flags
+ # are being used for an add or a delete. For each flag, we store
+ # a list of uids to which it should be added. Then, we can call
+ # addmessagesflags() to apply them in bulk, rather than one
+ # call per message as before. This should result in some significant
+ # performance improvements.
+
+ addflaglist = {}
+ delflaglist = {}
+
for uid in self.getmessagelist().keys():
if uid < 0: # Ignore messages missed by pass 1
continue
destflags = dest.getmessageflags(uid)
addflags = [x for x in selfflags if x not in destflags]
- if len(addflags):
- __main__.ui.addingflags(uid, addflags, applyto)
- for object in applyto:
- object.addmessageflags(uid, addflags)
- delflags = [x for x in destflags if x not in selfflags]
- if len(delflags):
- __main__.ui.deletingflags(uid, delflags, applyto)
- for object in applyto:
- object.deletemessageflags(uid, delflags)
+ for flag in addflags:
+ if not flag in addflaglist:
+ addflaglist[flag] = []
+ addflaglist[flag].append(uid)
+ delflags = [x for x in destflags if x not in selfflags]
+ for flag in delflags:
+ if not flag in delflaglist:
+ delflaglist[flag] = []
+ delflaglist[flag].append(uid)
+
+ for object in applyto:
+ for flag in addflaglist.keys():
+ UIBase.getglobalui().addingflags(addflaglist[flag], flag, [object])
+ object.addmessagesflags(addflaglist[flag], [flag])
+ for flag in delflaglist.keys():
+ UIBase.getglobalui().deletingflags(delflaglist[flag], flag, [object])
+ object.deletemessagesflags(delflaglist[flag], [flag])
+
def syncmessagesto(self, dest, applyto = None):
"""Syncs messages in this folder to the destination.
If applyto is specified, it should be a list of folders (don't forget