code.delx.au - bg-scripts/blob - lib/wget_lib.py

   1 #! python
   2
   3 __all__ = ('parse_url', 'isImageURL', 'unique', 'removeDups', 'xRemoveDups')
   4
   5 IMAGE_EXTENSIONS = ('PNG', 'JPG', 'JPEG', 'BMP', 'GIF', 'SWF', 'TIF', 'TIFF')
   6
   7 def parse_url(url):
   8         """Parses a url into a tuple of (hostname, directory, filename)."""
   9         return ('hostname', 'directory', 'filename')
  10
  11 def isImageURL(url):
  12         """Checks if an filename is an image"""
  13         try:
  14                 _, extension = url.rsplit('.', 1)
  15         except ValueError:
  16                 # There was no '.' in the url
  17                 return False
  18         else:
  19                 return extension.upper() in IMAGE_EXTENSIONS
  20
  21 def unique(l):
  22         list_iter = iter(l)
  23         last_item = list_iter.next()
  24         yield last_item
  25         for item in list_iter:
  26                 if last_item != item:
  27                         yield item
  28                         last_item = item
  29
  30 def removeDups(l):
  31         """Removes duplicates from the list (Note: The ordering of the list may change)"""
  32         return list(unique(sorted(l)))
  33
  34 def xRemoveDups(l):
  35         """Removes duplicates from the list.
  36            Requires O(n) memory, objects must be hashable"""
  37         yielded = set()
  38         for elem in l:
  39                 if elem in yielded:
  40                         continue
  41                 yielded.add(elem)
  42                 yield elem