From c4336df677690d0d3392cbe380a16e8fda616892 Mon Sep 17 00:00:00 2001 From: James Bunton Date: Sat, 21 Jun 2014 08:09:22 +1000 Subject: [PATCH] Switch from tabs to spaces --- autograbber.py | 111 +++++++------ autosocks.py | 140 ++++++++-------- brightcove.py | 265 +++++++++++++++--------------- common.py | 428 ++++++++++++++++++++++++------------------------- grabber.py | 93 ++++++----- iview.py | 225 +++++++++++++------------- plus7.py | 203 ++++++++++++----------- sbs.py | 175 ++++++++++---------- 8 files changed, 812 insertions(+), 828 deletions(-) diff --git a/autograbber.py b/autograbber.py index 61b0673..84b8469 100755 --- a/autograbber.py +++ b/autograbber.py @@ -1,5 +1,4 @@ #!/usr/bin/python2 -# vim:ts=4:sts=4:sw=4:noet from common import load_root_node import fnmatch @@ -7,72 +6,72 @@ import os import sys DOWNLOAD_HISTORY_FILES = [ - ".downloaded_auto.txt", - "downloaded_auto.txt", + ".downloaded_auto.txt", + "downloaded_auto.txt", ] class DownloadList(object): - def __init__(self): - self.seen_list = set() - for filename in DOWNLOAD_HISTORY_FILES: - if os.path.isfile(filename): - break - else: - filename = DOWNLOAD_HISTORY_FILES[0] - try: - self.f = open(filename, "r") - for line in self.f: - self.seen_list.add(line.decode("utf-8").strip()) - self.f.close() - except Exception, e: - print >>sys.stderr, "Could not open:", filename, e - self.f = open(filename, "a") - - def has_seen(self, node): - return node.title in self.seen_list - - def mark_seen(self, node): - self.seen_list.add(node.title) - self.f.write(node.title.encode("utf-8") + "\n") - self.f.flush() + def __init__(self): + self.seen_list = set() + for filename in DOWNLOAD_HISTORY_FILES: + if os.path.isfile(filename): + break + else: + filename = DOWNLOAD_HISTORY_FILES[0] + try: + self.f = open(filename, "r") + for line in self.f: + self.seen_list.add(line.decode("utf-8").strip()) + self.f.close() + except Exception, e: + print >>sys.stderr, "Could not open:", filename, e + self.f = open(filename, "a") + + def has_seen(self, node): + return node.title in self.seen_list + + def mark_seen(self, node): + self.seen_list.add(node.title) + self.f.write(node.title.encode("utf-8") + "\n") + self.f.flush() def match(download_list, node, pattern, count=0): - if node.can_download: - if not download_list.has_seen(node): - if node.download(): - download_list.mark_seen(node) - else: - print >>sys.stderr, "Failed to download!", node.title - return + if node.can_download: + if not download_list.has_seen(node): + if node.download(): + download_list.mark_seen(node) + else: + print >>sys.stderr, "Failed to download!", node.title + return - if count >= len(pattern): - print "No match found for pattern:", "/".join(pattern) - return - p = pattern[count] - for child in node.get_children(): - if fnmatch.fnmatch(child.title, p): - match(download_list, child, pattern, count+1) + if count >= len(pattern): + print "No match found for pattern:", "/".join(pattern) + return + p = pattern[count] + for child in node.get_children(): + if fnmatch.fnmatch(child.title, p): + match(download_list, child, pattern, count+1) def main(destdir, patternfile): - os.chdir(destdir) - node = load_root_node() - download_list = DownloadList() + os.chdir(destdir) + node = load_root_node() + download_list = DownloadList() - for line in open(patternfile): - search = line.strip().split("/") - match(download_list, node, search) + for line in open(patternfile): + search = line.strip().split("/") + match(download_list, node, search) if __name__ == "__main__": - try: - destdir = os.path.abspath(sys.argv[1]) - patternfile = os.path.abspath(sys.argv[2]) - except IndexError: - print >>sys.stderr, "Usage: %s destdir patternfile" % sys.argv[0] - sys.exit(1) - try: - main(destdir, patternfile) - except (KeyboardInterrupt, EOFError): - print "\nExiting..." + try: + destdir = os.path.abspath(sys.argv[1]) + patternfile = os.path.abspath(sys.argv[2]) + except IndexError: + print >>sys.stderr, "Usage: %s destdir patternfile" % sys.argv[0] + sys.exit(1) + try: + main(destdir, patternfile) + except (KeyboardInterrupt, EOFError): + print "\nExiting..." diff --git a/autosocks.py b/autosocks.py index bae35c6..329f073 100644 --- a/autosocks.py +++ b/autosocks.py @@ -1,90 +1,88 @@ -# vim:ts=4:sts=4:sw=4:noet - import subprocess import sys def detect_gnome(): - """ Gnome via python-gconf """ - from gconf import client_get_default - gconf_client = client_get_default() - mode = gconf_client.get_string("/system/proxy/mode") - if mode != "manual": - return None, None - host = gconf_client.get_string("/system/proxy/socks_host") - port = gconf_client.get_int("/system/proxy/socks_port") - return host, port + """ Gnome via python-gconf """ + from gconf import client_get_default + gconf_client = client_get_default() + mode = gconf_client.get_string("/system/proxy/mode") + if mode != "manual": + return None, None + host = gconf_client.get_string("/system/proxy/socks_host") + port = gconf_client.get_int("/system/proxy/socks_port") + return host, port def detect_osx(): - """ OS X 10.5 and up via PyObjC """ - from SystemConfiguration import SCDynamicStoreCopyProxies - osx_proxy = SCDynamicStoreCopyProxies(None) - if osx_proxy.get("SOCKSEnable"): - host = osx_proxy.get("SOCKSProxy") - port = int(osx_proxy.get("SOCKSPort")) - return host, port - return None, None + """ OS X 10.5 and up via PyObjC """ + from SystemConfiguration import SCDynamicStoreCopyProxies + osx_proxy = SCDynamicStoreCopyProxies(None) + if osx_proxy.get("SOCKSEnable"): + host = osx_proxy.get("SOCKSProxy") + port = int(osx_proxy.get("SOCKSPort")) + return host, port + return None, None def detect_kde(): - """ KDE via command line, why no python bindings for KDE proxy settings? """ - if os.environ.get("KDE_FULL_SESSION") != "true": - return None, None - p = subprocess.Popen( - [ - "kreadconfig", - "--file", - "kioslaverc", - "--group", - "Proxy Settings", - "--key", - "socksProxy", - ], - shell=True, - stdout=subprocess.PIPE, - ) - host, port = p.stdout.readline()[:-1].split(":") - p.close() - port = int(port) - return host, port + """ KDE via command line, why no python bindings for KDE proxy settings? """ + if os.environ.get("KDE_FULL_SESSION") != "true": + return None, None + p = subprocess.Popen( + [ + "kreadconfig", + "--file", + "kioslaverc", + "--group", + "Proxy Settings", + "--key", + "socksProxy", + ], + shell=True, + stdout=subprocess.PIPE, + ) + host, port = p.stdout.readline()[:-1].split(":") + p.close() + port = int(port) + return host, port def detect_env(): - """ fallback to environment variables """ - socks_environ = os.environ.get("SOCKS_SERVER") - if not socks_environ: - return None, None - host, port = socks_environ - port = int(port) - return host, port + """ fallback to environment variables """ + socks_environ = os.environ.get("SOCKS_SERVER") + if not socks_environ: + return None, None + host, port = socks_environ + port = int(port) + return host, port def configure_socks(host, port): - """ hijack socket.socket using SocksiPy """ - try: - import socks, socket - except ImportError: - print >>sys.stderr, "Failed to use configured SOCKS proxy:", host, port - print >>sys.stderr, "Try installing SocksiPy: http://socksipy.sf.net" - return False + """ hijack socket.socket using SocksiPy """ + try: + import socks, socket + except ImportError: + print >>sys.stderr, "Failed to use configured SOCKS proxy:", host, port + print >>sys.stderr, "Try installing SocksiPy: http://socksipy.sf.net" + return False - socket.socket = socks.socksocket - socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, host, port) - return True + socket.socket = socks.socksocket + socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, host, port) + return True def try_autosocks(): - functions = [ - detect_gnome, - detect_osx, - detect_kde, - detect_env, - ] - for func in functions: - host, port = None, None - try: - host, port = func() - except Exception, e: - pass - if host is not None and port is not None: - return configure_socks(host, port) - return False + functions = [ + detect_gnome, + detect_osx, + detect_kde, + detect_env, + ] + for func in functions: + host, port = None, None + try: + host, port = func() + except Exception, e: + pass + if host is not None and port is not None: + return configure_socks(host, port) + return False diff --git a/brightcove.py b/brightcove.py index 4c4e096..4933adc 100644 --- a/brightcove.py +++ b/brightcove.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# vim:ts=4:sts=4:sw=4:noet - import re import sys @@ -14,150 +11,150 @@ HASH_URL = "http://admin.brightcove.com/viewer/us20130530.1140/connection/Extern class BrightcoveVideoNode(Node): - def __init__(self, title, parent, token, video_id): - Node.__init__(self, title, parent) - self.can_download = True - self.token = token - self.video_id = video_id - - def download(self): - desc_url = append_to_qs(BRIGHTCOVE_API, { - "token": self.token, - "command": "find_video_by_id", - "video_fields": "renditions", - "video_id": self.video_id, - }) - print "video desc_url", desc_url - - doc = grab_json(desc_url, 3600) - - best_encoding_rate = 0 - best_url = None - for item in doc['renditions']: - encoding_rate = item['encodingRate'] - if encoding_rate > best_encoding_rate: - best_encoding_rate = encoding_rate - best_url = item['url'] - - if best_url is None: - raise Exception("Could not find video URL: " + desc_url) - - vbase, vpath = best_url.split("&") - filename = self.title + ".mp4" - return download_rtmp(filename, vbase, vpath, HASH_URL) + def __init__(self, title, parent, token, video_id): + Node.__init__(self, title, parent) + self.can_download = True + self.token = token + self.video_id = video_id + + def download(self): + desc_url = append_to_qs(BRIGHTCOVE_API, { + "token": self.token, + "command": "find_video_by_id", + "video_fields": "renditions", + "video_id": self.video_id, + }) + print "video desc_url", desc_url + + doc = grab_json(desc_url, 3600) + + best_encoding_rate = 0 + best_url = None + for item in doc['renditions']: + encoding_rate = item['encodingRate'] + if encoding_rate > best_encoding_rate: + best_encoding_rate = encoding_rate + best_url = item['url'] + + if best_url is None: + raise Exception("Could not find video URL: " + desc_url) + + vbase, vpath = best_url.split("&") + filename = self.title + ".mp4" + return download_rtmp(filename, vbase, vpath, HASH_URL) class BrightcoveRootNode(Node): - def __init__(self, title, parent, token): - Node.__init__(self, title, parent) - self.token = token - self.series_nodes = {} - - def get_series_node(self, series_name): - series_name = series_name.split("-")[0].strip() - key = series_name.lower() - node = self.series_nodes.get(key, None) - if node is None: - node = Node(series_name, self) - self.series_nodes[key] = node - return node - - def fill_children(self): - page_number = 0 - while page_number < 100: - sys.stdout.write(".") - sys.stdout.flush() - url = self.get_all_videos_url(page_number) - page_number += 1 - - page = grab_json(url, 3600) - items = page["items"] - if len(items) == 0: - break - - for video_desc in items: - self.process_video(video_desc) - print - - def process_video(self, video_desc): - if not video_desc["customFields"]: - return - - video_id = video_desc["id"] - title = self.get_video_title(video_desc) - series_name = self.get_series_name(video_desc) - - parent_node = self.get_series_node(series_name) - BrightcoveVideoNode(title, parent_node, self.token, video_id) - - def get_video_title(self, video_desc): - raise NotImplementedError() - - def get_series_name(self, video_desc): - raise NotImplementedError() - - def get_all_videos_url(self, page_number): - raise NotImplementedError() + def __init__(self, title, parent, token): + Node.__init__(self, title, parent) + self.token = token + self.series_nodes = {} + + def get_series_node(self, series_name): + series_name = series_name.split("-")[0].strip() + key = series_name.lower() + node = self.series_nodes.get(key, None) + if node is None: + node = Node(series_name, self) + self.series_nodes[key] = node + return node + + def fill_children(self): + page_number = 0 + while page_number < 100: + sys.stdout.write(".") + sys.stdout.flush() + url = self.get_all_videos_url(page_number) + page_number += 1 + + page = grab_json(url, 3600) + items = page["items"] + if len(items) == 0: + break + + for video_desc in items: + self.process_video(video_desc) + print + + def process_video(self, video_desc): + if not video_desc["customFields"]: + return + + video_id = video_desc["id"] + title = self.get_video_title(video_desc) + series_name = self.get_series_name(video_desc) + + parent_node = self.get_series_node(series_name) + BrightcoveVideoNode(title, parent_node, self.token, video_id) + + def get_video_title(self, video_desc): + raise NotImplementedError() + + def get_series_name(self, video_desc): + raise NotImplementedError() + + def get_all_videos_url(self, page_number): + raise NotImplementedError() class Ch9RootNode(BrightcoveRootNode): - def __init__(self, root_node): - BrightcoveRootNode.__init__(self, "Nine", root_node, CH9_TOKEN) - - def get_video_title(self, video_desc): - title = video_desc["name"] - series = video_desc["customFields"]["series"] - season = video_desc["customFields"].get("season", "") - episode = video_desc["customFields"].get("episode", "1").rjust(2, "0") - - if re.match(R"ep(isode)?\s*[0-9]+\s*:", title.lower()): - title = title.split(":", 1)[1].strip() - - title = "%s - %sx%s - %s" % ( - series, - season, - episode, - title - ) - return title - - def get_series_name(self, video_desc): - return video_desc["customFields"]["series"] - - def get_all_videos_url(self, page_number): - return append_to_qs(BRIGHTCOVE_API, { - "token": self.token, - "command": "search_videos", - "video_fields": "id,name,customFields", - "custom_fields": "series,season,episode", - "sort_by": "PUBLISH_DATE", - "page_number": str(page_number), - }) + def __init__(self, root_node): + BrightcoveRootNode.__init__(self, "Nine", root_node, CH9_TOKEN) + + def get_video_title(self, video_desc): + title = video_desc["name"] + series = video_desc["customFields"]["series"] + season = video_desc["customFields"].get("season", "") + episode = video_desc["customFields"].get("episode", "1").rjust(2, "0") + + if re.match(R"ep(isode)?\s*[0-9]+\s*:", title.lower()): + title = title.split(":", 1)[1].strip() + + title = "%s - %sx%s - %s" % ( + series, + season, + episode, + title + ) + return title + + def get_series_name(self, video_desc): + return video_desc["customFields"]["series"] + + def get_all_videos_url(self, page_number): + return append_to_qs(BRIGHTCOVE_API, { + "token": self.token, + "command": "search_videos", + "video_fields": "id,name,customFields", + "custom_fields": "series,season,episode", + "sort_by": "PUBLISH_DATE", + "page_number": str(page_number), + }) class Ch10RootNode(BrightcoveRootNode): - def __init__(self, root_node): - BrightcoveRootNode.__init__(self, "Ten", root_node, CH10_TOKEN) + def __init__(self, root_node): + BrightcoveRootNode.__init__(self, "Ten", root_node, CH10_TOKEN) - def get_video_title(self, video_desc): - return video_desc["name"] + def get_video_title(self, video_desc): + return video_desc["name"] - def get_series_name(self, video_desc): - return video_desc["customFields"]["tv_show"] + def get_series_name(self, video_desc): + return video_desc["customFields"]["tv_show"] - def get_all_videos_url(self, page_number): - return append_to_qs(BRIGHTCOVE_API, { - "token": self.token, - "command": "search_videos", - "video_fields": "id,name,customFields", - "custom_fields": "tv_show", - "sort_by": "PUBLISH_DATE", - "any": "video_type_long_form:Full Episode", - "page_number": str(page_number), - }) + def get_all_videos_url(self, page_number): + return append_to_qs(BRIGHTCOVE_API, { + "token": self.token, + "command": "search_videos", + "video_fields": "id,name,customFields", + "custom_fields": "tv_show", + "sort_by": "PUBLISH_DATE", + "any": "video_type_long_form:Full Episode", + "page_number": str(page_number), + }) def fill_nodes(root_node): - Ch9RootNode(root_node) - Ch10RootNode(root_node) + Ch9RootNode(root_node) + Ch10RootNode(root_node) diff --git a/common.py b/common.py index e6191e7..9ef738a 100644 --- a/common.py +++ b/common.py @@ -1,12 +1,10 @@ -# vim:ts=4:sts=4:sw=4:noet - from lxml import etree, html import cookielib import json try: - import hashlib + import hashlib except ImportError: - import md5 as hashlib + import md5 as hashlib import os import re import shutil @@ -21,264 +19,264 @@ import urlparse try: - import autosocks - autosocks.try_autosocks() + import autosocks + autosocks.try_autosocks() except ImportError: - pass + pass CACHE_DIR = os.path.expanduser("~/.cache/webdl") USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:21.0) Gecko/20100101 Firefox/21.0" class Node(object): - def __init__(self, title, parent=None): - self.title = title - if parent: - parent.children.append(self) - self.parent = parent - self.children = [] - self.can_download = False + def __init__(self, title, parent=None): + self.title = title + if parent: + parent.children.append(self) + self.parent = parent + self.children = [] + self.can_download = False - def get_children(self): - if not self.children: - self.fill_children() - return self.children + def get_children(self): + if not self.children: + self.fill_children() + return self.children - def fill_children(self): - pass + def fill_children(self): + pass - def download(self): - raise NotImplemented + def download(self): + raise NotImplemented def load_root_node(): - root_node = Node("Root") + root_node = Node("Root") - import iview - iview.fill_nodes(root_node) + import iview + iview.fill_nodes(root_node) - import sbs - sbs.fill_nodes(root_node) + import sbs + sbs.fill_nodes(root_node) - import plus7 - plus7.fill_nodes(root_node) + import plus7 + plus7.fill_nodes(root_node) - import brightcove - brightcove.fill_nodes(root_node) + import brightcove + brightcove.fill_nodes(root_node) - return root_node + return root_node valid_chars = frozenset("-_.()!@#%^ abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") def sanify_filename(filename): - filename = filename.encode("ascii", "ignore") - filename = "".join(c for c in filename if c in valid_chars) - return filename + filename = filename.encode("ascii", "ignore") + filename = "".join(c for c in filename if c in valid_chars) + return filename cookiejar = cookielib.CookieJar() urlopener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar)) def _urlopen(url, referrer=None): - req = urllib2.Request(url) - req.add_header("User-Agent", USER_AGENT) - if referrer: - req.add_header("Referer", referrer) - return urlopener.open(req) + req = urllib2.Request(url) + req.add_header("User-Agent", USER_AGENT) + if referrer: + req.add_header("Referer", referrer) + return urlopener.open(req) def urlopen(url, max_age): -### print url - if not os.path.isdir(CACHE_DIR): - os.makedirs(CACHE_DIR) - - if max_age <= 0: - return _urlopen(url) - - filename = hashlib.md5(url).hexdigest() - filename = os.path.join(CACHE_DIR, filename) - if os.path.exists(filename): - file_age = int(time.time()) - os.path.getmtime(filename) - if file_age < max_age: - return open(filename) - - src = _urlopen(url) - dst = open(filename, "w") - try: - shutil.copyfileobj(src, dst) - except Exception, e: - try: - os.unlink(filename) - except OSError: - pass - raise e - src.close() - dst.close() - - return open(filename) +### print url + if not os.path.isdir(CACHE_DIR): + os.makedirs(CACHE_DIR) + + if max_age <= 0: + return _urlopen(url) + + filename = hashlib.md5(url).hexdigest() + filename = os.path.join(CACHE_DIR, filename) + if os.path.exists(filename): + file_age = int(time.time()) - os.path.getmtime(filename) + if file_age < max_age: + return open(filename) + + src = _urlopen(url) + dst = open(filename, "w") + try: + shutil.copyfileobj(src, dst) + except Exception, e: + try: + os.unlink(filename) + except OSError: + pass + raise e + src.close() + dst.close() + + return open(filename) def grab_text(url, max_age): - f = urlopen(url, max_age) - text = f.read().decode("utf-8") - f.close() - return text + f = urlopen(url, max_age) + text = f.read().decode("utf-8") + f.close() + return text def grab_html(url, max_age): - f = urlopen(url, max_age) - doc = html.parse(f, html.HTMLParser(encoding="utf-8", recover=True)) - f.close() - return doc + f = urlopen(url, max_age) + doc = html.parse(f, html.HTMLParser(encoding="utf-8", recover=True)) + f.close() + return doc def grab_xml(url, max_age): - f = urlopen(url, max_age) - doc = etree.parse(f, etree.XMLParser(encoding="utf-8", recover=True)) - f.close() - return doc + f = urlopen(url, max_age) + doc = etree.parse(f, etree.XMLParser(encoding="utf-8", recover=True)) + f.close() + return doc def grab_json(url, max_age, skip_assignment=False, skip_function=False): - f = urlopen(url, max_age) - if skip_assignment: - text = f.read() - pos = text.find("=") - doc = json.loads(text[pos+1:]) - elif skip_function: - text = f.read() - pos = text.find("(") - rpos = text.rfind(")") - doc = json.loads(text[pos+1:rpos]) - else: - doc = json.load(f) - f.close() - return doc + f = urlopen(url, max_age) + if skip_assignment: + text = f.read() + pos = text.find("=") + doc = json.loads(text[pos+1:]) + elif skip_function: + text = f.read() + pos = text.find("(") + rpos = text.rfind(")") + doc = json.loads(text[pos+1:rpos]) + else: + doc = json.load(f) + f.close() + return doc def exec_subprocess(cmd): - try: - p = subprocess.Popen(cmd) - ret = p.wait() - if ret != 0: - print >>sys.stderr, cmd[0], "exited with error code:", ret - return False - else: - return True - except OSError, e: - print >>sys.stderr, "Failed to run", cmd[0], e - except KeyboardInterrupt: - print "Cancelled", cmd - try: - p.terminate() - p.wait() - except KeyboardInterrupt: - p.send_signal(signal.SIGKILL) - p.wait() - return False + try: + p = subprocess.Popen(cmd) + ret = p.wait() + if ret != 0: + print >>sys.stderr, cmd[0], "exited with error code:", ret + return False + else: + return True + except OSError, e: + print >>sys.stderr, "Failed to run", cmd[0], e + except KeyboardInterrupt: + print "Cancelled", cmd + try: + p.terminate() + p.wait() + except KeyboardInterrupt: + p.send_signal(signal.SIGKILL) + p.wait() + return False def convert_flv_mp4(orig_filename): - basename = os.path.splitext(orig_filename)[0] - flv_filename = basename + ".flv" - mp4_filename = basename + ".mp4" - if orig_filename != flv_filename: - os.rename(orig_filename, flv_filename) - print "Converting %s to mp4" % flv_filename - cmd = [ - "ffmpeg", - "-i", flv_filename, - "-acodec", "copy", - "-vcodec", "copy", - mp4_filename, - ] - if not exec_subprocess(cmd): - return - try: - flv_size = os.stat(flv_filename).st_size - mp4_size = os.stat(mp4_filename).st_size - if abs(flv_size - mp4_size) < 0.05 * flv_size: - os.unlink(flv_filename) - else: - print >>sys.stderr, "The size of", mp4_filename, "is suspicious, did ffmpeg fail?" - except Exception, e: - print "Conversion failed", e + basename = os.path.splitext(orig_filename)[0] + flv_filename = basename + ".flv" + mp4_filename = basename + ".mp4" + if orig_filename != flv_filename: + os.rename(orig_filename, flv_filename) + print "Converting %s to mp4" % flv_filename + cmd = [ + "ffmpeg", + "-i", flv_filename, + "-acodec", "copy", + "-vcodec", "copy", + mp4_filename, + ] + if not exec_subprocess(cmd): + return + try: + flv_size = os.stat(flv_filename).st_size + mp4_size = os.stat(mp4_filename).st_size + if abs(flv_size - mp4_size) < 0.05 * flv_size: + os.unlink(flv_filename) + else: + print >>sys.stderr, "The size of", mp4_filename, "is suspicious, did ffmpeg fail?" + except Exception, e: + print "Conversion failed", e def convert_filename(filename): - if os.path.splitext(filename.lower())[1] in (".mp4", ".flv"): - f = open(filename) - fourcc = f.read(4) - f.close() - if fourcc == "FLV\x01": - convert_flv_mp4(filename) + if os.path.splitext(filename.lower())[1] in (".mp4", ".flv"): + f = open(filename) + fourcc = f.read(4) + f.close() + if fourcc == "FLV\x01": + convert_flv_mp4(filename) def download_rtmp(filename, vbase, vpath, hash_url=None): - filename = sanify_filename(filename) - print "Downloading: %s" % filename - if vpath.endswith(".flv"): - vpath = vpath[:-4] - cmd = [ - "rtmpdump", - "-o", filename, - "-r", vbase, - "-y", vpath, - ] - if hash_url is not None: - cmd += ["--swfVfy", hash_url] - if exec_subprocess(cmd): - convert_filename(filename) - return True - else: - return False + filename = sanify_filename(filename) + print "Downloading: %s" % filename + if vpath.endswith(".flv"): + vpath = vpath[:-4] + cmd = [ + "rtmpdump", + "-o", filename, + "-r", vbase, + "-y", vpath, + ] + if hash_url is not None: + cmd += ["--swfVfy", hash_url] + if exec_subprocess(cmd): + convert_filename(filename) + return True + else: + return False def download_urllib(filename, url, referrer=None): - filename = sanify_filename(filename) - print "Downloading: %s" % filename - try: - src = _urlopen(url, referrer) - dst = open(filename, "w") - while True: - buf = src.read(1024*1024) - if not buf: - break - dst.write(buf) - sys.stdout.write(".") - sys.stdout.flush() - print - except KeyboardInterrupt: - print "\nCancelled", url - return False - finally: - try: - src.close() - except: - pass - try: - dst.close() - except: - pass - - convert_filename(filename) - return True + filename = sanify_filename(filename) + print "Downloading: %s" % filename + try: + src = _urlopen(url, referrer) + dst = open(filename, "w") + while True: + buf = src.read(1024*1024) + if not buf: + break + dst.write(buf) + sys.stdout.write(".") + sys.stdout.flush() + print + except KeyboardInterrupt: + print "\nCancelled", url + return False + finally: + try: + src.close() + except: + pass + try: + dst.close() + except: + pass + + convert_filename(filename) + return True def natural_sort(l, key=None): - ignore_list = ["a", "the"] - def key_func(k): - if key is not None: - k = key(k) - k = k.lower() - newk = [] - for c in re.split("([0-9]+)", k): - c = c.strip() - if c.isdigit(): - newk.append(int(c)) - else: - for subc in c.split(): - if subc not in ignore_list: - newk.append(subc) - return newk - - return sorted(l, key=key_func) + ignore_list = ["a", "the"] + def key_func(k): + if key is not None: + k = key(k) + k = k.lower() + newk = [] + for c in re.split("([0-9]+)", k): + c = c.strip() + if c.isdigit(): + newk.append(int(c)) + else: + for subc in c.split(): + if subc not in ignore_list: + newk.append(subc) + return newk + + return sorted(l, key=key_func) def append_to_qs(url, params): - r = list(urlparse.urlsplit(url)) - qs = urlparse.parse_qs(r[3]) - for k, v in params.iteritems(): - if v is not None: - qs[k] = v - elif qs.has_key(k): - del qs[k] - r[3] = urllib.urlencode(qs, True) - url = urlparse.urlunsplit(r) - return url + r = list(urlparse.urlsplit(url)) + qs = urlparse.parse_qs(r[3]) + for k, v in params.iteritems(): + if v is not None: + qs[k] = v + elif qs.has_key(k): + del qs[k] + r[3] = urllib.urlencode(qs, True) + url = urlparse.urlunsplit(r) + return url diff --git a/grabber.py b/grabber.py index 5c3c948..90708e7 100755 --- a/grabber.py +++ b/grabber.py @@ -1,59 +1,58 @@ #!/usr/bin/python2 -# vim:ts=4:sts=4:sw=4:noet from common import load_root_node, natural_sort import sys def choose(options, allow_multi): - reverse_map = {} - for i, (key, value) in enumerate(options): - print "%3d) %s" % (i+1, key) - reverse_map[i+1] = value - print " 0) Back" - while True: - try: - values = map(int, raw_input("Choose> ").split()) - if len(values) == 0: - continue - if 0 in values: - return - values = [reverse_map[value] for value in values if value in reverse_map] - if allow_multi: - return values - else: - if len(values) == 1: - return values[0] - except (ValueError, IndexError): - print >>sys.stderr, "Invalid input, please try again" - pass + reverse_map = {} + for i, (key, value) in enumerate(options): + print "%3d) %s" % (i+1, key) + reverse_map[i+1] = value + print " 0) Back" + while True: + try: + values = map(int, raw_input("Choose> ").split()) + if len(values) == 0: + continue + if 0 in values: + return + values = [reverse_map[value] for value in values if value in reverse_map] + if allow_multi: + return values + else: + if len(values) == 1: + return values[0] + except (ValueError, IndexError): + print >>sys.stderr, "Invalid input, please try again" + pass def main(): - node = load_root_node() + node = load_root_node() - while True: - options = [] - will_download = True - for n in node.get_children(): - options.append((n.title, n)) - if not n.can_download: - will_download = False - options = natural_sort(options, key=lambda x: x[0]) - result = choose(options, allow_multi=will_download) - if result is None: - if node.parent is not None: - node = node.parent - else: - break - elif will_download: - for n in result: - if not n.download(): - raw_input("Press return to continue...\n") - else: - node = result + while True: + options = [] + will_download = True + for n in node.get_children(): + options.append((n.title, n)) + if not n.can_download: + will_download = False + options = natural_sort(options, key=lambda x: x[0]) + result = choose(options, allow_multi=will_download) + if result is None: + if node.parent is not None: + node = node.parent + else: + break + elif will_download: + for n in result: + if not n.download(): + raw_input("Press return to continue...\n") + else: + node = result if __name__ == "__main__": - try: - main() - except (KeyboardInterrupt, EOFError): - print "\nExiting..." + try: + main() + except (KeyboardInterrupt, EOFError): + print "\nExiting..." diff --git a/iview.py b/iview.py index 05d0472..c51d73f 100644 --- a/iview.py +++ b/iview.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# vim:ts=4:sts=4:sw=4:noet - from common import grab_xml, grab_json, download_rtmp, Node import itertools @@ -8,129 +5,129 @@ BASE_URL = "http://www.abc.net.au/iview/" CONFIG_URL = BASE_URL + "xml/config.xml" HASH_URL = BASE_URL + "images/iview.jpg" NS = { - "auth": "http://www.abc.net.au/iView/Services/iViewHandshaker", + "auth": "http://www.abc.net.au/iView/Services/iViewHandshaker", } class IviewNode(Node): - def __init__(self, title, parent, params, vpath): - Node.__init__(self, title, parent) - self.params = params - self.vpath = vpath - self.filename = self.title + "." + vpath.rsplit(".", 1)[1] - self.can_download = True - - def download(self): - auth_doc = grab_xml(self.params["auth"], 0) - server = self.params["server_streaming"] - token = auth_doc.xpath("//auth:token/text()", namespaces=NS)[0] - playpath = auth_doc.xpath("//auth:path/text()", namespaces=NS)[0] - if playpath == "playback/_definst_/": - playpath = "flash/" + playpath - vbase = server + "?auth=" + token - vpath, ext = self.vpath.rsplit(".", 1) - vpath = ext + ":" + playpath + vpath - return download_rtmp(self.filename, vbase, vpath, HASH_URL) + def __init__(self, title, parent, params, vpath): + Node.__init__(self, title, parent) + self.params = params + self.vpath = vpath + self.filename = self.title + "." + vpath.rsplit(".", 1)[1] + self.can_download = True + + def download(self): + auth_doc = grab_xml(self.params["auth"], 0) + server = self.params["server_streaming"] + token = auth_doc.xpath("//auth:token/text()", namespaces=NS)[0] + playpath = auth_doc.xpath("//auth:path/text()", namespaces=NS)[0] + if playpath == "playback/_definst_/": + playpath = "flash/" + playpath + vbase = server + "?auth=" + token + vpath, ext = self.vpath.rsplit(".", 1) + vpath = ext + ":" + playpath + vpath + return download_rtmp(self.filename, vbase, vpath, HASH_URL) class IviewSeriesNode(Node): - def __init__(self, title, parent, params, series_ids): - Node.__init__(self, title, parent) - self.params = params - self.series_ids = series_ids - - def fill_children(self): - for series_id in self.series_ids: - self.fill_children_for_id(series_id) - - def fill_children_for_id(self, series_id): - series_doc = grab_json(self.params["api"] + "series=" + series_id, 3600) - for episode_list in series_doc: - if episode_list["a"] == series_id: - episode_list = episode_list["f"] - break - else: - return - - for episode in episode_list: - vpath = episode["n"] - episode_title = episode["b"].strip() - if not episode_title.startswith(self.title): - episode_title = self.title + " " + episode_title - if episode_title.lower().endswith(" (final)"): - episode_title = episode_title[:-8] - IviewNode(episode_title, self, self.params, vpath) + def __init__(self, title, parent, params, series_ids): + Node.__init__(self, title, parent) + self.params = params + self.series_ids = series_ids + + def fill_children(self): + for series_id in self.series_ids: + self.fill_children_for_id(series_id) + + def fill_children_for_id(self, series_id): + series_doc = grab_json(self.params["api"] + "series=" + series_id, 3600) + for episode_list in series_doc: + if episode_list["a"] == series_id: + episode_list = episode_list["f"] + break + else: + return + + for episode in episode_list: + vpath = episode["n"] + episode_title = episode["b"].strip() + if not episode_title.startswith(self.title): + episode_title = self.title + " " + episode_title + if episode_title.lower().endswith(" (final)"): + episode_title = episode_title[:-8] + IviewNode(episode_title, self, self.params, vpath) class SeriesInfo(object): - def __init__(self, title): - self.title = title - self.series_ids = set() - self.categories = set() + def __init__(self, title): + self.title = title + self.series_ids = set() + self.categories = set() - def add_series_id(self, series_id): - self.series_ids.add(series_id) + def add_series_id(self, series_id): + self.series_ids.add(series_id) - def add_categories(self, categories): - self.categories.update(categories) + def add_categories(self, categories): + self.categories.update(categories) class IviewRootNode(Node): - def __init__(self, parent): - Node.__init__(self, "ABC iView", parent) - self.params = {} - self.series_info = {} - self.categories_map = {} - - def load_params(self): - config_doc = grab_xml(CONFIG_URL, 24*3600) - for p in config_doc.xpath("/config/param"): - key = p.attrib["name"] - value = p.attrib["value"] - self.params[key] = value - - def load_series(self): - series_list_doc = grab_json(self.params["api"] + "seriesIndex", 3600) - for series in series_list_doc: - title = series["b"].replace("&", "&") - sid = series["a"] - categories = series["e"].split() - info = self.series_info.get(title, None) - if not info: - info = SeriesInfo(title) - self.series_info[title] = info - info.add_categories(categories) - info.add_series_id(sid) - - def load_categories(self): - categories_doc = grab_xml(BASE_URL + self.params["categories"], 24*3600) - by_channel = Node("By Channel", self) - by_genre = Node("By Genre", self) - for category in categories_doc.xpath("//category"): - cid = category.attrib["id"] - category_name = category.xpath("name/text()")[0] - if "genre" in category.attrib: - parent = by_genre - elif cid in ["abc1", "abc2", "abc3", "abc4", "original"]: - parent = by_channel - elif cid in ["featured", "recent", "last-chance", "trailers"]: - parent = self - else: - continue - node = Node(category_name, parent) - self.categories_map[cid] = node - - def link_series(self): - # Create a duplicate within each category for each series - for s in self.series_info.itervalues(): - for cid in s.categories: - parent = self.categories_map.get(cid) - if parent: - IviewSeriesNode(s.title, parent, self.params, s.series_ids) - - def fill_children(self): - self.load_params() - self.load_series() - self.load_categories() - self.link_series() + def __init__(self, parent): + Node.__init__(self, "ABC iView", parent) + self.params = {} + self.series_info = {} + self.categories_map = {} + + def load_params(self): + config_doc = grab_xml(CONFIG_URL, 24*3600) + for p in config_doc.xpath("/config/param"): + key = p.attrib["name"] + value = p.attrib["value"] + self.params[key] = value + + def load_series(self): + series_list_doc = grab_json(self.params["api"] + "seriesIndex", 3600) + for series in series_list_doc: + title = series["b"].replace("&", "&") + sid = series["a"] + categories = series["e"].split() + info = self.series_info.get(title, None) + if not info: + info = SeriesInfo(title) + self.series_info[title] = info + info.add_categories(categories) + info.add_series_id(sid) + + def load_categories(self): + categories_doc = grab_xml(BASE_URL + self.params["categories"], 24*3600) + by_channel = Node("By Channel", self) + by_genre = Node("By Genre", self) + for category in categories_doc.xpath("//category"): + cid = category.attrib["id"] + category_name = category.xpath("name/text()")[0] + if "genre" in category.attrib: + parent = by_genre + elif cid in ["abc1", "abc2", "abc3", "abc4", "original"]: + parent = by_channel + elif cid in ["featured", "recent", "last-chance", "trailers"]: + parent = self + else: + continue + node = Node(category_name, parent) + self.categories_map[cid] = node + + def link_series(self): + # Create a duplicate within each category for each series + for s in self.series_info.itervalues(): + for cid in s.categories: + parent = self.categories_map.get(cid) + if parent: + IviewSeriesNode(s.title, parent, self.params, s.series_ids) + + def fill_children(self): + self.load_params() + self.load_series() + self.load_categories() + self.link_series() def fill_nodes(root_node): - IviewRootNode(root_node) + IviewRootNode(root_node) diff --git a/plus7.py b/plus7.py index 44727ee..9414931 100644 --- a/plus7.py +++ b/plus7.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# vim:ts=4:sts=4:sw=4:noet - import json import random import string @@ -14,124 +11,124 @@ BROWSE = BASE + "/plus7/browse/" METADATA_BASE = "http://video.query.yahoo.com/v1/public/yql?" METADATA_QUERY = { - 'q': 'SELECT streams,status FROM yahoo.media.video.streams WHERE id="%s" AND format="mp4,flv" AND protocol="rtmp,http" AND plrs="%s" AND offnetwork="false" AND site="autv_plus7" AND lang="en-AU" AND region="AU" AND override="none";', - 'callback': 'jsonp_callback', - 'env': 'prod', - 'format': 'json' + 'q': 'SELECT streams,status FROM yahoo.media.video.streams WHERE id="%s" AND format="mp4,flv" AND protocol="rtmp,http" AND plrs="%s" AND offnetwork="false" AND site="autv_plus7" AND lang="en-AU" AND region="AU" AND override="none";', + 'callback': 'jsonp_callback', + 'env': 'prod', + 'format': 'json' } HASH_URL = "http://d.yimg.com/m/up/ypp/au/player.swf" def extract_and_remove(tokens, key): - lowertokens = [x.lower() for x in tokens] - pos = lowertokens.index(key) + lowertokens = [x.lower() for x in tokens] + pos = lowertokens.index(key) - value = int(tokens[pos+1]) - tokens = tokens[:pos] + tokens[pos+2:] + value = int(tokens[pos+1]) + tokens = tokens[:pos] + tokens[pos+2:] - return value, tokens + return value, tokens def demangle_title(title, subtitle): - tokens = title.split() - insert_pos = len(tokens) - if subtitle: - insert_pos += 1 - tokens += ["-"] + subtitle.split() - - try: - season, tokens = extract_and_remove(tokens, "series") - episode, tokens = extract_and_remove(tokens, "episode") - if insert_pos < len(tokens): - tokens.insert(insert_pos, "-") - tokens.insert(insert_pos, "%sx%s" % (season, str(episode).zfill(2))) - except ValueError: - pass - - return " ".join(tokens) + tokens = title.split() + insert_pos = len(tokens) + if subtitle: + insert_pos += 1 + tokens += ["-"] + subtitle.split() + + try: + season, tokens = extract_and_remove(tokens, "series") + episode, tokens = extract_and_remove(tokens, "episode") + if insert_pos < len(tokens): + tokens.insert(insert_pos, "-") + tokens.insert(insert_pos, "%sx%s" % (season, str(episode).zfill(2))) + except ValueError: + pass + + return " ".join(tokens) class Plus7Node(Node): - def __init__(self, title, parent, url): - Node.__init__(self, title, parent) - self.url = url - self.can_download = True - - def get_video_id(self): - doc = grab_html(self.url, 3600) - for script in doc.xpath("//script"): - if not script.text: - continue - for line in script.text.split(";"): - line = line.strip() - if line.find("new Y.VideoPlatform.VideoPlayer") <= 0: - continue - -### vidparams = line[line.find("(")+1 : line.rfind(")")] -### vidparams = json.loads(vidparams) -### return vidparams["playlist"]["mediaItems"][0]["id"] - - # Cannot parse it as JSON :( - pos1 = line.find('"mediaItems":') - if pos1 < 0: - continue - pos2 = line.find('"id":', pos1) - if pos2 < 0: - continue - pos3 = line.find('"', pos2+5) - pos4 = line.find('"', pos2+6) - if pos3 < 0 or pos4 < 0: - continue - return line[pos3+1:pos4] - - raise Exception("Could not find video id on page " + self.url) - - def generate_session(self): - return "".join([random.choice(string.ascii_letters) for x in xrange(22)]) - - def download(self): - vid_id = self.get_video_id() - qs = dict(METADATA_QUERY.items()) # copy.. - qs["q"] = qs["q"] % (vid_id, self.generate_session()) - url = METADATA_BASE + urllib.urlencode(qs) - doc = grab_json(url, 0, skip_function=True) - stream_data = doc["query"]["results"]["mediaObj"][0]["streams"][0] - vbase = stream_data["host"] - vpath = stream_data["path"] - filename = self.title + ".flv" - return download_rtmp(filename, vbase, vpath, HASH_URL) + def __init__(self, title, parent, url): + Node.__init__(self, title, parent) + self.url = url + self.can_download = True + + def get_video_id(self): + doc = grab_html(self.url, 3600) + for script in doc.xpath("//script"): + if not script.text: + continue + for line in script.text.split(";"): + line = line.strip() + if line.find("new Y.VideoPlatform.VideoPlayer") <= 0: + continue + +### vidparams = line[line.find("(")+1 : line.rfind(")")] +### vidparams = json.loads(vidparams) +### return vidparams["playlist"]["mediaItems"][0]["id"] + + # Cannot parse it as JSON :( + pos1 = line.find('"mediaItems":') + if pos1 < 0: + continue + pos2 = line.find('"id":', pos1) + if pos2 < 0: + continue + pos3 = line.find('"', pos2+5) + pos4 = line.find('"', pos2+6) + if pos3 < 0 or pos4 < 0: + continue + return line[pos3+1:pos4] + + raise Exception("Could not find video id on page " + self.url) + + def generate_session(self): + return "".join([random.choice(string.ascii_letters) for x in xrange(22)]) + + def download(self): + vid_id = self.get_video_id() + qs = dict(METADATA_QUERY.items()) # copy.. + qs["q"] = qs["q"] % (vid_id, self.generate_session()) + url = METADATA_BASE + urllib.urlencode(qs) + doc = grab_json(url, 0, skip_function=True) + stream_data = doc["query"]["results"]["mediaObj"][0]["streams"][0] + vbase = stream_data["host"] + vpath = stream_data["path"] + filename = self.title + ".flv" + return download_rtmp(filename, vbase, vpath, HASH_URL) class Plus7Series(Node): - def __init__(self, title, parent, url): - Node.__init__(self, title, parent) - self.url = url - - def fill_children(self): - doc = grab_html(self.url, 3600) - for item in CSSSelector("#related-episodes div.itemdetails")(doc): - title = CSSSelector("span.title")(item)[0].text - subtitle = CSSSelector("span.subtitle")(item)[0].xpath("string()") - title = demangle_title(title, subtitle) - url = CSSSelector("a")(item)[0].attrib["href"] - Plus7Node(title, self, BASE + url) + def __init__(self, title, parent, url): + Node.__init__(self, title, parent) + self.url = url + + def fill_children(self): + doc = grab_html(self.url, 3600) + for item in CSSSelector("#related-episodes div.itemdetails")(doc): + title = CSSSelector("span.title")(item)[0].text + subtitle = CSSSelector("span.subtitle")(item)[0].xpath("string()") + title = demangle_title(title, subtitle) + url = CSSSelector("a")(item)[0].attrib["href"] + Plus7Node(title, self, BASE + url) class Plus7Root(Node): - def __init__(self, parent): - Node.__init__(self, "Yahoo Plus7 (broken!)", parent) - - def fill_children(self): - doc = grab_html(BROWSE, 3600) - shows = [] - for script in doc.xpath("//script"): - if not script.text or not script.text.startswith("var shows = "): - continue - shows = script.text[12:] - shows = shows.rstrip("; \n") - shows = json.loads(shows) - for show in shows: - Plus7Series(show["title"], self, show["url"]) + def __init__(self, parent): + Node.__init__(self, "Yahoo Plus7 (broken!)", parent) + + def fill_children(self): + doc = grab_html(BROWSE, 3600) + shows = [] + for script in doc.xpath("//script"): + if not script.text or not script.text.startswith("var shows = "): + continue + shows = script.text[12:] + shows = shows.rstrip("; \n") + shows = json.loads(shows) + for show in shows: + Plus7Series(show["title"], self, show["url"]) def fill_nodes(root_node): - Plus7Root(root_node) + Plus7Root(root_node) diff --git a/sbs.py b/sbs.py index 4af3632..7796a2a 100644 --- a/sbs.py +++ b/sbs.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# vim:ts=4:sts=4:sw=4:noet from common import grab_text, grab_html, grab_json, grab_xml, download_rtmp, download_urllib, Node, append_to_qs @@ -10,105 +9,105 @@ BASE = "http://www.sbs.com.au" VIDEO_MENU = BASE + "/ondemand/js/video-menu" VIDEO_URL = BASE + "/ondemand/video/single/%s" VIDEO_MAGIC = { - "v": "2.5.14", - "fp": "MAC 11,1,102,55", - "r": "FLQDD", - "g": "YNANAXRIYFYO", + "v": "2.5.14", + "fp": "MAC 11,1,102,55", + "r": "FLQDD", + "g": "YNANAXRIYFYO", } SWF_URL = "http://resources.sbs.com.au/vod/theplatform/core/current/swf/flvPlayer.swf" NS = { - "smil": "http://www.w3.org/2005/SMIL21/Language", + "smil": "http://www.w3.org/2005/SMIL21/Language", } class SbsNode(Node): - def __init__(self, title, parent, video_id): - Node.__init__(self, title, parent) - self.title = title - self.video_id = video_id.split("/")[-1] - self.can_download = True - - def download(self): - doc = grab_html(VIDEO_URL % self.video_id, 0) - meta_video = doc.xpath("//meta[@property='og:video']")[0] - swf_url = meta_video.attrib["content"] - swf_url_qs = urlparse.parse_qs(urlparse.urlparse(swf_url).query) - desc_url = swf_url_qs["v"][0] - - doc = grab_text(desc_url, 0) - doc_qs = urlparse.parse_qs(doc) - desc_url = doc_qs["releaseUrl"][0] - - doc = grab_xml(desc_url, 0) - error = doc.xpath("//smil:param[@name='exception']/@value", namespaces=NS) - if error: - raise Exception("Error downloading, SBS said: " + error[0]) - - video = doc.xpath("//smil:video", namespaces=NS)[0] - video_url = video.attrib["src"] - if not video_url: - raise Exception("Unsupported video '%s': %s" % (self.title, desc_url)) - ext = urlparse.urlsplit(video_url).path.rsplit(".", 1)[1] - filename = self.title + "." + ext - video_url = append_to_qs(video_url, VIDEO_MAGIC) - return download_urllib(filename, video_url, referrer=SWF_URL) + def __init__(self, title, parent, video_id): + Node.__init__(self, title, parent) + self.title = title + self.video_id = video_id.split("/")[-1] + self.can_download = True + + def download(self): + doc = grab_html(VIDEO_URL % self.video_id, 0) + meta_video = doc.xpath("//meta[@property='og:video']")[0] + swf_url = meta_video.attrib["content"] + swf_url_qs = urlparse.parse_qs(urlparse.urlparse(swf_url).query) + desc_url = swf_url_qs["v"][0] + + doc = grab_text(desc_url, 0) + doc_qs = urlparse.parse_qs(doc) + desc_url = doc_qs["releaseUrl"][0] + + doc = grab_xml(desc_url, 0) + error = doc.xpath("//smil:param[@name='exception']/@value", namespaces=NS) + if error: + raise Exception("Error downloading, SBS said: " + error[0]) + + video = doc.xpath("//smil:video", namespaces=NS)[0] + video_url = video.attrib["src"] + if not video_url: + raise Exception("Unsupported video '%s': %s" % (self.title, desc_url)) + ext = urlparse.urlsplit(video_url).path.rsplit(".", 1)[1] + filename = self.title + "." + ext + video_url = append_to_qs(video_url, VIDEO_MAGIC) + return download_urllib(filename, video_url, referrer=SWF_URL) class SbsNavNode(Node): - def __init__(self, title, parent, url): - Node.__init__(self, title, parent) - self.url = url - - def fill_children(self): - try: - doc = grab_json(BASE + self.url, 3600) - except ValueError: - # SBS sends XML as an error message :\ - return - if len(doc.get("entries", [])) == 0: - return - for entry in doc["entries"]: - self.fill_entry(entry) - - def fill_entry(self, entry): - title = entry["title"] - video_id = entry["id"] - SbsNode(title, self, video_id) + def __init__(self, title, parent, url): + Node.__init__(self, title, parent) + self.url = url + + def fill_children(self): + try: + doc = grab_json(BASE + self.url, 3600) + except ValueError: + # SBS sends XML as an error message :\ + return + if len(doc.get("entries", [])) == 0: + return + for entry in doc["entries"]: + self.fill_entry(entry) + + def fill_entry(self, entry): + title = entry["title"] + video_id = entry["id"] + SbsNode(title, self, video_id) class SbsRootNode(Node): - def __init__(self, parent): - Node.__init__(self, "SBS", parent) - - def fill_children(self): - menu = grab_json(VIDEO_MENU, 3600, skip_assignment=True) - for name in menu.keys(): - self.fill_category(self, menu[name]) - - def create_nav_node(self, name, parent, cat_data, url_key): - try: - url = cat_data[url_key] - except KeyError: - return - if url.strip(): - SbsNavNode(name, parent, url) - - def fill_category(self, parent, cat_data): - if not cat_data.has_key("children"): - name = cat_data["name"] - self.create_nav_node(name, parent, cat_data, "url") - return - - node = Node(cat_data["name"], parent) - self.create_nav_node("-Featured", node, cat_data, "furl") - self.create_nav_node("-Latest", node, cat_data, "url") - self.create_nav_node("-Most Popular", node, cat_data, "purl") - - children = cat_data.get("children", []) - if isinstance(children, dict): - children = [children[k] for k in sorted(children.keys())] - for child_cat in children: - self.fill_category(node, child_cat) + def __init__(self, parent): + Node.__init__(self, "SBS", parent) + + def fill_children(self): + menu = grab_json(VIDEO_MENU, 3600, skip_assignment=True) + for name in menu.keys(): + self.fill_category(self, menu[name]) + + def create_nav_node(self, name, parent, cat_data, url_key): + try: + url = cat_data[url_key] + except KeyError: + return + if url.strip(): + SbsNavNode(name, parent, url) + + def fill_category(self, parent, cat_data): + if not cat_data.has_key("children"): + name = cat_data["name"] + self.create_nav_node(name, parent, cat_data, "url") + return + + node = Node(cat_data["name"], parent) + self.create_nav_node("-Featured", node, cat_data, "furl") + self.create_nav_node("-Latest", node, cat_data, "url") + self.create_nav_node("-Most Popular", node, cat_data, "purl") + + children = cat_data.get("children", []) + if isinstance(children, dict): + children = [children[k] for k in sorted(children.keys())] + for child_cat in children: + self.fill_category(node, child_cat) def fill_nodes(root_node): - SbsRootNode(root_node) + SbsRootNode(root_node) -- 2.39.2