#!/usr/bin/python2
-# vim:ts=4:sts=4:sw=4:noet
from common import load_root_node
import fnmatch
import sys
DOWNLOAD_HISTORY_FILES = [
- ".downloaded_auto.txt",
- "downloaded_auto.txt",
+ ".downloaded_auto.txt",
+ "downloaded_auto.txt",
]
class DownloadList(object):
- def __init__(self):
- self.seen_list = set()
- for filename in DOWNLOAD_HISTORY_FILES:
- if os.path.isfile(filename):
- break
- else:
- filename = DOWNLOAD_HISTORY_FILES[0]
- try:
- self.f = open(filename, "r")
- for line in self.f:
- self.seen_list.add(line.decode("utf-8").strip())
- self.f.close()
- except Exception, e:
- print >>sys.stderr, "Could not open:", filename, e
- self.f = open(filename, "a")
-
- def has_seen(self, node):
- return node.title in self.seen_list
-
- def mark_seen(self, node):
- self.seen_list.add(node.title)
- self.f.write(node.title.encode("utf-8") + "\n")
- self.f.flush()
+ def __init__(self):
+ self.seen_list = set()
+ for filename in DOWNLOAD_HISTORY_FILES:
+ if os.path.isfile(filename):
+ break
+ else:
+ filename = DOWNLOAD_HISTORY_FILES[0]
+ try:
+ self.f = open(filename, "r")
+ for line in self.f:
+ self.seen_list.add(line.decode("utf-8").strip())
+ self.f.close()
+ except Exception, e:
+ print >>sys.stderr, "Could not open:", filename, e
+ self.f = open(filename, "a")
+
+ def has_seen(self, node):
+ return node.title in self.seen_list
+
+ def mark_seen(self, node):
+ self.seen_list.add(node.title)
+ self.f.write(node.title.encode("utf-8") + "\n")
+ self.f.flush()
def match(download_list, node, pattern, count=0):
- if node.can_download:
- if not download_list.has_seen(node):
- if node.download():
- download_list.mark_seen(node)
- else:
- print >>sys.stderr, "Failed to download!", node.title
- return
+ if node.can_download:
+ if not download_list.has_seen(node):
+ if node.download():
+ download_list.mark_seen(node)
+ else:
+ print >>sys.stderr, "Failed to download!", node.title
+ return
- if count >= len(pattern):
- print "No match found for pattern:", "/".join(pattern)
- return
- p = pattern[count]
- for child in node.get_children():
- if fnmatch.fnmatch(child.title, p):
- match(download_list, child, pattern, count+1)
+ if count >= len(pattern):
+ print "No match found for pattern:", "/".join(pattern)
+ return
+ p = pattern[count]
+ for child in node.get_children():
+ if fnmatch.fnmatch(child.title, p):
+ match(download_list, child, pattern, count+1)
def main(destdir, patternfile):
- os.chdir(destdir)
- node = load_root_node()
- download_list = DownloadList()
+ os.chdir(destdir)
+ node = load_root_node()
+ download_list = DownloadList()
- for line in open(patternfile):
- search = line.strip().split("/")
- match(download_list, node, search)
+ for line in open(patternfile):
+ search = line.strip().split("/")
+ match(download_list, node, search)
if __name__ == "__main__":
- try:
- destdir = os.path.abspath(sys.argv[1])
- patternfile = os.path.abspath(sys.argv[2])
- except IndexError:
- print >>sys.stderr, "Usage: %s destdir patternfile" % sys.argv[0]
- sys.exit(1)
- try:
- main(destdir, patternfile)
- except (KeyboardInterrupt, EOFError):
- print "\nExiting..."
+ try:
+ destdir = os.path.abspath(sys.argv[1])
+ patternfile = os.path.abspath(sys.argv[2])
+ except IndexError:
+ print >>sys.stderr, "Usage: %s destdir patternfile" % sys.argv[0]
+ sys.exit(1)
+ try:
+ main(destdir, patternfile)
+ except (KeyboardInterrupt, EOFError):
+ print "\nExiting..."
-# vim:ts=4:sts=4:sw=4:noet
-
import subprocess
import sys
def detect_gnome():
- """ Gnome via python-gconf """
- from gconf import client_get_default
- gconf_client = client_get_default()
- mode = gconf_client.get_string("/system/proxy/mode")
- if mode != "manual":
- return None, None
- host = gconf_client.get_string("/system/proxy/socks_host")
- port = gconf_client.get_int("/system/proxy/socks_port")
- return host, port
+ """ Gnome via python-gconf """
+ from gconf import client_get_default
+ gconf_client = client_get_default()
+ mode = gconf_client.get_string("/system/proxy/mode")
+ if mode != "manual":
+ return None, None
+ host = gconf_client.get_string("/system/proxy/socks_host")
+ port = gconf_client.get_int("/system/proxy/socks_port")
+ return host, port
def detect_osx():
- """ OS X 10.5 and up via PyObjC """
- from SystemConfiguration import SCDynamicStoreCopyProxies
- osx_proxy = SCDynamicStoreCopyProxies(None)
- if osx_proxy.get("SOCKSEnable"):
- host = osx_proxy.get("SOCKSProxy")
- port = int(osx_proxy.get("SOCKSPort"))
- return host, port
- return None, None
+ """ OS X 10.5 and up via PyObjC """
+ from SystemConfiguration import SCDynamicStoreCopyProxies
+ osx_proxy = SCDynamicStoreCopyProxies(None)
+ if osx_proxy.get("SOCKSEnable"):
+ host = osx_proxy.get("SOCKSProxy")
+ port = int(osx_proxy.get("SOCKSPort"))
+ return host, port
+ return None, None
def detect_kde():
- """ KDE via command line, why no python bindings for KDE proxy settings? """
- if os.environ.get("KDE_FULL_SESSION") != "true":
- return None, None
- p = subprocess.Popen(
- [
- "kreadconfig",
- "--file",
- "kioslaverc",
- "--group",
- "Proxy Settings",
- "--key",
- "socksProxy",
- ],
- shell=True,
- stdout=subprocess.PIPE,
- )
- host, port = p.stdout.readline()[:-1].split(":")
- p.close()
- port = int(port)
- return host, port
+ """ KDE via command line, why no python bindings for KDE proxy settings? """
+ if os.environ.get("KDE_FULL_SESSION") != "true":
+ return None, None
+ p = subprocess.Popen(
+ [
+ "kreadconfig",
+ "--file",
+ "kioslaverc",
+ "--group",
+ "Proxy Settings",
+ "--key",
+ "socksProxy",
+ ],
+ shell=True,
+ stdout=subprocess.PIPE,
+ )
+ host, port = p.stdout.readline()[:-1].split(":")
+ p.close()
+ port = int(port)
+ return host, port
def detect_env():
- """ fallback to environment variables """
- socks_environ = os.environ.get("SOCKS_SERVER")
- if not socks_environ:
- return None, None
- host, port = socks_environ
- port = int(port)
- return host, port
+ """ fallback to environment variables """
+ socks_environ = os.environ.get("SOCKS_SERVER")
+ if not socks_environ:
+ return None, None
+ host, port = socks_environ
+ port = int(port)
+ return host, port
def configure_socks(host, port):
- """ hijack socket.socket using SocksiPy """
- try:
- import socks, socket
- except ImportError:
- print >>sys.stderr, "Failed to use configured SOCKS proxy:", host, port
- print >>sys.stderr, "Try installing SocksiPy: http://socksipy.sf.net"
- return False
+ """ hijack socket.socket using SocksiPy """
+ try:
+ import socks, socket
+ except ImportError:
+ print >>sys.stderr, "Failed to use configured SOCKS proxy:", host, port
+ print >>sys.stderr, "Try installing SocksiPy: http://socksipy.sf.net"
+ return False
- socket.socket = socks.socksocket
- socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, host, port)
- return True
+ socket.socket = socks.socksocket
+ socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, host, port)
+ return True
def try_autosocks():
- functions = [
- detect_gnome,
- detect_osx,
- detect_kde,
- detect_env,
- ]
- for func in functions:
- host, port = None, None
- try:
- host, port = func()
- except Exception, e:
- pass
- if host is not None and port is not None:
- return configure_socks(host, port)
- return False
+ functions = [
+ detect_gnome,
+ detect_osx,
+ detect_kde,
+ detect_env,
+ ]
+ for func in functions:
+ host, port = None, None
+ try:
+ host, port = func()
+ except Exception, e:
+ pass
+ if host is not None and port is not None:
+ return configure_socks(host, port)
+ return False
-#!/usr/bin/env python
-# vim:ts=4:sts=4:sw=4:noet
-
import re
import sys
class BrightcoveVideoNode(Node):
- def __init__(self, title, parent, token, video_id):
- Node.__init__(self, title, parent)
- self.can_download = True
- self.token = token
- self.video_id = video_id
-
- def download(self):
- desc_url = append_to_qs(BRIGHTCOVE_API, {
- "token": self.token,
- "command": "find_video_by_id",
- "video_fields": "renditions",
- "video_id": self.video_id,
- })
- print "video desc_url", desc_url
-
- doc = grab_json(desc_url, 3600)
-
- best_encoding_rate = 0
- best_url = None
- for item in doc['renditions']:
- encoding_rate = item['encodingRate']
- if encoding_rate > best_encoding_rate:
- best_encoding_rate = encoding_rate
- best_url = item['url']
-
- if best_url is None:
- raise Exception("Could not find video URL: " + desc_url)
-
- vbase, vpath = best_url.split("&")
- filename = self.title + ".mp4"
- return download_rtmp(filename, vbase, vpath, HASH_URL)
+ def __init__(self, title, parent, token, video_id):
+ Node.__init__(self, title, parent)
+ self.can_download = True
+ self.token = token
+ self.video_id = video_id
+
+ def download(self):
+ desc_url = append_to_qs(BRIGHTCOVE_API, {
+ "token": self.token,
+ "command": "find_video_by_id",
+ "video_fields": "renditions",
+ "video_id": self.video_id,
+ })
+ print "video desc_url", desc_url
+
+ doc = grab_json(desc_url, 3600)
+
+ best_encoding_rate = 0
+ best_url = None
+ for item in doc['renditions']:
+ encoding_rate = item['encodingRate']
+ if encoding_rate > best_encoding_rate:
+ best_encoding_rate = encoding_rate
+ best_url = item['url']
+
+ if best_url is None:
+ raise Exception("Could not find video URL: " + desc_url)
+
+ vbase, vpath = best_url.split("&")
+ filename = self.title + ".mp4"
+ return download_rtmp(filename, vbase, vpath, HASH_URL)
class BrightcoveRootNode(Node):
- def __init__(self, title, parent, token):
- Node.__init__(self, title, parent)
- self.token = token
- self.series_nodes = {}
-
- def get_series_node(self, series_name):
- series_name = series_name.split("-")[0].strip()
- key = series_name.lower()
- node = self.series_nodes.get(key, None)
- if node is None:
- node = Node(series_name, self)
- self.series_nodes[key] = node
- return node
-
- def fill_children(self):
- page_number = 0
- while page_number < 100:
- sys.stdout.write(".")
- sys.stdout.flush()
- url = self.get_all_videos_url(page_number)
- page_number += 1
-
- page = grab_json(url, 3600)
- items = page["items"]
- if len(items) == 0:
- break
-
- for video_desc in items:
- self.process_video(video_desc)
- print
-
- def process_video(self, video_desc):
- if not video_desc["customFields"]:
- return
-
- video_id = video_desc["id"]
- title = self.get_video_title(video_desc)
- series_name = self.get_series_name(video_desc)
-
- parent_node = self.get_series_node(series_name)
- BrightcoveVideoNode(title, parent_node, self.token, video_id)
-
- def get_video_title(self, video_desc):
- raise NotImplementedError()
-
- def get_series_name(self, video_desc):
- raise NotImplementedError()
-
- def get_all_videos_url(self, page_number):
- raise NotImplementedError()
+ def __init__(self, title, parent, token):
+ Node.__init__(self, title, parent)
+ self.token = token
+ self.series_nodes = {}
+
+ def get_series_node(self, series_name):
+ series_name = series_name.split("-")[0].strip()
+ key = series_name.lower()
+ node = self.series_nodes.get(key, None)
+ if node is None:
+ node = Node(series_name, self)
+ self.series_nodes[key] = node
+ return node
+
+ def fill_children(self):
+ page_number = 0
+ while page_number < 100:
+ sys.stdout.write(".")
+ sys.stdout.flush()
+ url = self.get_all_videos_url(page_number)
+ page_number += 1
+
+ page = grab_json(url, 3600)
+ items = page["items"]
+ if len(items) == 0:
+ break
+
+ for video_desc in items:
+ self.process_video(video_desc)
+ print
+
+ def process_video(self, video_desc):
+ if not video_desc["customFields"]:
+ return
+
+ video_id = video_desc["id"]
+ title = self.get_video_title(video_desc)
+ series_name = self.get_series_name(video_desc)
+
+ parent_node = self.get_series_node(series_name)
+ BrightcoveVideoNode(title, parent_node, self.token, video_id)
+
+ def get_video_title(self, video_desc):
+ raise NotImplementedError()
+
+ def get_series_name(self, video_desc):
+ raise NotImplementedError()
+
+ def get_all_videos_url(self, page_number):
+ raise NotImplementedError()
class Ch9RootNode(BrightcoveRootNode):
- def __init__(self, root_node):
- BrightcoveRootNode.__init__(self, "Nine", root_node, CH9_TOKEN)
-
- def get_video_title(self, video_desc):
- title = video_desc["name"]
- series = video_desc["customFields"]["series"]
- season = video_desc["customFields"].get("season", "")
- episode = video_desc["customFields"].get("episode", "1").rjust(2, "0")
-
- if re.match(R"ep(isode)?\s*[0-9]+\s*:", title.lower()):
- title = title.split(":", 1)[1].strip()
-
- title = "%s - %sx%s - %s" % (
- series,
- season,
- episode,
- title
- )
- return title
-
- def get_series_name(self, video_desc):
- return video_desc["customFields"]["series"]
-
- def get_all_videos_url(self, page_number):
- return append_to_qs(BRIGHTCOVE_API, {
- "token": self.token,
- "command": "search_videos",
- "video_fields": "id,name,customFields",
- "custom_fields": "series,season,episode",
- "sort_by": "PUBLISH_DATE",
- "page_number": str(page_number),
- })
+ def __init__(self, root_node):
+ BrightcoveRootNode.__init__(self, "Nine", root_node, CH9_TOKEN)
+
+ def get_video_title(self, video_desc):
+ title = video_desc["name"]
+ series = video_desc["customFields"]["series"]
+ season = video_desc["customFields"].get("season", "")
+ episode = video_desc["customFields"].get("episode", "1").rjust(2, "0")
+
+ if re.match(R"ep(isode)?\s*[0-9]+\s*:", title.lower()):
+ title = title.split(":", 1)[1].strip()
+
+ title = "%s - %sx%s - %s" % (
+ series,
+ season,
+ episode,
+ title
+ )
+ return title
+
+ def get_series_name(self, video_desc):
+ return video_desc["customFields"]["series"]
+
+ def get_all_videos_url(self, page_number):
+ return append_to_qs(BRIGHTCOVE_API, {
+ "token": self.token,
+ "command": "search_videos",
+ "video_fields": "id,name,customFields",
+ "custom_fields": "series,season,episode",
+ "sort_by": "PUBLISH_DATE",
+ "page_number": str(page_number),
+ })
class Ch10RootNode(BrightcoveRootNode):
- def __init__(self, root_node):
- BrightcoveRootNode.__init__(self, "Ten", root_node, CH10_TOKEN)
+ def __init__(self, root_node):
+ BrightcoveRootNode.__init__(self, "Ten", root_node, CH10_TOKEN)
- def get_video_title(self, video_desc):
- return video_desc["name"]
+ def get_video_title(self, video_desc):
+ return video_desc["name"]
- def get_series_name(self, video_desc):
- return video_desc["customFields"]["tv_show"]
+ def get_series_name(self, video_desc):
+ return video_desc["customFields"]["tv_show"]
- def get_all_videos_url(self, page_number):
- return append_to_qs(BRIGHTCOVE_API, {
- "token": self.token,
- "command": "search_videos",
- "video_fields": "id,name,customFields",
- "custom_fields": "tv_show",
- "sort_by": "PUBLISH_DATE",
- "any": "video_type_long_form:Full Episode",
- "page_number": str(page_number),
- })
+ def get_all_videos_url(self, page_number):
+ return append_to_qs(BRIGHTCOVE_API, {
+ "token": self.token,
+ "command": "search_videos",
+ "video_fields": "id,name,customFields",
+ "custom_fields": "tv_show",
+ "sort_by": "PUBLISH_DATE",
+ "any": "video_type_long_form:Full Episode",
+ "page_number": str(page_number),
+ })
def fill_nodes(root_node):
- Ch9RootNode(root_node)
- Ch10RootNode(root_node)
+ Ch9RootNode(root_node)
+ Ch10RootNode(root_node)
-# vim:ts=4:sts=4:sw=4:noet
-
from lxml import etree, html
import cookielib
import json
try:
- import hashlib
+ import hashlib
except ImportError:
- import md5 as hashlib
+ import md5 as hashlib
import os
import re
import shutil
try:
- import autosocks
- autosocks.try_autosocks()
+ import autosocks
+ autosocks.try_autosocks()
except ImportError:
- pass
+ pass
CACHE_DIR = os.path.expanduser("~/.cache/webdl")
USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:21.0) Gecko/20100101 Firefox/21.0"
class Node(object):
- def __init__(self, title, parent=None):
- self.title = title
- if parent:
- parent.children.append(self)
- self.parent = parent
- self.children = []
- self.can_download = False
+ def __init__(self, title, parent=None):
+ self.title = title
+ if parent:
+ parent.children.append(self)
+ self.parent = parent
+ self.children = []
+ self.can_download = False
- def get_children(self):
- if not self.children:
- self.fill_children()
- return self.children
+ def get_children(self):
+ if not self.children:
+ self.fill_children()
+ return self.children
- def fill_children(self):
- pass
+ def fill_children(self):
+ pass
- def download(self):
- raise NotImplemented
+ def download(self):
+ raise NotImplemented
def load_root_node():
- root_node = Node("Root")
+ root_node = Node("Root")
- import iview
- iview.fill_nodes(root_node)
+ import iview
+ iview.fill_nodes(root_node)
- import sbs
- sbs.fill_nodes(root_node)
+ import sbs
+ sbs.fill_nodes(root_node)
- import plus7
- plus7.fill_nodes(root_node)
+ import plus7
+ plus7.fill_nodes(root_node)
- import brightcove
- brightcove.fill_nodes(root_node)
+ import brightcove
+ brightcove.fill_nodes(root_node)
- return root_node
+ return root_node
valid_chars = frozenset("-_.()!@#%^ abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
def sanify_filename(filename):
- filename = filename.encode("ascii", "ignore")
- filename = "".join(c for c in filename if c in valid_chars)
- return filename
+ filename = filename.encode("ascii", "ignore")
+ filename = "".join(c for c in filename if c in valid_chars)
+ return filename
cookiejar = cookielib.CookieJar()
urlopener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar))
def _urlopen(url, referrer=None):
- req = urllib2.Request(url)
- req.add_header("User-Agent", USER_AGENT)
- if referrer:
- req.add_header("Referer", referrer)
- return urlopener.open(req)
+ req = urllib2.Request(url)
+ req.add_header("User-Agent", USER_AGENT)
+ if referrer:
+ req.add_header("Referer", referrer)
+ return urlopener.open(req)
def urlopen(url, max_age):
-### print url
- if not os.path.isdir(CACHE_DIR):
- os.makedirs(CACHE_DIR)
-
- if max_age <= 0:
- return _urlopen(url)
-
- filename = hashlib.md5(url).hexdigest()
- filename = os.path.join(CACHE_DIR, filename)
- if os.path.exists(filename):
- file_age = int(time.time()) - os.path.getmtime(filename)
- if file_age < max_age:
- return open(filename)
-
- src = _urlopen(url)
- dst = open(filename, "w")
- try:
- shutil.copyfileobj(src, dst)
- except Exception, e:
- try:
- os.unlink(filename)
- except OSError:
- pass
- raise e
- src.close()
- dst.close()
-
- return open(filename)
+### print url
+ if not os.path.isdir(CACHE_DIR):
+ os.makedirs(CACHE_DIR)
+
+ if max_age <= 0:
+ return _urlopen(url)
+
+ filename = hashlib.md5(url).hexdigest()
+ filename = os.path.join(CACHE_DIR, filename)
+ if os.path.exists(filename):
+ file_age = int(time.time()) - os.path.getmtime(filename)
+ if file_age < max_age:
+ return open(filename)
+
+ src = _urlopen(url)
+ dst = open(filename, "w")
+ try:
+ shutil.copyfileobj(src, dst)
+ except Exception, e:
+ try:
+ os.unlink(filename)
+ except OSError:
+ pass
+ raise e
+ src.close()
+ dst.close()
+
+ return open(filename)
def grab_text(url, max_age):
- f = urlopen(url, max_age)
- text = f.read().decode("utf-8")
- f.close()
- return text
+ f = urlopen(url, max_age)
+ text = f.read().decode("utf-8")
+ f.close()
+ return text
def grab_html(url, max_age):
- f = urlopen(url, max_age)
- doc = html.parse(f, html.HTMLParser(encoding="utf-8", recover=True))
- f.close()
- return doc
+ f = urlopen(url, max_age)
+ doc = html.parse(f, html.HTMLParser(encoding="utf-8", recover=True))
+ f.close()
+ return doc
def grab_xml(url, max_age):
- f = urlopen(url, max_age)
- doc = etree.parse(f, etree.XMLParser(encoding="utf-8", recover=True))
- f.close()
- return doc
+ f = urlopen(url, max_age)
+ doc = etree.parse(f, etree.XMLParser(encoding="utf-8", recover=True))
+ f.close()
+ return doc
def grab_json(url, max_age, skip_assignment=False, skip_function=False):
- f = urlopen(url, max_age)
- if skip_assignment:
- text = f.read()
- pos = text.find("=")
- doc = json.loads(text[pos+1:])
- elif skip_function:
- text = f.read()
- pos = text.find("(")
- rpos = text.rfind(")")
- doc = json.loads(text[pos+1:rpos])
- else:
- doc = json.load(f)
- f.close()
- return doc
+ f = urlopen(url, max_age)
+ if skip_assignment:
+ text = f.read()
+ pos = text.find("=")
+ doc = json.loads(text[pos+1:])
+ elif skip_function:
+ text = f.read()
+ pos = text.find("(")
+ rpos = text.rfind(")")
+ doc = json.loads(text[pos+1:rpos])
+ else:
+ doc = json.load(f)
+ f.close()
+ return doc
def exec_subprocess(cmd):
- try:
- p = subprocess.Popen(cmd)
- ret = p.wait()
- if ret != 0:
- print >>sys.stderr, cmd[0], "exited with error code:", ret
- return False
- else:
- return True
- except OSError, e:
- print >>sys.stderr, "Failed to run", cmd[0], e
- except KeyboardInterrupt:
- print "Cancelled", cmd
- try:
- p.terminate()
- p.wait()
- except KeyboardInterrupt:
- p.send_signal(signal.SIGKILL)
- p.wait()
- return False
+ try:
+ p = subprocess.Popen(cmd)
+ ret = p.wait()
+ if ret != 0:
+ print >>sys.stderr, cmd[0], "exited with error code:", ret
+ return False
+ else:
+ return True
+ except OSError, e:
+ print >>sys.stderr, "Failed to run", cmd[0], e
+ except KeyboardInterrupt:
+ print "Cancelled", cmd
+ try:
+ p.terminate()
+ p.wait()
+ except KeyboardInterrupt:
+ p.send_signal(signal.SIGKILL)
+ p.wait()
+ return False
def convert_flv_mp4(orig_filename):
- basename = os.path.splitext(orig_filename)[0]
- flv_filename = basename + ".flv"
- mp4_filename = basename + ".mp4"
- if orig_filename != flv_filename:
- os.rename(orig_filename, flv_filename)
- print "Converting %s to mp4" % flv_filename
- cmd = [
- "ffmpeg",
- "-i", flv_filename,
- "-acodec", "copy",
- "-vcodec", "copy",
- mp4_filename,
- ]
- if not exec_subprocess(cmd):
- return
- try:
- flv_size = os.stat(flv_filename).st_size
- mp4_size = os.stat(mp4_filename).st_size
- if abs(flv_size - mp4_size) < 0.05 * flv_size:
- os.unlink(flv_filename)
- else:
- print >>sys.stderr, "The size of", mp4_filename, "is suspicious, did ffmpeg fail?"
- except Exception, e:
- print "Conversion failed", e
+ basename = os.path.splitext(orig_filename)[0]
+ flv_filename = basename + ".flv"
+ mp4_filename = basename + ".mp4"
+ if orig_filename != flv_filename:
+ os.rename(orig_filename, flv_filename)
+ print "Converting %s to mp4" % flv_filename
+ cmd = [
+ "ffmpeg",
+ "-i", flv_filename,
+ "-acodec", "copy",
+ "-vcodec", "copy",
+ mp4_filename,
+ ]
+ if not exec_subprocess(cmd):
+ return
+ try:
+ flv_size = os.stat(flv_filename).st_size
+ mp4_size = os.stat(mp4_filename).st_size
+ if abs(flv_size - mp4_size) < 0.05 * flv_size:
+ os.unlink(flv_filename)
+ else:
+ print >>sys.stderr, "The size of", mp4_filename, "is suspicious, did ffmpeg fail?"
+ except Exception, e:
+ print "Conversion failed", e
def convert_filename(filename):
- if os.path.splitext(filename.lower())[1] in (".mp4", ".flv"):
- f = open(filename)
- fourcc = f.read(4)
- f.close()
- if fourcc == "FLV\x01":
- convert_flv_mp4(filename)
+ if os.path.splitext(filename.lower())[1] in (".mp4", ".flv"):
+ f = open(filename)
+ fourcc = f.read(4)
+ f.close()
+ if fourcc == "FLV\x01":
+ convert_flv_mp4(filename)
def download_rtmp(filename, vbase, vpath, hash_url=None):
- filename = sanify_filename(filename)
- print "Downloading: %s" % filename
- if vpath.endswith(".flv"):
- vpath = vpath[:-4]
- cmd = [
- "rtmpdump",
- "-o", filename,
- "-r", vbase,
- "-y", vpath,
- ]
- if hash_url is not None:
- cmd += ["--swfVfy", hash_url]
- if exec_subprocess(cmd):
- convert_filename(filename)
- return True
- else:
- return False
+ filename = sanify_filename(filename)
+ print "Downloading: %s" % filename
+ if vpath.endswith(".flv"):
+ vpath = vpath[:-4]
+ cmd = [
+ "rtmpdump",
+ "-o", filename,
+ "-r", vbase,
+ "-y", vpath,
+ ]
+ if hash_url is not None:
+ cmd += ["--swfVfy", hash_url]
+ if exec_subprocess(cmd):
+ convert_filename(filename)
+ return True
+ else:
+ return False
def download_urllib(filename, url, referrer=None):
- filename = sanify_filename(filename)
- print "Downloading: %s" % filename
- try:
- src = _urlopen(url, referrer)
- dst = open(filename, "w")
- while True:
- buf = src.read(1024*1024)
- if not buf:
- break
- dst.write(buf)
- sys.stdout.write(".")
- sys.stdout.flush()
- print
- except KeyboardInterrupt:
- print "\nCancelled", url
- return False
- finally:
- try:
- src.close()
- except:
- pass
- try:
- dst.close()
- except:
- pass
-
- convert_filename(filename)
- return True
+ filename = sanify_filename(filename)
+ print "Downloading: %s" % filename
+ try:
+ src = _urlopen(url, referrer)
+ dst = open(filename, "w")
+ while True:
+ buf = src.read(1024*1024)
+ if not buf:
+ break
+ dst.write(buf)
+ sys.stdout.write(".")
+ sys.stdout.flush()
+ print
+ except KeyboardInterrupt:
+ print "\nCancelled", url
+ return False
+ finally:
+ try:
+ src.close()
+ except:
+ pass
+ try:
+ dst.close()
+ except:
+ pass
+
+ convert_filename(filename)
+ return True
def natural_sort(l, key=None):
- ignore_list = ["a", "the"]
- def key_func(k):
- if key is not None:
- k = key(k)
- k = k.lower()
- newk = []
- for c in re.split("([0-9]+)", k):
- c = c.strip()
- if c.isdigit():
- newk.append(int(c))
- else:
- for subc in c.split():
- if subc not in ignore_list:
- newk.append(subc)
- return newk
-
- return sorted(l, key=key_func)
+ ignore_list = ["a", "the"]
+ def key_func(k):
+ if key is not None:
+ k = key(k)
+ k = k.lower()
+ newk = []
+ for c in re.split("([0-9]+)", k):
+ c = c.strip()
+ if c.isdigit():
+ newk.append(int(c))
+ else:
+ for subc in c.split():
+ if subc not in ignore_list:
+ newk.append(subc)
+ return newk
+
+ return sorted(l, key=key_func)
def append_to_qs(url, params):
- r = list(urlparse.urlsplit(url))
- qs = urlparse.parse_qs(r[3])
- for k, v in params.iteritems():
- if v is not None:
- qs[k] = v
- elif qs.has_key(k):
- del qs[k]
- r[3] = urllib.urlencode(qs, True)
- url = urlparse.urlunsplit(r)
- return url
+ r = list(urlparse.urlsplit(url))
+ qs = urlparse.parse_qs(r[3])
+ for k, v in params.iteritems():
+ if v is not None:
+ qs[k] = v
+ elif qs.has_key(k):
+ del qs[k]
+ r[3] = urllib.urlencode(qs, True)
+ url = urlparse.urlunsplit(r)
+ return url
#!/usr/bin/python2
-# vim:ts=4:sts=4:sw=4:noet
from common import load_root_node, natural_sort
import sys
def choose(options, allow_multi):
- reverse_map = {}
- for i, (key, value) in enumerate(options):
- print "%3d) %s" % (i+1, key)
- reverse_map[i+1] = value
- print " 0) Back"
- while True:
- try:
- values = map(int, raw_input("Choose> ").split())
- if len(values) == 0:
- continue
- if 0 in values:
- return
- values = [reverse_map[value] for value in values if value in reverse_map]
- if allow_multi:
- return values
- else:
- if len(values) == 1:
- return values[0]
- except (ValueError, IndexError):
- print >>sys.stderr, "Invalid input, please try again"
- pass
+ reverse_map = {}
+ for i, (key, value) in enumerate(options):
+ print "%3d) %s" % (i+1, key)
+ reverse_map[i+1] = value
+ print " 0) Back"
+ while True:
+ try:
+ values = map(int, raw_input("Choose> ").split())
+ if len(values) == 0:
+ continue
+ if 0 in values:
+ return
+ values = [reverse_map[value] for value in values if value in reverse_map]
+ if allow_multi:
+ return values
+ else:
+ if len(values) == 1:
+ return values[0]
+ except (ValueError, IndexError):
+ print >>sys.stderr, "Invalid input, please try again"
+ pass
def main():
- node = load_root_node()
+ node = load_root_node()
- while True:
- options = []
- will_download = True
- for n in node.get_children():
- options.append((n.title, n))
- if not n.can_download:
- will_download = False
- options = natural_sort(options, key=lambda x: x[0])
- result = choose(options, allow_multi=will_download)
- if result is None:
- if node.parent is not None:
- node = node.parent
- else:
- break
- elif will_download:
- for n in result:
- if not n.download():
- raw_input("Press return to continue...\n")
- else:
- node = result
+ while True:
+ options = []
+ will_download = True
+ for n in node.get_children():
+ options.append((n.title, n))
+ if not n.can_download:
+ will_download = False
+ options = natural_sort(options, key=lambda x: x[0])
+ result = choose(options, allow_multi=will_download)
+ if result is None:
+ if node.parent is not None:
+ node = node.parent
+ else:
+ break
+ elif will_download:
+ for n in result:
+ if not n.download():
+ raw_input("Press return to continue...\n")
+ else:
+ node = result
if __name__ == "__main__":
- try:
- main()
- except (KeyboardInterrupt, EOFError):
- print "\nExiting..."
+ try:
+ main()
+ except (KeyboardInterrupt, EOFError):
+ print "\nExiting..."
-#!/usr/bin/env python
-# vim:ts=4:sts=4:sw=4:noet
-
from common import grab_xml, grab_json, download_rtmp, Node
import itertools
CONFIG_URL = BASE_URL + "xml/config.xml"
HASH_URL = BASE_URL + "images/iview.jpg"
NS = {
- "auth": "http://www.abc.net.au/iView/Services/iViewHandshaker",
+ "auth": "http://www.abc.net.au/iView/Services/iViewHandshaker",
}
class IviewNode(Node):
- def __init__(self, title, parent, params, vpath):
- Node.__init__(self, title, parent)
- self.params = params
- self.vpath = vpath
- self.filename = self.title + "." + vpath.rsplit(".", 1)[1]
- self.can_download = True
-
- def download(self):
- auth_doc = grab_xml(self.params["auth"], 0)
- server = self.params["server_streaming"]
- token = auth_doc.xpath("//auth:token/text()", namespaces=NS)[0]
- playpath = auth_doc.xpath("//auth:path/text()", namespaces=NS)[0]
- if playpath == "playback/_definst_/":
- playpath = "flash/" + playpath
- vbase = server + "?auth=" + token
- vpath, ext = self.vpath.rsplit(".", 1)
- vpath = ext + ":" + playpath + vpath
- return download_rtmp(self.filename, vbase, vpath, HASH_URL)
+ def __init__(self, title, parent, params, vpath):
+ Node.__init__(self, title, parent)
+ self.params = params
+ self.vpath = vpath
+ self.filename = self.title + "." + vpath.rsplit(".", 1)[1]
+ self.can_download = True
+
+ def download(self):
+ auth_doc = grab_xml(self.params["auth"], 0)
+ server = self.params["server_streaming"]
+ token = auth_doc.xpath("//auth:token/text()", namespaces=NS)[0]
+ playpath = auth_doc.xpath("//auth:path/text()", namespaces=NS)[0]
+ if playpath == "playback/_definst_/":
+ playpath = "flash/" + playpath
+ vbase = server + "?auth=" + token
+ vpath, ext = self.vpath.rsplit(".", 1)
+ vpath = ext + ":" + playpath + vpath
+ return download_rtmp(self.filename, vbase, vpath, HASH_URL)
class IviewSeriesNode(Node):
- def __init__(self, title, parent, params, series_ids):
- Node.__init__(self, title, parent)
- self.params = params
- self.series_ids = series_ids
-
- def fill_children(self):
- for series_id in self.series_ids:
- self.fill_children_for_id(series_id)
-
- def fill_children_for_id(self, series_id):
- series_doc = grab_json(self.params["api"] + "series=" + series_id, 3600)
- for episode_list in series_doc:
- if episode_list["a"] == series_id:
- episode_list = episode_list["f"]
- break
- else:
- return
-
- for episode in episode_list:
- vpath = episode["n"]
- episode_title = episode["b"].strip()
- if not episode_title.startswith(self.title):
- episode_title = self.title + " " + episode_title
- if episode_title.lower().endswith(" (final)"):
- episode_title = episode_title[:-8]
- IviewNode(episode_title, self, self.params, vpath)
+ def __init__(self, title, parent, params, series_ids):
+ Node.__init__(self, title, parent)
+ self.params = params
+ self.series_ids = series_ids
+
+ def fill_children(self):
+ for series_id in self.series_ids:
+ self.fill_children_for_id(series_id)
+
+ def fill_children_for_id(self, series_id):
+ series_doc = grab_json(self.params["api"] + "series=" + series_id, 3600)
+ for episode_list in series_doc:
+ if episode_list["a"] == series_id:
+ episode_list = episode_list["f"]
+ break
+ else:
+ return
+
+ for episode in episode_list:
+ vpath = episode["n"]
+ episode_title = episode["b"].strip()
+ if not episode_title.startswith(self.title):
+ episode_title = self.title + " " + episode_title
+ if episode_title.lower().endswith(" (final)"):
+ episode_title = episode_title[:-8]
+ IviewNode(episode_title, self, self.params, vpath)
class SeriesInfo(object):
- def __init__(self, title):
- self.title = title
- self.series_ids = set()
- self.categories = set()
+ def __init__(self, title):
+ self.title = title
+ self.series_ids = set()
+ self.categories = set()
- def add_series_id(self, series_id):
- self.series_ids.add(series_id)
+ def add_series_id(self, series_id):
+ self.series_ids.add(series_id)
- def add_categories(self, categories):
- self.categories.update(categories)
+ def add_categories(self, categories):
+ self.categories.update(categories)
class IviewRootNode(Node):
- def __init__(self, parent):
- Node.__init__(self, "ABC iView", parent)
- self.params = {}
- self.series_info = {}
- self.categories_map = {}
-
- def load_params(self):
- config_doc = grab_xml(CONFIG_URL, 24*3600)
- for p in config_doc.xpath("/config/param"):
- key = p.attrib["name"]
- value = p.attrib["value"]
- self.params[key] = value
-
- def load_series(self):
- series_list_doc = grab_json(self.params["api"] + "seriesIndex", 3600)
- for series in series_list_doc:
- title = series["b"].replace("&", "&")
- sid = series["a"]
- categories = series["e"].split()
- info = self.series_info.get(title, None)
- if not info:
- info = SeriesInfo(title)
- self.series_info[title] = info
- info.add_categories(categories)
- info.add_series_id(sid)
-
- def load_categories(self):
- categories_doc = grab_xml(BASE_URL + self.params["categories"], 24*3600)
- by_channel = Node("By Channel", self)
- by_genre = Node("By Genre", self)
- for category in categories_doc.xpath("//category"):
- cid = category.attrib["id"]
- category_name = category.xpath("name/text()")[0]
- if "genre" in category.attrib:
- parent = by_genre
- elif cid in ["abc1", "abc2", "abc3", "abc4", "original"]:
- parent = by_channel
- elif cid in ["featured", "recent", "last-chance", "trailers"]:
- parent = self
- else:
- continue
- node = Node(category_name, parent)
- self.categories_map[cid] = node
-
- def link_series(self):
- # Create a duplicate within each category for each series
- for s in self.series_info.itervalues():
- for cid in s.categories:
- parent = self.categories_map.get(cid)
- if parent:
- IviewSeriesNode(s.title, parent, self.params, s.series_ids)
-
- def fill_children(self):
- self.load_params()
- self.load_series()
- self.load_categories()
- self.link_series()
+ def __init__(self, parent):
+ Node.__init__(self, "ABC iView", parent)
+ self.params = {}
+ self.series_info = {}
+ self.categories_map = {}
+
+ def load_params(self):
+ config_doc = grab_xml(CONFIG_URL, 24*3600)
+ for p in config_doc.xpath("/config/param"):
+ key = p.attrib["name"]
+ value = p.attrib["value"]
+ self.params[key] = value
+
+ def load_series(self):
+ series_list_doc = grab_json(self.params["api"] + "seriesIndex", 3600)
+ for series in series_list_doc:
+ title = series["b"].replace("&", "&")
+ sid = series["a"]
+ categories = series["e"].split()
+ info = self.series_info.get(title, None)
+ if not info:
+ info = SeriesInfo(title)
+ self.series_info[title] = info
+ info.add_categories(categories)
+ info.add_series_id(sid)
+
+ def load_categories(self):
+ categories_doc = grab_xml(BASE_URL + self.params["categories"], 24*3600)
+ by_channel = Node("By Channel", self)
+ by_genre = Node("By Genre", self)
+ for category in categories_doc.xpath("//category"):
+ cid = category.attrib["id"]
+ category_name = category.xpath("name/text()")[0]
+ if "genre" in category.attrib:
+ parent = by_genre
+ elif cid in ["abc1", "abc2", "abc3", "abc4", "original"]:
+ parent = by_channel
+ elif cid in ["featured", "recent", "last-chance", "trailers"]:
+ parent = self
+ else:
+ continue
+ node = Node(category_name, parent)
+ self.categories_map[cid] = node
+
+ def link_series(self):
+ # Create a duplicate within each category for each series
+ for s in self.series_info.itervalues():
+ for cid in s.categories:
+ parent = self.categories_map.get(cid)
+ if parent:
+ IviewSeriesNode(s.title, parent, self.params, s.series_ids)
+
+ def fill_children(self):
+ self.load_params()
+ self.load_series()
+ self.load_categories()
+ self.link_series()
def fill_nodes(root_node):
- IviewRootNode(root_node)
+ IviewRootNode(root_node)
-#!/usr/bin/env python
-# vim:ts=4:sts=4:sw=4:noet
-
import json
import random
import string
METADATA_BASE = "http://video.query.yahoo.com/v1/public/yql?"
METADATA_QUERY = {
- 'q': 'SELECT streams,status FROM yahoo.media.video.streams WHERE id="%s" AND format="mp4,flv" AND protocol="rtmp,http" AND plrs="%s" AND offnetwork="false" AND site="autv_plus7" AND lang="en-AU" AND region="AU" AND override="none";',
- 'callback': 'jsonp_callback',
- 'env': 'prod',
- 'format': 'json'
+ 'q': 'SELECT streams,status FROM yahoo.media.video.streams WHERE id="%s" AND format="mp4,flv" AND protocol="rtmp,http" AND plrs="%s" AND offnetwork="false" AND site="autv_plus7" AND lang="en-AU" AND region="AU" AND override="none";',
+ 'callback': 'jsonp_callback',
+ 'env': 'prod',
+ 'format': 'json'
}
HASH_URL = "http://d.yimg.com/m/up/ypp/au/player.swf"
def extract_and_remove(tokens, key):
- lowertokens = [x.lower() for x in tokens]
- pos = lowertokens.index(key)
+ lowertokens = [x.lower() for x in tokens]
+ pos = lowertokens.index(key)
- value = int(tokens[pos+1])
- tokens = tokens[:pos] + tokens[pos+2:]
+ value = int(tokens[pos+1])
+ tokens = tokens[:pos] + tokens[pos+2:]
- return value, tokens
+ return value, tokens
def demangle_title(title, subtitle):
- tokens = title.split()
- insert_pos = len(tokens)
- if subtitle:
- insert_pos += 1
- tokens += ["-"] + subtitle.split()
-
- try:
- season, tokens = extract_and_remove(tokens, "series")
- episode, tokens = extract_and_remove(tokens, "episode")
- if insert_pos < len(tokens):
- tokens.insert(insert_pos, "-")
- tokens.insert(insert_pos, "%sx%s" % (season, str(episode).zfill(2)))
- except ValueError:
- pass
-
- return " ".join(tokens)
+ tokens = title.split()
+ insert_pos = len(tokens)
+ if subtitle:
+ insert_pos += 1
+ tokens += ["-"] + subtitle.split()
+
+ try:
+ season, tokens = extract_and_remove(tokens, "series")
+ episode, tokens = extract_and_remove(tokens, "episode")
+ if insert_pos < len(tokens):
+ tokens.insert(insert_pos, "-")
+ tokens.insert(insert_pos, "%sx%s" % (season, str(episode).zfill(2)))
+ except ValueError:
+ pass
+
+ return " ".join(tokens)
class Plus7Node(Node):
- def __init__(self, title, parent, url):
- Node.__init__(self, title, parent)
- self.url = url
- self.can_download = True
-
- def get_video_id(self):
- doc = grab_html(self.url, 3600)
- for script in doc.xpath("//script"):
- if not script.text:
- continue
- for line in script.text.split(";"):
- line = line.strip()
- if line.find("new Y.VideoPlatform.VideoPlayer") <= 0:
- continue
-
-### vidparams = line[line.find("(")+1 : line.rfind(")")]
-### vidparams = json.loads(vidparams)
-### return vidparams["playlist"]["mediaItems"][0]["id"]
-
- # Cannot parse it as JSON :(
- pos1 = line.find('"mediaItems":')
- if pos1 < 0:
- continue
- pos2 = line.find('"id":', pos1)
- if pos2 < 0:
- continue
- pos3 = line.find('"', pos2+5)
- pos4 = line.find('"', pos2+6)
- if pos3 < 0 or pos4 < 0:
- continue
- return line[pos3+1:pos4]
-
- raise Exception("Could not find video id on page " + self.url)
-
- def generate_session(self):
- return "".join([random.choice(string.ascii_letters) for x in xrange(22)])
-
- def download(self):
- vid_id = self.get_video_id()
- qs = dict(METADATA_QUERY.items()) # copy..
- qs["q"] = qs["q"] % (vid_id, self.generate_session())
- url = METADATA_BASE + urllib.urlencode(qs)
- doc = grab_json(url, 0, skip_function=True)
- stream_data = doc["query"]["results"]["mediaObj"][0]["streams"][0]
- vbase = stream_data["host"]
- vpath = stream_data["path"]
- filename = self.title + ".flv"
- return download_rtmp(filename, vbase, vpath, HASH_URL)
+ def __init__(self, title, parent, url):
+ Node.__init__(self, title, parent)
+ self.url = url
+ self.can_download = True
+
+ def get_video_id(self):
+ doc = grab_html(self.url, 3600)
+ for script in doc.xpath("//script"):
+ if not script.text:
+ continue
+ for line in script.text.split(";"):
+ line = line.strip()
+ if line.find("new Y.VideoPlatform.VideoPlayer") <= 0:
+ continue
+
+### vidparams = line[line.find("(")+1 : line.rfind(")")]
+### vidparams = json.loads(vidparams)
+### return vidparams["playlist"]["mediaItems"][0]["id"]
+
+ # Cannot parse it as JSON :(
+ pos1 = line.find('"mediaItems":')
+ if pos1 < 0:
+ continue
+ pos2 = line.find('"id":', pos1)
+ if pos2 < 0:
+ continue
+ pos3 = line.find('"', pos2+5)
+ pos4 = line.find('"', pos2+6)
+ if pos3 < 0 or pos4 < 0:
+ continue
+ return line[pos3+1:pos4]
+
+ raise Exception("Could not find video id on page " + self.url)
+
+ def generate_session(self):
+ return "".join([random.choice(string.ascii_letters) for x in xrange(22)])
+
+ def download(self):
+ vid_id = self.get_video_id()
+ qs = dict(METADATA_QUERY.items()) # copy..
+ qs["q"] = qs["q"] % (vid_id, self.generate_session())
+ url = METADATA_BASE + urllib.urlencode(qs)
+ doc = grab_json(url, 0, skip_function=True)
+ stream_data = doc["query"]["results"]["mediaObj"][0]["streams"][0]
+ vbase = stream_data["host"]
+ vpath = stream_data["path"]
+ filename = self.title + ".flv"
+ return download_rtmp(filename, vbase, vpath, HASH_URL)
class Plus7Series(Node):
- def __init__(self, title, parent, url):
- Node.__init__(self, title, parent)
- self.url = url
-
- def fill_children(self):
- doc = grab_html(self.url, 3600)
- for item in CSSSelector("#related-episodes div.itemdetails")(doc):
- title = CSSSelector("span.title")(item)[0].text
- subtitle = CSSSelector("span.subtitle")(item)[0].xpath("string()")
- title = demangle_title(title, subtitle)
- url = CSSSelector("a")(item)[0].attrib["href"]
- Plus7Node(title, self, BASE + url)
+ def __init__(self, title, parent, url):
+ Node.__init__(self, title, parent)
+ self.url = url
+
+ def fill_children(self):
+ doc = grab_html(self.url, 3600)
+ for item in CSSSelector("#related-episodes div.itemdetails")(doc):
+ title = CSSSelector("span.title")(item)[0].text
+ subtitle = CSSSelector("span.subtitle")(item)[0].xpath("string()")
+ title = demangle_title(title, subtitle)
+ url = CSSSelector("a")(item)[0].attrib["href"]
+ Plus7Node(title, self, BASE + url)
class Plus7Root(Node):
- def __init__(self, parent):
- Node.__init__(self, "Yahoo Plus7 (broken!)", parent)
-
- def fill_children(self):
- doc = grab_html(BROWSE, 3600)
- shows = []
- for script in doc.xpath("//script"):
- if not script.text or not script.text.startswith("var shows = "):
- continue
- shows = script.text[12:]
- shows = shows.rstrip("; \n")
- shows = json.loads(shows)
- for show in shows:
- Plus7Series(show["title"], self, show["url"])
+ def __init__(self, parent):
+ Node.__init__(self, "Yahoo Plus7 (broken!)", parent)
+
+ def fill_children(self):
+ doc = grab_html(BROWSE, 3600)
+ shows = []
+ for script in doc.xpath("//script"):
+ if not script.text or not script.text.startswith("var shows = "):
+ continue
+ shows = script.text[12:]
+ shows = shows.rstrip("; \n")
+ shows = json.loads(shows)
+ for show in shows:
+ Plus7Series(show["title"], self, show["url"])
def fill_nodes(root_node):
- Plus7Root(root_node)
+ Plus7Root(root_node)
#!/usr/bin/env python
-# vim:ts=4:sts=4:sw=4:noet
from common import grab_text, grab_html, grab_json, grab_xml, download_rtmp, download_urllib, Node, append_to_qs
VIDEO_MENU = BASE + "/ondemand/js/video-menu"
VIDEO_URL = BASE + "/ondemand/video/single/%s"
VIDEO_MAGIC = {
- "v": "2.5.14",
- "fp": "MAC 11,1,102,55",
- "r": "FLQDD",
- "g": "YNANAXRIYFYO",
+ "v": "2.5.14",
+ "fp": "MAC 11,1,102,55",
+ "r": "FLQDD",
+ "g": "YNANAXRIYFYO",
}
SWF_URL = "http://resources.sbs.com.au/vod/theplatform/core/current/swf/flvPlayer.swf"
NS = {
- "smil": "http://www.w3.org/2005/SMIL21/Language",
+ "smil": "http://www.w3.org/2005/SMIL21/Language",
}
class SbsNode(Node):
- def __init__(self, title, parent, video_id):
- Node.__init__(self, title, parent)
- self.title = title
- self.video_id = video_id.split("/")[-1]
- self.can_download = True
-
- def download(self):
- doc = grab_html(VIDEO_URL % self.video_id, 0)
- meta_video = doc.xpath("//meta[@property='og:video']")[0]
- swf_url = meta_video.attrib["content"]
- swf_url_qs = urlparse.parse_qs(urlparse.urlparse(swf_url).query)
- desc_url = swf_url_qs["v"][0]
-
- doc = grab_text(desc_url, 0)
- doc_qs = urlparse.parse_qs(doc)
- desc_url = doc_qs["releaseUrl"][0]
-
- doc = grab_xml(desc_url, 0)
- error = doc.xpath("//smil:param[@name='exception']/@value", namespaces=NS)
- if error:
- raise Exception("Error downloading, SBS said: " + error[0])
-
- video = doc.xpath("//smil:video", namespaces=NS)[0]
- video_url = video.attrib["src"]
- if not video_url:
- raise Exception("Unsupported video '%s': %s" % (self.title, desc_url))
- ext = urlparse.urlsplit(video_url).path.rsplit(".", 1)[1]
- filename = self.title + "." + ext
- video_url = append_to_qs(video_url, VIDEO_MAGIC)
- return download_urllib(filename, video_url, referrer=SWF_URL)
+ def __init__(self, title, parent, video_id):
+ Node.__init__(self, title, parent)
+ self.title = title
+ self.video_id = video_id.split("/")[-1]
+ self.can_download = True
+
+ def download(self):
+ doc = grab_html(VIDEO_URL % self.video_id, 0)
+ meta_video = doc.xpath("//meta[@property='og:video']")[0]
+ swf_url = meta_video.attrib["content"]
+ swf_url_qs = urlparse.parse_qs(urlparse.urlparse(swf_url).query)
+ desc_url = swf_url_qs["v"][0]
+
+ doc = grab_text(desc_url, 0)
+ doc_qs = urlparse.parse_qs(doc)
+ desc_url = doc_qs["releaseUrl"][0]
+
+ doc = grab_xml(desc_url, 0)
+ error = doc.xpath("//smil:param[@name='exception']/@value", namespaces=NS)
+ if error:
+ raise Exception("Error downloading, SBS said: " + error[0])
+
+ video = doc.xpath("//smil:video", namespaces=NS)[0]
+ video_url = video.attrib["src"]
+ if not video_url:
+ raise Exception("Unsupported video '%s': %s" % (self.title, desc_url))
+ ext = urlparse.urlsplit(video_url).path.rsplit(".", 1)[1]
+ filename = self.title + "." + ext
+ video_url = append_to_qs(video_url, VIDEO_MAGIC)
+ return download_urllib(filename, video_url, referrer=SWF_URL)
class SbsNavNode(Node):
- def __init__(self, title, parent, url):
- Node.__init__(self, title, parent)
- self.url = url
-
- def fill_children(self):
- try:
- doc = grab_json(BASE + self.url, 3600)
- except ValueError:
- # SBS sends XML as an error message :\
- return
- if len(doc.get("entries", [])) == 0:
- return
- for entry in doc["entries"]:
- self.fill_entry(entry)
-
- def fill_entry(self, entry):
- title = entry["title"]
- video_id = entry["id"]
- SbsNode(title, self, video_id)
+ def __init__(self, title, parent, url):
+ Node.__init__(self, title, parent)
+ self.url = url
+
+ def fill_children(self):
+ try:
+ doc = grab_json(BASE + self.url, 3600)
+ except ValueError:
+ # SBS sends XML as an error message :\
+ return
+ if len(doc.get("entries", [])) == 0:
+ return
+ for entry in doc["entries"]:
+ self.fill_entry(entry)
+
+ def fill_entry(self, entry):
+ title = entry["title"]
+ video_id = entry["id"]
+ SbsNode(title, self, video_id)
class SbsRootNode(Node):
- def __init__(self, parent):
- Node.__init__(self, "SBS", parent)
-
- def fill_children(self):
- menu = grab_json(VIDEO_MENU, 3600, skip_assignment=True)
- for name in menu.keys():
- self.fill_category(self, menu[name])
-
- def create_nav_node(self, name, parent, cat_data, url_key):
- try:
- url = cat_data[url_key]
- except KeyError:
- return
- if url.strip():
- SbsNavNode(name, parent, url)
-
- def fill_category(self, parent, cat_data):
- if not cat_data.has_key("children"):
- name = cat_data["name"]
- self.create_nav_node(name, parent, cat_data, "url")
- return
-
- node = Node(cat_data["name"], parent)
- self.create_nav_node("-Featured", node, cat_data, "furl")
- self.create_nav_node("-Latest", node, cat_data, "url")
- self.create_nav_node("-Most Popular", node, cat_data, "purl")
-
- children = cat_data.get("children", [])
- if isinstance(children, dict):
- children = [children[k] for k in sorted(children.keys())]
- for child_cat in children:
- self.fill_category(node, child_cat)
+ def __init__(self, parent):
+ Node.__init__(self, "SBS", parent)
+
+ def fill_children(self):
+ menu = grab_json(VIDEO_MENU, 3600, skip_assignment=True)
+ for name in menu.keys():
+ self.fill_category(self, menu[name])
+
+ def create_nav_node(self, name, parent, cat_data, url_key):
+ try:
+ url = cat_data[url_key]
+ except KeyError:
+ return
+ if url.strip():
+ SbsNavNode(name, parent, url)
+
+ def fill_category(self, parent, cat_data):
+ if not cat_data.has_key("children"):
+ name = cat_data["name"]
+ self.create_nav_node(name, parent, cat_data, "url")
+ return
+
+ node = Node(cat_data["name"], parent)
+ self.create_nav_node("-Featured", node, cat_data, "furl")
+ self.create_nav_node("-Latest", node, cat_data, "url")
+ self.create_nav_node("-Most Popular", node, cat_data, "purl")
+
+ children = cat_data.get("children", [])
+ if isinstance(children, dict):
+ children = [children[k] for k in sorted(children.keys())]
+ for child_cat in children:
+ self.fill_category(node, child_cat)
def fill_nodes(root_node):
- SbsRootNode(root_node)
+ SbsRootNode(root_node)