From: James Bunton Date: Wed, 19 Sep 2012 02:52:42 +0000 (+1000) Subject: Reorganised everything X-Git-Url: https://code.delx.au/monosys/commitdiff_plain/f31f8b598dd867e1f0558a74b4376a9243ea818b Reorganised everything --- diff --git a/filters/bin2ascii.py b/bin/bin2ascii similarity index 100% rename from filters/bin2ascii.py rename to bin/bin2ascii diff --git a/filters/csv2txt.py b/bin/csv2txt similarity index 100% rename from filters/csv2txt.py rename to bin/csv2txt diff --git a/filters/dos2unix b/bin/dos2unix similarity index 100% rename from filters/dos2unix rename to bin/dos2unix diff --git a/scripts/passwdgen b/bin/passwdgen similarity index 100% rename from scripts/passwdgen rename to bin/passwdgen diff --git a/scripts/photocopy b/bin/photocopy similarity index 100% rename from scripts/photocopy rename to bin/photocopy diff --git a/ripping/pulse_rip b/bin/pulse_rip similarity index 100% rename from ripping/pulse_rip rename to bin/pulse_rip diff --git a/mail/sendmail.py b/bin/sendmail.py similarity index 100% rename from mail/sendmail.py rename to bin/sendmail.py diff --git a/scripts/wepkeygen b/bin/wepkeygen similarity index 100% rename from scripts/wepkeygen rename to bin/wepkeygen diff --git a/filters/xmlpp b/bin/xmlpp similarity index 100% rename from filters/xmlpp rename to bin/xmlpp diff --git a/filters/filter_printable.py b/filters/filter_printable.py deleted file mode 100755 index f15d46c..0000000 --- a/filters/filter_printable.py +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env python - -import string, sys - -while True: - c = sys.stdin.read(1) - if c in string.printable: - sys.stdout.write(c) - diff --git a/quines/quine.py b/misc/quine.py similarity index 100% rename from quines/quine.py rename to misc/quine.py diff --git a/quines/smallquine.py b/misc/smallquine.py similarity index 100% rename from quines/smallquine.py rename to misc/smallquine.py diff --git a/mythtv/mythmusic_importm3u.py b/mythtv/mythmusic_importm3u.py deleted file mode 100755 index 160f5f4..0000000 --- a/mythtv/mythmusic_importm3u.py +++ /dev/null @@ -1,191 +0,0 @@ -#!/usr/bin/env python - -# This script reads in m3u playlists and inserts them into the mythtv database - -# Copyright 2008 James Bunton -# Licensed for distribution under the GPL version 2 - -# Copyright 2005 Patrick Riley (http://pathacks.blogspot.com/). You are -# free to use and modify this code as you see fit, but it comes with -# no warranty whatsoever. - - -import logging, optparse, os.path, re, sys - -import MySQLdb - -class PlaylistBuilder(object): - findPlaylistQuery = """ - SELECT playlist_id - FROM music_playlists - WHERE playlist_name = %s - """ - findItemQuery = """ - SELECT song.song_id - FROM music_directories dir JOIN music_songs song ON dir.directory_id = song.directory_id - WHERE concat(dir.path, '/', song.filename) = %s - """ - insertPlaylistQuery = """ - INSERT INTO - music_playlists(playlist_name, playlist_songs) - VALUES(%s, %s) - """ - updatePlaylistQuery = """ - UPDATE music_playlists - SET playlist_songs = %s - WHERE playlist_id = %s - """ - - def __init__(self, dbHost, dbUser, dbPassword, dbName="mythconverg"): - self.dbconnection = MySQLdb.connect(db=dbName, host=dbHost, user=dbUser, passwd=dbPassword) - self.cursor = self.dbconnection.cursor() - - def startPlaylist(self, name): - self.playlistName = name - self.playlistId = -1 - self.cursor.execute(self.findPlaylistQuery, [self.playlistName]) - if self.cursor.rowcount < 0: - raise ValueError("rowcount < 0?") - elif self.cursor.rowcount > 0: - assert self.cursor.rowcount == 1, "More than one playlist with the same name?" - self.playlistId = self.cursor.fetchone()[0] - self.ids = [] - self.addFile = self.addFileFirst - - def addFileNormal(self, filename): - filename = filename.split(os.path.sep, self.strip)[-1] - self.cursor.execute(self.findItemQuery, [filename]) - if self.cursor.rowcount != 1: - logging.warning("Did not find an entry for '%s' (return=%d)" % (filename, self.cursor.rowcount)) - return - self.ids.append("%d" % self.cursor.fetchone()[0]) - - def addFileFirst(self, filename): - self.addFile = self.addFileNormal - - # Pull off path components until we find it - self.strip = 0 - while filename: - self.cursor.execute(self.findItemQuery, [filename]) - if self.cursor.rowcount == 1: - break - filename = filename.split(os.path.sep, 1)[1] - self.strip += 1 - else: - logging.warning("Did not find entry for first file in playlist: '%s' No autodetection of strip amount." % filename) - self.strip = 0 - return - - logging.info("Found file: '%s', auto detected strip amount: %d" % (filename, self.strip)) - self.ids.append("%d" % self.cursor.fetchone()[0]) - - def finishPlaylist(self): - idsString = ",".join(self.ids) - logging.info("Playlist '%s' is: %s" % (self.playlistName, idsString)) - if self.playlistId < 0: - self.cursor.execute(self.insertPlaylistQuery, [self.playlistName, idsString]) - else: - self.cursor.execute(self.updatePlaylistQuery, [idsString, self.playlistId]) - # We can't check the rowcount here because update will say 0 if no values actually changed - # assert cursor.rowcount == 1, "Insert/update failed? %d" % (playlistId) - - -def stripComments(it): - re_comment = re.compile(r"^\s*#") - re_all_whitespace = re.compile(r"^\s*$") - - for line in it: - if re_comment.match(line): - continue - if re_all_whitespace.match(line): - continue - line = line.strip() - yield line - - -def readMysqlTxt(filename, vars): - for line in stripComments(open(filename)): - try: - key, value = line.split("=") - except: - logging.warning("Couldn't parse mysql.txt line -- %s" % line) - vars[key.lower().strip()] = value.strip() - -def readArgs(vars): - parser = optparse.OptionParser(usage="""%prog [options] Playlist.m3u [Another.m3u] ... - -Converts m3u style playlists into playlists for MythTV - -A m3u file is a list of filenames where # begins comments. This script -makes one playlist for each m3u file you give it (stripping the extention -from the filename to get the playlist name) - -This script works by connecting to your MythTV MySQL database and querying -for the filenames found in the .m3u file. - -The database connection settings will be read from the .mythtv/mysql.txt -file if possible, otherwise specify them on the command line. - """) - parser.add_option("--dbhost", help="MythTV database host to connect to", - action="store", dest="host", default=vars["dbhostname"]) - parser.add_option("--dbuser", help="MythTV database username", - action="store", dest="user", default=vars["dbusername"]) - parser.add_option("--dbpassword", help="MythTV database password", - action="store", dest="password", default=vars["dbpassword"]) - parser.add_option("--dbname", help="MythTV database name", - action="store", dest="database", default=vars["dbname"]) - parser.add_option("-v", "--verbose", help="Be verbose", - action="store_true", dest="verbose") - - options, filenames = parser.parse_args() - vars["dbhostname"] = options.host - vars["dbusername"] = options.user - vars["dbpassword"] = options.password - vars["dbname"] = options.database - if options.verbose: - logging.basicConfig(level=logging.INFO) - else: - logging.basicConfig(level=logging.WARNING) - - return filenames - - -def main(): - vars = { - "dbhostname": "localhost", - "dbusername": "mythtv", - "dbpassword": "mythtv", - "dbname": "mythconverg", - } - - found = False - for path in ["~/.mythtv/mysql.txt", "~mythtv/.mythtv/mysql.txt", "/etc/mythtv/mysql.txt"]: - try: - readMysqlTxt(os.path.expanduser(path), vars) - found = True - break - except IOError: - continue - filenames = readArgs(vars) - if not found: - logging.warning("Could not read mysql.txt, try running as the mythtv user.") - - builder = PlaylistBuilder(vars["dbhostname"], vars["dbusername"], vars["dbpassword"], vars["dbname"]) - for filename in filenames: - playlistName = os.path.basename(filename) - playlistName = playlistName.rsplit(".", 1)[0] - logging.info("Processing '%s' as playlist '%s'" % (filename, playlistName)) - - baseDir = os.path.dirname(filename) - builder.startPlaylist(playlistName) - for line in stripComments(open(filename)): - line = re.sub("^(\.\./)*", "", line) - if not line.startswith("/"): - line = os.path.abspath(os.path.join(baseDir, line)) - builder.addFile(line) - builder.finishPlaylist() - - -if __name__ == '__main__': - main() - diff --git a/mail/abook2mutt/Makefile b/osx_bin/abook2mutt/Makefile similarity index 100% rename from mail/abook2mutt/Makefile rename to osx_bin/abook2mutt/Makefile diff --git a/mail/abook2mutt/ab2mutt.m b/osx_bin/abook2mutt/ab2mutt.m similarity index 100% rename from mail/abook2mutt/ab2mutt.m rename to osx_bin/abook2mutt/ab2mutt.m diff --git a/scripts/osx_battery b/osx_bin/battery similarity index 100% rename from scripts/osx_battery rename to osx_bin/battery diff --git a/mail/maildirstat.py b/osx_bin/maildirstat.py similarity index 100% rename from mail/maildirstat.py rename to osx_bin/maildirstat.py diff --git a/scripts/pgrep b/osx_bin/pgrep similarity index 100% rename from scripts/pgrep rename to osx_bin/pgrep diff --git a/scripts/osx_preview b/osx_bin/preview similarity index 100% rename from scripts/osx_preview rename to osx_bin/preview diff --git a/scripts/osx_ps b/osx_bin/ps similarity index 100% rename from scripts/osx_ps rename to osx_bin/ps diff --git a/scripts/ssh-agent-setup b/osx_bin/ssh-agent-setup similarity index 100% rename from scripts/ssh-agent-setup rename to osx_bin/ssh-agent-setup diff --git a/scripts/osx_top b/osx_bin/top similarity index 100% rename from scripts/osx_top rename to osx_bin/top diff --git a/ripping/grab-abc-stream b/ripping/grab-abc-stream deleted file mode 100755 index d338020..0000000 --- a/ripping/grab-abc-stream +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python - -from lxml import etree -import os -import subprocess -import sys -import tempfile -import urllib -import urlparse - - -def exec_subprocess(cmd): - try: - p = subprocess.Popen(cmd) - ret = p.wait() - if ret != 0: - print >>sys.stderr, cmd[0], "exited with error code:", ret - return False - else: - return True - except OSError, e: - print >>sys.stderr, "Failed to run", cmd[0], e - except KeyboardInterrupt: - print "Cancelled", cmd - try: - p.terminate() - p.wait() - except KeyboardInterrupt: - p.send_signal(signal.SIGKILL) - p.wait() - return False - -def mplayer_convert(stream, author, title): - print "Downloading", stream - wmfile = tempfile.NamedTemporaryFile() - cmd = [ - "mplayer", - "-nocache", - "-noconsolecontrols", - "-ao", - "pcm:file=" + wmfile.name, - stream, - ] - if not exec_subprocess(cmd): - return False - - print "Converting", wmfile.name, "to mp3" - cmd = [ - "lame", - "--add-id3v2", - "--ta", author, - "--tt", title, - wmfile.name, - os.path.splitext(os.path.basename(stream))[0] + ".mp3", - ] - if not exec_subprocess(cmd): - return False - - return True - - -def grab(u): - qs = urlparse.parse_qs(urlparse.urlparse(u).query) - wmfile = qs["w"][0] - doc = etree.parse(urllib.urlopen(wmfile)) - streams = doc.xpath("//ref/@href") - - author = qs["pgm"][0] - title = qs["t"][0] - - for stream in streams: - if not stream.startswith("mms://"): - continue - if mplayer_convert(stream, author, title): - return - -print "Paste 'Listen Now' URLs from ABC... Press CTRL-D to finish" -try: - for line in sys.stdin: - if not line.strip(): - continue - grab(line) -except KeyboardInterrupt: - print "\nExiting..." - diff --git a/ripping/virginmobile.rb b/ripping/virginmobile.rb deleted file mode 100755 index 411bc46..0000000 --- a/ripping/virginmobile.rb +++ /dev/null @@ -1,192 +0,0 @@ -#!/usr/bin/env ruby - -# # Sample config file -# ROOTCA = "/etc/ssl/certs/ca-certificates.crt" -# SMTPSERVER = "mail.example.com" -# FROMADDR = "cron@example.com" -# SLEEPTIME = 15 -# ACCOUNTS = [ -# ["person1@example.com", "0499999999", "000000", 100, 30], -# ["person2@example.com", "0499999998", "000000", 100, 30], -# ] - -require 'optparse' -require 'net/http' -require 'net/https' -require 'net/smtp' -require 'uri' - -class VirginMobile - def initialize(sleep_time, username, password) - @sleep_time = sleep_time - @username = username - @password = password - @cookie = nil - end - - def do_request(path, form_data=nil) - sleep(@sleep_time) # Don't look like a bot - http = Net::HTTP.new("www.virginmobile.com.au", 443) - http.use_ssl = true - if File.exist? ROOTCA - http.ca_file = ROOTCA - http.verify_mode = OpenSSL::SSL::VERIFY_PEER - http.verify_depth = 5 - end - if @cookie - req = Net::HTTP::Get.new(path) - req["Cookie"] = @cookie - end - if form_data - req = Net::HTTP::Post.new(path) - req.form_data = form_data - end - return http.request(req) - end - - def do_login - form_data = { - "username" => @username, - "password" => @password, - } - res = do_request("/selfcare/MyAccount/LogoutLoginPre.jsp", form_data) - @cookie = res.fetch("Set-Cookie") - - while location = res.get_fields("Location") - location = URI.parse(location[0]) - res = do_request(location.path) - end - end - - def do_logout - do_request("/selfcare/dispatch/Logout") - end - - def request_bill_preview - res = do_request("/selfcare/dispatch/PostPayUnbilledUsage") - usage = res.body.scan(/\$([\d\.]+)/).flatten - last_bill_date = res.body.gsub(/\n/, '').scan(/Last bill date:.*>(\d\d?\/\d\d?\/\d\d\d\d)/).flatten - return usage, last_bill_date - end - - def request_mobile_browsing - res = do_request("/selfcare/dispatch/DataUsageRequest") - data_usage = res.body.gsub(/\n/, '').scan(/Data used this billing month:.*>(\d+) MB \(/).flatten - return data_usage - end - - def dump_result(res) - res.each_capitalized do |key, value| - print "#{key}: #{value}\n" - end - print res.body + "\n" - end -end - -def check_usage(sleep_time, ignore_limits, destination, username, password, usageLimit, dataLimit) - message = "" - data = VirginMobile.new(sleep_time, username, password) - data.do_login - - usage, last_bill_date = data.request_bill_preview - usage.each do |x| - if ignore_limits or (usageLimit >= 0 and x.to_f >= usageLimit) - message += "Unbilled usage: $#{x}\n" - end - end - - data_usage = data.request_mobile_browsing - data_usage.each do |x| - if ignore_limits or (dataLimit >= 0 and x.to_f >= dataLimit) - message += "Data usage: #{x} MiB\n" - end - end - - data.do_logout - - if message.length > 0 - return destination, < - - - delx.net.au - YouTube Scraper - - - - -

delx.net.au - YouTube Scraper

- {0} -
-

This page will let you easily download YouTube videos to watch offline. It - will automatically grab the highest quality version.

-
-
-
-

Tip! Use this bookmarklet: YouTube Download - to easily download videos. Right-click the link and add it to bookmarks, - then when you're looking at a YouTube page select that bookmark from your - browser's bookmarks menu to download the video straight away.

- - -""".replace("{0}", msg).replace("{1}", url).replace("{2}", script_url) - -cookiejar = cookielib.CookieJar() -urlopener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar)) -referrer = "" - -def urlopen(url): - global referrer - req = urllib2.Request(url) - if referrer: - req.add_header("Referer", referrer) - referrer = url - req.add_header("User-Agent", USER_AGENT) - return urlopener.open(req) - -def parse_url(url): - f = urlopen(url) - doc = document_fromstring(f.read()) - f.close() - return doc - -def append_to_qs(url, params): - r = list(urlparse.urlsplit(url)) - qs = urlparse.parse_qs(r[3]) - qs.update(params) - r[3] = urllib.urlencode(qs, True) - url = urlparse.urlunsplit(r) - return url - -def convert_from_old_itag(player_config): - url_data = urlparse.parse_qs(player_config["args"]["url_encoded_fmt_stream_map"]) - url_data["url"] = [] - for itag_url in url_data["itag"]: - pos = itag_url.find("url=") - url_data["url"].append(itag_url[pos+4:]) - player_config["args"]["url_encoded_fmt_stream_map"] = urllib.urlencode(url_data, True) - -def get_player_config(doc): - player_config = None - for script in doc.xpath("//script"): - if not script.text: - continue - for line in script.text.split("\n"): - if "yt.playerConfig =" in line: - p1 = line.find("=") - p2 = line.rfind(";") - if p1 >= 0 and p2 > 0: - return json.loads(line[p1+1:p2]) - if "'PLAYER_CONFIG': " in line: - p1 = line.find(":") - if p1 >= 0: - player_config = json.loads(line[p1+1:]) - convert_from_old_itag(player_config) - return player_config - -def get_best_video(player_config): - url_data = urlparse.parse_qs(player_config["args"]["url_encoded_fmt_stream_map"]) - url_data = itertools.izip_longest( - url_data["url"], - url_data["type"], - url_data["quality"], - url_data.get("sig", []), - ) - best_url = None - best_quality = None - best_extension = None - for video_url, mimetype, quality, signature in url_data: - mimetype = mimetype.split(";")[0] - if mimetype not in MIMETYPES: - continue - extension = "." + MIMETYPES[mimetype] - quality = QUALITIES.get(quality.split(",")[0], -1) - if best_quality is None or quality > best_quality: - if signature: - video_url = append_to_qs(video_url, {"signature": signature}) - best_url = video_url - best_quality = quality - best_extension = extension - - return best_url, best_extension - -def get_video_url(doc): - unavailable = doc.xpath("//div[@id='unavailable-message']/text()") - if unavailable: - raise VideoUnavailable(unavailable[0].strip()) - - player_config = get_player_config(doc) - if not player_config: - raise VideoUnavailable("Could not find video URL") - - video_url, extension = get_best_video(player_config) - if not video_url: - return None, None - - title = doc.xpath("/html/head/title/text()")[0] - title = re.sub("\s+", " ", title.strip()) - valid_chars = frozenset("-_.() abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") - filename = "".join(c for c in title.encode("ascii", "ignore") if c in valid_chars) - filename += extension - - return video_url, filename - -def cgimain(): - args = cgi.parse() - try: - url = args["url"][0] - except: - print_form(url="http://www.youtube.com/watch?v=FOOBAR") - return - - try: - doc = parse_url(url) - video_url, filename = get_video_url(doc) - data = urlopen(video_url) - httpinfo = data.info() - sys.stdout.write("Content-Disposition: attachment; filename=\"%s\"\r\n" % filename) - sys.stdout.write("Content-Length: %s\r\n" % httpinfo.getheader("Content-Length")) - sys.stdout.write("\r\n") - shutil.copyfileobj(data, sys.stdout) - data.close() - except VideoUnavailable, e: - print_form( - url=url, - msg="

Sorry, there was an error: %s

" % cgi.escape(e.message) - ) - except Exception, e: - print_form( - url=url, - msg="

Sorry, there was an error. Check your URL?

" - ) - return - -def main(): - try: - url = sys.argv[1] - except: - print >>sys.stderr, "Usage: %s http://youtube.com/watch?v=FOOBAR" % sys.argv[0] - sys.exit(1) - doc = parse_url(url) - video_url, filename = get_video_url(doc) - data = urlopen(video_url) - outfile = open(filename, "w") - shutil.copyfileobj(data, outfile) - data.close() - outfile.close() - - -if __name__ == "__main__": - resource.setrlimit(resource.RLIMIT_AS, (MAX_MEMORY_BYTES, MAX_MEMORY_BYTES)) - if os.environ.has_key("SCRIPT_NAME"): - cgimain() - else: - main() - diff --git a/scripts/linux_colorwrap b/scripts/linux_colorwrap deleted file mode 100755 index 3273f47..0000000 --- a/scripts/linux_colorwrap +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python - -PAGERS = ["less", "more", "most", "tail", "head"] - -# Usage: -# * Put this file into ~/bin/colorwrap -# * Add these aliases to your bash config -# alias ls="colorwrap ls" -# alias grep="colorwrap grep" - - -import os, os.path, sys, time - -class Process(object): - def __init__(self, pid): - self.pid = pid - - @property - def stdin(self): - return os.readlink("/proc/%s/fd/0" % self.pid) - - @property - def stdout(self): - return os.readlink("/proc/%s/fd/1" % self.pid) - - @property - def cmdline(self): - return file("/proc/%s/cmdline" % self.pid).read().split("\x00") - -def validProcesses(): - for pid in filter(str.isdigit, os.listdir("/proc/")): - if os.path.exists("/proc/%s/fd/0" % pid): - yield Process(pid) - -def isatty(filename): - try: - stdout = file(cur.stdout) - return os.isatty(stdout.fileno()) - except IOError: - return False - -# Follow from current pid until the end of the chain -count = 0 -cur = Process(os.getpid()) -while count < 32 and cur.stdout.startswith("pipe:"): - last = cur - for cur in validProcesses(): - try: - if last.stdout == cur.stdin: - count = 0 - break - except OSError: - continue - else: - cur = last - count += 1 - time.sleep(0.001) - -# Use colour or not -command = sys.argv[1] -args = sys.argv[1:] -if cur.cmdline in PAGERS or isatty(cur.stdout): - args.insert(1, "--color=always") - -os.execvp(command, args) - diff --git a/scripts/mount.fuse b/scripts/mount.fuse deleted file mode 100755 index 75d21bf..0000000 --- a/scripts/mount.fuse +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# -# FUSE mount helper -# Petr Klima -# James Bunton -# Thanks to Miklos Szeredi -# to kick me to the right way -# - -VERSION="0.0.2" -PRGNAME=`basename $0` -HOME=/root - -USAGE="${PRGNAME} version ${VERSION} -usage: ${PRGNAME} fusefs_type#[mountpath] mountpoint [FUSE options] - - example: ${PRGNAME} sshfs#root@tux:/ /mnt/tuxssh -o rw -" - -function die { - echo -e "$PRGNAME# $1" >&2 - [ -z "$2" ] && exit 128 - exit "$2" -} - -[ "$#" -ge 2 ] || die "${USAGE}" - -FSTYPE=${1%%\#*} # for now i have to be same as FUSE mount binary - # should be configurable - -export PATH -FSBIN=`which ${FSTYPE} 2>/dev/null` \ - || die "Can not find FUSE mount binary for FS ${FSTYPE}" 1 - -MOUNTPATH=${1#*#} - -# was there an # in $1 -[ "$1" = "$MOUNTPATH" ] && MOUNTPATH="" - -MOUNTPOINT="$2" -[ -d "${MOUNTPOINT}" ] || die "Directory ${MOUNTPOINT} does not exist" - -shift -shift -shift - -ignore_opts="(user|nouser|users|auto|noauto|_netdev)" - -OPTIONS=`echo $@ | sed -r "s/(,${ignore_opts}|${ignore_opts},)//g"` -NEWOPTIONS="" -OLDIFS="${IFS}" -IFS=',' -for opt in ${OPTIONS}; do - NEWOPTIONS="${NEWOPTIONS} -o ${opt}" -done -IFS="${OLDIFS}" - -${FSTYPE} ${MOUNTPATH} ${MOUNTPOINT} ${NEWOPTIONS} diff --git a/mythtv/mythsymlink b/scripts/mythsymlink similarity index 100% rename from mythtv/mythsymlink rename to scripts/mythsymlink