-
-
Save archagon/4071535 to your computer and use it in GitHub Desktop.
# GDC Vault videos can't be watched on mobile devices and this is a very sad thing indeed! | |
# (Note: this has changed for GDC2013, which lets you watch raw MP4 streams. Kudos!) | |
# This script is designed to circumvent this by downloading the lecture and slideshow | |
# videos which can then be re-encoded into whatever format you wish. Obviously, you | |
# won't be able to do this without access to the Vault. This is strictly for the | |
# convenience of legitimate Vault users! | |
# Note: this code is rather flimsy and was written as fast as possible for my own personal use. | |
# The code only works for the most recent GDC Vault videos, since they all use the same player | |
# format. If the XML format used to run the player is changed (as it has in the past), the code | |
# will have to be reconfigured. In the past, I was able to feed a wget-compatible cookies.txt | |
# file into the wget call, but I can't get it to trigger anymore. So for now, the way I download | |
# each video is I look at the source for the video page, find the player.html URL, and feed | |
# that into the script. Ugly and slow, but hey, it works. | |
# I generally hate reinventing the wheel and it does look like youtube-dl does some of the same | |
# stuff I'm doing, but I couldn't get it to work with the GDC URLs. So off to Python land we go!!! | |
# Usage is as follows: | |
# gdc-downloader.py "[GDC player.html URL]" [output dir] | |
# | |
# A GDC video URL looks like this: | |
# http://www.gdcvault.com/play/1015662/Creative-Panic-How-Agility-Turned | |
# | |
# A GDC player.html URL looks like this: | |
# http://evt.dispeak.com/ubm/gdc/sf12/player.html?xmlURL=xml/201203238_1331124629609NXXJ.xml&token=1234567890 | |
# | |
# The output dir should be the name of your video. For example, suppling TestDir/GDCVid will create | |
# TestDir/GDCVid/GDCVid.xml, TestDir/GDCVid/GDCVid-slide.flv, etc. | |
# You need to have Python 2.7 and rtmpdump installed in order for this script to work. I recommend macports. | |
############# | |
# Constants # | |
############# | |
xml_prefixes = { "default":"", "sf13":"xml/" } | |
rtmp_suffixes = { "default":"/fcs/ident", "gdc2009":"/ondemand" } | |
player_regular_expression = r"^(.*[/](.*?)[/])(.*?player\.html)(.*?xml.*?=(.*?)([&].*?)?)$" | |
swf_name_regular_expression = r"^.*embed the Flash Content SWF when all tests are passed.*?\"src\".*?\"(.*?)\".*$" | |
# DEPRECATED: This URL was retrieved from the player SWF, and may change in the future. | |
# rtmp_url = "rtmp://fms.digitallyspeaking.com/cfx/st/ondemand/fcs/ident" | |
######## | |
# Code # | |
######## | |
import sys | |
import os | |
import subprocess | |
import re | |
import argparse | |
from urlparse import urlparse | |
from urllib2 import urlopen | |
from urllib2 import Request | |
from urllib2 import HTTPError | |
from urllib2 import URLError | |
from xml.dom import minidom | |
def text(msg): | |
print "[gdc_downloader] " + msg | |
def error(message): | |
print "[gdc-downloader] Error: " + message | |
sys.exit(1) | |
def message(msg): | |
print "[gdc-downloader] Message: " + msg | |
def check_dependencies(): | |
# TODO: check rtmpdump | |
pass | |
def dump_to_file(data, dest): | |
dest_dir = os.path.abspath(os.path.split(dest)[0]) | |
if not os.path.exists(dest_dir): | |
os.makedirs(dest_dir) | |
debug_file = open(dest, "w") | |
debug_file.write(data) | |
debug_file.close() | |
def download_url(url): | |
r = Request(url=url) | |
message("downloading url: " + url) | |
try: | |
response = urlopen(r) | |
except HTTPError, e: | |
error("http error: " + "\"" + str(e) + "\"") | |
except URLError, e: | |
error("url error (make sure you're online): " + "\"" + str(e) + "\"") | |
return response.read() | |
def retrieve_data_from_base_url(url, xml_dest): | |
html = url | |
player_regex = re.compile(player_regular_expression) | |
player_results = player_regex.match(html) | |
if not player_results: | |
error("player URL not found") | |
dump_to_file(html, xml_dest) | |
base_url = player_results.group(1) | |
event_name = player_results.group(2) | |
player_url = player_results.group(3) | |
player_arguments = player_results.group(4) | |
xml_url = player_results.group(5) | |
full_xml_url = base_url + (xml_prefixes[event_name] if event_name in xml_prefixes else xml_prefixes["default"]) + xml_url | |
message("player url is " + base_url + player_url) | |
message("event name is " + event_name) | |
message("player arguments are " + player_arguments) | |
message("xml url is " + full_xml_url) | |
player_html = download_url(base_url + player_url).replace('\n', '').replace('\r', '') | |
swf_name_regex = re.compile(swf_name_regular_expression) | |
swf_name_results = swf_name_regex.match(player_html) | |
if not swf_name_results: | |
error("SWF URL not found") | |
swf_url = base_url + swf_name_results.group(1) + ".swf" | |
message("swf url is " + swf_url) | |
data = {} | |
data["player_url"] = base_url + player_url + player_arguments | |
data["event_name"] = event_name | |
data["swf_url"] = swf_url | |
data["xml_url"] = full_xml_url | |
return data | |
def parse_xml_from_url(url, xml_dest, event_name): | |
xml = download_url(url) | |
dump_to_file(xml, xml_dest) | |
parsed_xml = minidom.parseString(xml) | |
# GDC2013 abandons the old rtmp streams in favor of raw mp4 urls | |
mbr_video_tags = parsed_xml.getElementsByTagName("MBRVideo") | |
mp4_video_tags = parsed_xml.getElementsByTagName("mp4video") | |
small_video_tags = parsed_xml.getElementsByTagName("smallResolutionVideo") | |
if (mbr_video_tags or mp4_video_tags or small_video_tags): # list the mp4 urls without downloading | |
text("") | |
text("Found raw mp4 urls, please download one of these with your favorite browser:") | |
mp4_urls = [] | |
base_mp4_url = "" | |
# the "mp4video" tag is likely to exist, and will provide a base url for the "MBRVideo" tags | |
if mp4_video_tags: | |
if mp4_video_tags[0].firstChild is not None: | |
parsed_mp4_url = urlparse(mp4_video_tags[0].firstChild.nodeValue) | |
if not parsed_mp4_url.netloc: | |
message("base mp4 url not found, please do some digging or contact the gist author") | |
else: | |
base_mp4_url = "http://" + parsed_mp4_url.netloc + "/" | |
text(" * base mp4 url is " + base_mp4_url) | |
text(" * default mp4 url is " + mp4_video_tags[0].firstChild.nodeValue) | |
# this isn't really necessary, but included for completion | |
if small_video_tags: | |
if small_video_tags[0].firstChild is not None: | |
text(" * small video url is " + small_video_tags[0].firstChild.nodeValue) | |
for subtag in mbr_video_tags: | |
stream = subtag.getElementsByTagName("streamName") | |
if stream: | |
if stream[0].firstChild is not None: | |
# the streamName should conform to "mp4:path/to/video.mp4" | |
text(" * video url is " + base_mp4_url + stream[0].firstChild.nodeValue.partition(":")[2]) | |
text("") | |
return {} | |
else: # download the rtmp streams | |
akamai_host_xml = parsed_xml.getElementsByTagName("akamaiHost") | |
speaker_video_xml = parsed_xml.getElementsByTagName("speakerVideo") | |
slide_video_xml = parsed_xml.getElementsByTagName("slideVideo") | |
if not akamai_host_xml or not speaker_video_xml or not slide_video_xml: | |
error("xml missing properties") | |
if akamai_host_xml[0].firstChild is None or speaker_video_xml[0].firstChild is None or slide_video_xml[0].firstChild is None: | |
error("xml missing properties") | |
akamai_host = "rtmp://" + akamai_host_xml[0].firstChild.nodeValue + (rtmp_suffixes[event_name] if event_name in rtmp_suffixes else rtmp_suffixes["default"]) | |
speaker_video = speaker_video_xml[0].firstChild.nodeValue.replace(".flv", "") | |
slide_video = slide_video_xml[0].firstChild.nodeValue.replace(".flv", "") | |
message("akamai host is " + akamai_host) | |
message("speaker video is " + speaker_video) | |
message("slide video is " + slide_video) | |
# some of the xml files contain exta audio tracks; we want those, don't we? | |
audios = parsed_xml.getElementsByTagName("audios") | |
audio_metadata = {} | |
if (audios): | |
for audio_node in audios[0].getElementsByTagName("audio"): | |
audio_url = None | |
code = None | |
for (name, value) in audio_node.attributes.items(): | |
if name == "url": | |
audio_url = value.replace(".flv", "") | |
elif name == "code": | |
code = value | |
if code: | |
audio_metadata[code] = audio_url | |
message("audio " + code + " is " + audio_url) | |
data = {} | |
data["akamai"] = akamai_host | |
data["speaker"] = speaker_video | |
data["slide"] = slide_video | |
data["audio"] = audio_metadata | |
return data | |
def download_video(rtmp, playpath, swf_url, page_url, filename): | |
args = ["rtmpdump", "--rtmp", rtmp, "--playpath", playpath, "--swfUrl", swf_url, "--pageUrl", page_url, "--flv", filename] | |
try: | |
retval = subprocess.call(args, stdin=None) | |
except Exception, e: | |
error("rtmpdump error") | |
return None | |
def download_gdc_video_at_url(url, dest=""): | |
dest_path = os.path.abspath(dest) | |
dest_name = "GDCVideo" if os.path.split(dest)[1] == "" else os.path.split(dest)[1] | |
# Step 0: Check dependencies. | |
check_dependencies() | |
# Step 1: Extract the following from the URL: player URL, SWF URL, XML URL. | |
data = retrieve_data_from_base_url(url, os.path.join(dest_path, dest_name + "-player-url.txt")) | |
# Step 2: Parse the XML and extract the speaker video URL, slide video URL, and metadata. | |
metadata = parse_xml_from_url(data["xml_url"], os.path.join(dest_path, dest_name + ".xml"), data["event_name"]) | |
# Step 3: Download the videos. | |
if (metadata): | |
download_video(metadata["akamai"], metadata["slide"], data["swf_url"], data["player_url"], os.path.join(dest_path, dest_name + "-slide.flv")) | |
download_video(metadata["akamai"], metadata["speaker"], data["swf_url"], data["player_url"], os.path.join(dest_path, dest_name + "-speaker.flv")) | |
for code in metadata["audio"]: | |
download_video(metadata["akamai"], metadata["audio"][code], data["swf_url"], data["player_url"], os.path.join(dest_path, dest_name + "-audio-" + code + ".flv")) | |
message("All done!") | |
def _main(): | |
parser = argparse.ArgumentParser() | |
parser.add_argument("player_url", help="the full player.html URL retrieved from the video page source") | |
parser.add_argument("output_name", nargs='?', help="the name of the output directory") | |
args = parser.parse_args() | |
download_gdc_video_at_url(args.player_url, ("" if args.output_name is None else args.output_name)); | |
if __name__ == "__main__": | |
_main() |
Yeah, it looks like the format is slightly different between GDC versions. I think this script works correctly for GDC12.
OK, should be fixed now. (Though the rtmp suffixes for older GDCs other than 2009 will have to be added manually, in the rtmp_suffixes dictionary.) Also removed the wget dependency. Thanks for the heads up! (Out of curiosity, how did you get that rtmp url?)
Спасибо за обновление! :)
Я уже не помню как гуглил.
Вероятно использовал слова "akamai rtmp ident" или что-то вроде этого + перебирал из возможных.
I've added a GDC Vault video extractor to youtube-dl now. It should work with flv, mp4 and member only videos (if you provide login details). I wouldn't have thought of using rtmpdump if it wasn't for this gist, thanks :)
Fantastic! Now I don't have to deal with this messy code anymore. :)
Great tool :) However, I didn't manage to download following talk, or any talk from years before 2012.
Do you have any advice?
http://evt.dispeak.com/ubm/gdc/sf12/player.html?xmlURL=xml/201203050_1331336633437YSCO.xml&token=3c6c000ab0766078310c
Could you say how to solve "rtmpdump error"??
Could you say how to solve "rtmpdump error"??
hello @arghavanaa, you need sure is rtmpdump has installed on your os
Trying to do this for GDC2016 videos, it doesn't seem to be working. The video URL is in the form of
C:\..\gdc-downloader>gdc-downloader.py http://evt.dispeak.com/ubm/gdc/sf16/player.html?xml=837752_YBOF.xml&videoid=fa830cc0fb0468b62f391022991 .\TestVideo
[gdc-downloader] Message: player url is http://evt.dispeak.com/ubm/gdc/sf16/player.html
[gdc-downloader] Message: event name is sf16
[gdc-downloader] Message: player arguments are ?xml=837752_YBOF.xml
[gdc-downloader] Message: xml url is http://evt.dispeak.com/ubm/gdc/sf16/837752_YBOF.xml
[gdc-downloader] Message: downloading url: http://evt.dispeak.com/ubm/gdc/sf16/player.html
[gdc-downloader] Error: SWF URL not found
'videoid' is not recognized as an internal or external command,operable program or batch file.
So the &token method seems for have been changed. Its not able to find the SWF file.
I do have some respite though. I have not tested this for all videos, but if you directly go to the video URL (http://evt.dispeak.com/ubm/gdc/sf16/player.html?xml=837752_YBOF.xml&videoid=fa830cc0fb0468b62f391022991) and Inspect element via chrome, you can get the .mp4 file link that you can download.
Hope that helps and maybe someone can modify the code for it to work with the GDC2016 videos.
Look: http://www.gdcvault.com/play/1277/HALO-WARS-The-Terrain-of
To have a script to work, I had to change a few lines:
I am not a programmer/coder, so I do not know how correctly to finish script.
Thanks for the script.
By the way, it is better to mention that the script runs on Python version 2.7.4, and 3 does not work. Other than that require wget and rtmpdump.