| import re, hashlib, uuid, json, random, os, urllib2, os.path, time, sys, SimpleHTTPServer, SocketServer, string, console, webbrowser, shutil, zipfile | |
| class SmarterHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): | |
| server_version = 'SimpleHTTP/0.6' | |
| file_name = '' | |
| def do_GET(self): | |
| if self.path.startswith('/transfer'): | |
| self.get_transfer() | |
| else: | |
| f = self.send_head() | |
| if f: | |
| self.copyfile(f, self.wfile) | |
| f.close() | |
| def get_transfer(self): | |
| global did_download | |
| try: | |
| # Perform the actual file download | |
| self.send_response(200) | |
| # Content-Disposition: attachment; filename="fname.ext" | |
| self.send_header('Content-Disposition', 'attachment; filename="%s"' % (self.file_name.split('/',1)[-1])) | |
| self.send_header('Content-Length', '%s' % (os.path.getsize(self.file_name))) | |
| self.send_header('Content-Type', 'application/octet-stream') | |
| self.end_headers() | |
| f = open(self.file_name, 'rb') | |
| self.copyfile(f, self.wfile) | |
| f.close() | |
| except: | |
| sys.exc_clear() | |
| did_download = True | |
| def log_message(self, format, *args): | |
| return | |
| def finish(self): | |
| # Fix for iDownload when it early terminates a transfer to get file details | |
| if not self.wfile.closed: | |
| try: | |
| self.wfile.flush() | |
| except: | |
| sys.exc_clear() | |
| try: | |
| self.wfile.close() | |
| except: | |
| sys.exc_clear() | |
| try: | |
| self.rfile.close() | |
| except: | |
| sys.exc_clear() | |
| class SmarterHTTPD(SocketServer.ThreadingTCPServer): | |
| keep_running = True | |
| requests_left = None | |
| did_timeout = False | |
| def serve_limited(self, timeout=None, max_requests=None): | |
| global ready_to_stop | |
| self.timeout = timeout | |
| if max_requests is None: | |
| self.requests_left = None | |
| else: | |
| self.requests_left = abs(int(max_requests)) | |
| self.keep_running = True | |
| self.did_timeout = False | |
| while self.keep_running: | |
| self.handle_request() | |
| # print "Request handled." | |
| if self.requests_left is not None: | |
| self.requests_left -= 1 | |
| if self.requests_left <= 0: | |
| self.keep_running = False | |
| # print "EXIT: HIT MAX REQUESTS" | |
| continue | |
| if ready_to_stop: | |
| self.keep_running = False | |
| # print "EXIT: TOLD TO STOP" | |
| continue | |
| def handle_timeout(self): | |
| self.did_timeout = True | |
| def handle_error(self, request, client_address): | |
| # Overidden, don't care to see any messages | |
| return | |
| def release(self): | |
| try: | |
| self.server_close() | |
| except Exception: | |
| sys.exc_clear() | |
| try: | |
| self.socket.close() | |
| except Exception: | |
| sys.exc_clear() | |
| class pyGroovClient: | |
| def __init__(self): | |
| self.client_url = 'http://html5.grooveshark.com/' | |
| self.s_client_url = self.client_url.replace('http', 'https') | |
| # This user agent is in the top 7% of user agents on the web | |
| self.user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31' | |
| # Trigger client setup | |
| pyGroovClient.setupClient(self) | |
| def _chunk_report(self, bytes_so_far, chunk_size, total_size): | |
| if (total_size != None): | |
| percent = float(bytes_so_far) / total_size | |
| percent = round(percent*100, 2) | |
| print 'Downloaded %d of %d bytes (%0.2f%%)' % (bytes_so_far, total_size, percent) | |
| if bytes_so_far >= total_size: | |
| print '' | |
| else: | |
| print 'Downloaded %d bytes' % (bytes_so_far) | |
| def _chunk_read(self, response, chunk_size=32768, report_hook=None, filename=None, streamDict=None): | |
| # Delete old file if it's present | |
| if os.path.exists(filename): | |
| os.remove(filename) | |
| # Set up the file handler | |
| try: | |
| f = open(filename, 'wb') | |
| except Exception: | |
| print '! Error:', sys.exc_info()[1] | |
| raise | |
| start_time = time.time() | |
| did_hit_30 = False | |
| time_so_far = 0 | |
| if response.info().has_key('Content-Length'): | |
| total_size = response.info().getheader('Content-Length').strip() | |
| total_size = int(total_size) | |
| else: | |
| # No size | |
| total_size = None | |
| if report_hook: | |
| print '* Warning: No total file size available.' | |
| bytes_so_far = 0 | |
| i = 0 | |
| while True: | |
| chunk = response.read(chunk_size) | |
| bytes_so_far += len(chunk) | |
| # Check time, notify when complete | |
| now_time = time.time() | |
| if not did_hit_30: | |
| if abs(now_time - start_time) >= 30: | |
| did_hit_30 = True | |
| print "* Notifying 30 seconds of play ..." | |
| self._markStreamKeyOver30Seconds(streamDict) | |
| # Set remaining time to 0 | |
| time_so_far = 31.0 | |
| if not chunk: | |
| break | |
| else: | |
| f.write(chunk) | |
| if not i: | |
| report_hook(bytes_so_far, chunk_size, total_size) | |
| i = (i+1)%5 | |
| if not did_hit_30: | |
| # We didn't hit 30 seconds during the download, better let parent method know | |
| # For safety, since we know we didn't send the message and don't want a race condition, assume no greater than 29 seconds | |
| time_so_far = min(29.0, time.time() - start_time) | |
| try: | |
| f.close() | |
| except: | |
| _ = False | |
| if bytes_so_far > 0: | |
| print '* Saved to:', filename | |
| return (os.path.abspath(filename), time_so_far) | |
| else: | |
| print '* Error: 0 bytes downloaded, not saved.' | |
| return (None, time_so_far) | |
| def _download(self, src_url, fname='download.mp3', streamDict=None): | |
| headers = {'User-Agent': self.user_agent, 'Cookie': 'PHPSESSID=%s' % self.session} | |
| print 'INFO: This download will take a *minimum* of 30 seconds, to keep Grooveshark from banning you.' | |
| print '* Downloading:', src_url | |
| req = urllib2.Request(src_url, headers=headers) | |
| response = urllib2.urlopen(req) | |
| filename,time_spent = self._chunk_read(response, report_hook=self._chunk_report, filename=fname, streamDict=streamDict) | |
| if filename: | |
| if time_spent < 30.0: | |
| # Need to sleep a little longer, then notify the 30 second download | |
| print "* Waiting remaining seconds to reach 30 ..." | |
| time.sleep(31.0 - time_spent) | |
| print "* Notifying 30 seconds of play." | |
| self._markStreamKeyOver30Seconds(streamDict) | |
| print "* Completed." | |
| return filename | |
| else: | |
| print "* Error, aborting." | |
| if os.path.exists(fname): | |
| os.remove(fname) | |
| def setupClient(self): | |
| # Generally only called by __init__, but can be called manually to create a new session | |
| # Create a single web browsing session to retain cookies, headers, etc. | |
| self.sess = requests.Session() | |
| # Fake our user agent | |
| self.sess.headers.update({'User-Agent': self.user_agent}) | |
| # Load the initial page to get a PHP session cookie and a few other configuration settings | |
| _ = self.sess.get(self.client_url) | |
| self.base_html = _.content | |
| # Some of these help fake out Grooveshark so it doesn't know this is a software library. | |
| # Download but ignore /build/app.min.css?#### | |
| _ = re.search(r'build/app\.min\.css\?[0-9]+', self.base_html).group() | |
| _ = self.sess.get(self.client_url + _) | |
| # Download but ignore /build/libs.min.js?#### | |
| _ = re.search(r'build/libs\.min\.js\?[0-9]+', self.base_html).group() | |
| _ = self.sess.get(self.client_url + _) | |
| # Download and keep /build/app.min.js?#### | |
| _ = re.search(r'build/app\.min\.js\?[0-9]+', self.base_html).group() | |
| _ = self.sess.get(self.client_url + _) | |
| self.app_js = _.content | |
| # Exctract the app and base configuration blocks | |
| app_snip = re.search(r'SERVICE_CREATE_TOKEN_FAIL.+?(var .+?lastRandomizer.+?;)', self.app_js).groups()[0] | |
| base_snip = re.search(r'window\.GS\.config.+?(\{.+?\});', self.base_html).groups()[0] | |
| # From app determine: client, clientRevision, revToken | |
| self.client = re.search(r'client:[ ]*"(.+?)"', app_snip).groups()[0] | |
| self.clientRevision = re.search(r'clientRevision:[ ]*"(.+?)"', app_snip).groups()[0] | |
| self.revToken = re.search(r'="([^"]+?)"', app_snip).groups()[0] | |
| # From base determine: privacy, country | |
| self.privacy = re.search(r'"Privacy":[ ]*([0-9]+?)', base_snip).groups()[0] | |
| self.country = re.search(r'"country":[ ]*(\{.+?\})', base_snip).groups()[0] | |
| # From sess determine: session, secretKey (it's present in base, but we're using sess's cookies to do future requests - should be same) | |
| self.session = self.sess.cookies['PHPSESSID'] | |
| self.secretKey = hashlib.md5(self.session).hexdigest() | |
| # Generate a UUID | |
| self.uuid = str(uuid.uuid4()).upper() | |
| # Get our communication token | |
| self._getCommunicatonToken() | |
| def search(self, searchStr): | |
| # Song search, in order of best match | |
| try: | |
| return self._getResultsFromSearch(searchStr.replace('"', '\\"'))['result']['result']['Songs'] | |
| except: | |
| return [] | |
| def download(self, songDict, filepath): | |
| stream_info = self._getStreamKeyFromSongIDEx(songDict) | |
| if not stream_info: | |
| # Song was either removed by Grooveshark or is not available to HTML5/mobile/non-Flash client | |
| return False | |
| # A download is available, download it - but mark it as downloaded first (HTML5 client does this) | |
| _ = self._markSongDownloadedEx(stream_info) | |
| # Then download it :) | |
| download_url = 'http://%s/stream.php?streamKey=%s' % (stream_info['ip'],stream_info['streamKey']) | |
| return self._download(download_url, fname=filepath, streamDict=stream_info) | |
| def _getCommunicatonToken(self): | |
| # Use the secure client for the token | |
| gCT_json = '{"header":{"client":"%s","clientRevision":"%s","privacy":%s,"country":%s,"uuid":"%s","session":"%s"},"method":"getCommunicationToken","parameters":{"secretKey":"%s"}}' | |
| _ = self.sess.post(self.s_client_url + '/more.php?getCommunicationToken', data=gCT_json % (self.client, self.clientRevision, self.privacy, self.country, self.uuid, self.session, self.secretKey)) | |
| # Use json lib to future proof against newer versions of requests | |
| self.token = json.loads(_.content)['result'] | |
| def _prepToken(self, method): | |
| rnd = hashlib.md5(str(random.random())).hexdigest()[:6] | |
| return rnd + hashlib.sha1(':'.join([method, self.token, self.revToken, rnd])).hexdigest() | |
| def _buildAPIcall(self, method, parameters): | |
| core_msg = '{"header":{"client":"%s","clientRevision":"%s","privacy":%s,"country":%s,"uuid":"%s","session":"%s","token":"%s"},"method":"%s","parameters":%s}' | |
| return core_msg % (self.client, self.clientRevision, self.privacy, self.country, self.uuid, self.session, self._prepToken(method), method, parameters) | |
| def _doAPIcall(self, method, parameters): | |
| _ = self.sess.post(self.client_url + 'more.php?%s' % method, data=self._buildAPIcall(method, parameters)) | |
| return _.content | |
| def _getResultsFromSearch(self, searchStr): | |
| params = '{"query":"%s","type":["Songs","Playlists","Albums"],"guts":0,"ppOverride":""}' | |
| return json.loads(self._doAPIcall('getResultsFromSearch', params % searchStr)) | |
| def _getStreamKeyFromSongIDEx(self, songDict): | |
| params = '{"prefetch":false,"mobile":true,"songID":%s,"country":%s}' | |
| return json.loads(self._doAPIcall('getStreamKeyFromSongIDEx', params % (songDict['SongID'], self.country)))['result'] | |
| def _markSongDownloadedEx(self, streamDict): | |
| # streamKey, streamServerID, songID | |
| params = '{"streamKey":"%s","streamServerID":%s,"songID":%s}' | |
| return json.loads(self._doAPIcall('markSongDownloadedEx', params % (streamDict['streamKey'],streamDict['streamServerID'],streamDict['SongID'])))['result'] | |
| def _markStreamKeyOver30Seconds(self, streamDict): | |
| params = '{"streamKey":"%s","streamServerID":%s,"songID":%s}' | |
| return json.loads(self._doAPIcall('markStreamKeyOver30Seconds', params % (streamDict['streamKey'],streamDict['streamServerID'],streamDict['SongID'])))['result'] | |
| # iDownloads | |
| def sanitize(filename): | |
| safe = string.letters + string.digits + "()-.,_+{}'" | |
| return ''.join([['_',x][x in safe] for x in filename]) | |
| def do_search_and_download(gsClient): | |
| print "\nEnter your search:" | |
| searchStr = raw_input('> ').strip() | |
| if not searchStr: | |
| print "* Cancelled." | |
| return | |
| print "* Searching ..." | |
| results = gsClient.search(searchStr) | |
| top8 = results[:8] | |
| if not top8: | |
| print "! No results found for:", searchStr | |
| return | |
| print "* Found, enter the number for download:" | |
| for i,x in enumerate(top8): | |
| print "%s) %s - %s - %s" % (i+1, x['SongName'], x['ArtistName'], x['AlbumName']) | |
| print '0) Cancel' | |
| choice = ''.join([x for x in raw_input('> ').strip() if x in '0123456789']) | |
| if (choice in ['0','']): | |
| print "* Cancelled." | |
| return | |
| else: | |
| nChoice = int(choice) - 1 | |
| cDict = top8[nChoice] | |
| print "*Downloading: %s" % cDict['SongName'] | |
| fname = "gs_dl/" + sanitize("%s - %s - %s.mp3" % (cDict['ArtistName'], cDict['AlbumName'],cDict['SongName'])) | |
| if not os.path.exists('gs_dl'): | |
| os.makedirs('gs_dl') | |
| if gsClient.download(cDict, fname) == False: | |
| print "* Song not available (removed or not for mobile)." | |
| return | |
| # Prep webserver | |
| global ready_to_stop, did_download | |
| ready_to_stop = False | |
| did_download = False | |
| port = 8000 | |
| handler = SmarterHTTPRequestHandler | |
| # Configure transfer settings | |
| handler.file_name = fname | |
| httpd = SmarterHTTPD(("", port), handler, False) | |
| httpd.allow_reuse_address = True | |
| httpd.server_bind() | |
| httpd.server_activate() | |
| download_url = 'http://127.0.0.1:8000/transfer' | |
| download_url = download_url.replace('http://', 'iDownloads://') | |
| print '* Transferring to browser ...' | |
| webbrowser.open(download_url) | |
| # print download_url | |
| httpd.serve_limited(timeout=3,max_requests=8) | |
| httpd.release() | |
| if did_download: | |
| print '* Transfer complete, deleting local copy.' | |
| else: | |
| print '* Transfer did not complete, deleting local copy.' | |
| try: | |
| os.remove(fname) | |
| except: | |
| _ = False | |
| return | |
| def _unzip(a_zip=None, path='.', altpath='unzipped'): | |
| if a_zip is None: | |
| return | |
| filename = os.path.abspath(a_zip) | |
| if not os.path.isfile(filename): | |
| return | |
| # PK magic marker check | |
| f = open(filename, 'rb') | |
| try: | |
| pk_check = f.read(2) | |
| except Exception: | |
| pk_check = '' | |
| finally: | |
| f.close() | |
| if pk_check != 'PK': | |
| print "unzip: %s: does not appear to be a zip file" % a_zip | |
| else: | |
| altpath = os.path.join(os.path.dirname(filename), altpath) | |
| location = os.path.abspath(altpath) | |
| if not os.path.exists(location): | |
| os.makedirs(location) | |
| zipfp = open(filename, 'rb') | |
| try: | |
| zipf = zipfile.ZipFile(zipfp) | |
| # check for a leading directory common to all files and remove it | |
| dirnames = [os.path.join(os.path.dirname(x), '') for x in zipf.namelist()] | |
| common_dir = os.path.commonprefix(dirnames or ['/']) | |
| # Check to make sure there aren't 2 or more sub directories with the same prefix | |
| if not common_dir.endswith('/'): | |
| common_dir = os.path.join(os.path.dirname(common_dir), '') | |
| for name in zipf.namelist(): | |
| data = zipf.read(name) | |
| fn = name | |
| if common_dir: | |
| if fn.startswith(common_dir): | |
| fn = fn.split(common_dir, 1)[-1] | |
| elif fn.startswith('/' + common_dir): | |
| fn = fn.split('/' + common_dir, 1)[-1] | |
| fn = fn.lstrip('/') | |
| fn = os.path.join(location, fn) | |
| dirf = os.path.dirname(fn) | |
| if not os.path.exists(dirf): | |
| os.makedirs(dirf) | |
| if fn.endswith('/'): | |
| # A directory | |
| if not os.path.exists(fn): | |
| os.makedirs(fn) | |
| else: | |
| fp = open(fn, 'wb') | |
| try: | |
| fp.write(data) | |
| finally: | |
| fp.close() | |
| except Exception: | |
| zipfp.close() | |
| print "unzip: %s: zip file is corrupt" % a_zip | |
| return | |
| zipfp.close() | |
| return os.path.abspath(location) | |
| def req12_setup(): | |
| import requests as old_req | |
| relative_dir = os.path.abspath(os.path.dirname(__file__)) | |
| curdir = os.getcwd() | |
| os.chdir(relative_dir) | |
| print '!!! requests-1.2.0 not installed, downloading rev.d06908d ...' | |
| zip_url = 'https://github.com/kennethreitz/requests/archive/v1.2.0.zip' | |
| print ' * Downloading: requests_1.2.0.zip (565KB)...' | |
| f = open('requests_1.2.0.zip', 'wb') | |
| try: | |
| f.write(old_req.get(zip_url).content) | |
| except Exception: | |
| sys.exc_clear() | |
| f.close() | |
| # Unload built-in requests module | |
| del old_req | |
| print "!!! zip downloaded, extracting ..." | |
| try: | |
| shutil.rmtree('requests_zip', ignore_errors=True) | |
| except Exception: | |
| sys.exc_clear() | |
| _ = _unzip('requests_1.2.0.zip', altpath='requests_zip') | |
| print "!!! Extraction complete, re-arranging ..." | |
| try: | |
| shutil.rmtree('requests_1_2', ignore_errors=True) | |
| except Exception: | |
| sys.exc_clear() | |
| os.rename('requests_zip/requests', 'requests_1_2') | |
| print "!!! Re-arranging complete, cleaning up ..." | |
| try: | |
| os.remove('requests_1.2.0.zip') | |
| shutil.rmtree('requests_zip', ignore_errors=True) | |
| except Exception: | |
| sys.exc_clear() | |
| os.chdir(curdir) | |
| def main(): | |
| global requests | |
| # Ensure that the requests v.1.2.0 module is available | |
| requests = None | |
| init_tries = 3 | |
| while init_tries > 0: | |
| try: | |
| import requests_1_2 as requests | |
| init_tries = 0 | |
| except: | |
| print "!!! Init failure %s of 3 ..." % (4 - init_tries) | |
| sys.exc_clear() | |
| req12_setup() | |
| init_tries -= 1 | |
| if not requests: | |
| print '!!! Please check your network connection and try again.' | |
| return | |
| try: | |
| gsc = pyGroovClient() | |
| except Exception: | |
| sys.exc_clear() | |
| print "* Error initializing GS client, make sure you're online." | |
| return | |
| console.clear() | |
| print "* Client successfully initialized" | |
| loop = True | |
| while loop: | |
| print "Enter a menu number choice:" | |
| print "--------------------------" | |
| print "1) Search and download" | |
| print "0) Quit" | |
| choice = ''.join([x for x in raw_input('> ').strip() if x in '0123456789']) | |
| if (choice in ['0','']): | |
| print "* Quit." | |
| loop = False | |
| elif (choice in ['1']): | |
| do_search_and_download(gsc) | |
| else: | |
| print "* Unknown choice number, try again." | |
| print "" | |
| if __name__ == "__main__": | |
| main() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
This comment has been minimized.
http://omz-software.com/pythonista/forums/discussion/293/grooveshark-downloader-enjoy-#Item_13 is a report of some problems reported because of changes to GS web pages.