Skip to content

Instantly share code, notes, and snippets.

@scola
Created April 17, 2013 12:13
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save scola/5403766 to your computer and use it in GitHub Desktop.
Save scola/5403766 to your computer and use it in GitHub Desktop.
This program can help you download binary just after build finished. If you try it,you will love it.
"""
This program can help you download binary just after build finished.
If you try it,you will love it.
"""
#!/usr/bin/env python
# coding:utf-8
__version__ = '1.0'
__date__ = '2013-03-28'
__author__ = "shaozheng.wu@gmail.com"
import optparse
import sys
import os
import time
import re
import threading
import getpass
import random
cscCode = {"oversea":r"(Download in HQ.+\bCODE_.+Download in oversea|Download in HQ.+\bCSC_CH.+Download in oversea)",
"HQ" :r'''(http.+\bCODE_.+\bHQ"><span>CODE_|http.+\bCSC_CH.+\bHQ"><span>CSC_CHN_)'''}
# Maybe it's good idea to make the friendlyPrompt in separate file and changable,but I don't want to implement it
friendlyPrompt = ["I wish you build successfully and fix issues",
"The evil build is wasting my time",
"Please have a drink and keep healthy",
"If you love coding,you'll love python",
"If you love surfing the internet,you'll love twitter"]
myfilter = lambda s:'md5' in s and 'http' in s
waitChild = lambda s:s.join()
_realgetpass = getpass.getpass
def getpass_getpass(prompt='Password:', stream=None):
"""
This getpass function is from goagent open source project
you can find it at https://github.com/goagent/goagent
"""
try:
import msvcrt
password = ''
sys.stdout.write(prompt)
while 1:
ch = msvcrt.getch()
if ch == '\b':
if password:
password = password[:-1]
sys.stdout.write('\b \b')
else:
continue
elif ch == '\r':
sys.stdout.write(os.linesep)
return password
else:
password += ch
sys.stdout.write('*')
except Exception, e:
return _realgetpass(prompt, stream)
#getpass.getpass = getpass_getpass
def getRsrcURL(id,area,urlLogined):
rsrcURL = re.findall(cscCode[area],urlLogined.urlopen('http://10.90.105.51:8810/build/%s' %id).read())
getCodeCscURL = lambda s:filter(myfilter,s.split('"'))
codeCscURL = map(getCodeCscURL,rsrcURL)
if not codeCscURL:
percentage = re.findall(r'''progress-percentage">\d+%''',urlLogined.urlopen('http://10.90.105.51:8810/build/%s' %id).read())
if percentage:
percent = re.findall(r"\d+",percentage[0])[0]
print 'build %s %s%s finished.%s' %(id,'%',percent,random.choice(friendlyPrompt))
if int(percent) < 95:
time.sleep(50) #adjust the refresh frequency
else:
print "Maybe build %s failed,please check the website or mailbox" %id
sys.exit(3)
return codeCscURL
def createThread(target,args):
t = threading.Thread(target=target,args=args)
t.setDaemon(1)
t.start()
#t.join()
return t
def downloadThread(url,id,urlLogined):
time.sleep(1)
try:
req = urlLogined.Request(url[0])
req.add_header('Referer','http://10.90.105.51:8810/build/%s' %id)
fp = urlLogined.urlretrieve(req,url[0].split('/')[-1]) # use your group
except:
print "Download %s failed,please check your network or code" %url[0].split('/')[-1]
sys.exit(4)
print "Congratulations!you've downloaded %s successfully" %url[0].split('/')[-1]
def threadcode(id,area,urlLogined):
time.sleep(1)
starttime = time.time()
while not getRsrcURL(id,area,urlLogined):
#print "Build %s not finished,please wait to download" %id
time.sleep(10)
#if int(time.time() - starttime) > 4800:
# print "Maybe build failed,please check the website or mailbox"
codeCscURL = getRsrcURL(id,area,urlLogined)
print "start to Download %s ..." %id
#for url in codeCscURL:
createDownloadThread = lambda s:createThread(target = downloadThread,args=(s,id,urlLogined))
downloadThreadList = map(createDownloadThread,codeCscURL)
map(waitChild,downloadThreadList)
def login(username,password):
"""
inspired by
http://code.activestate.com/recipes/391929-access-password-protected-web-applications-for-scr/
ClientCookie-1.0.3 and ClientForm-0.1.17 are open source project.
You can find it at http://wwwsearch.sourceforge.net/ClientCookie/src/ClientCookie-1.0.3.tar.gz
and http://wwwsearch.sourceforge.net/ClientForm/src/ClientForm-0.1.17.tar.gz
"""
sys.path.append('ClientCookie-1.0.3')
import ClientCookie
sys.path.append('ClientForm-0.1.17')
import ClientForm
# Create special URL opener (for User-Agent) and cookieJar
cookieJar = ClientCookie.CookieJar()
opener = ClientCookie.build_opener(ClientCookie.HTTPCookieProcessor(cookieJar))
opener.addheaders = [("User-agent","Mozilla/5.0 (compatible)")]
ClientCookie.install_opener(opener)
fp = ClientCookie.urlopen("http://10.90.105.51:8810/signin")
forms = ClientForm.ParseResponse(fp)
fp.close()
form = forms[0]
form["userName"] = username # use your userid
form["password"] = password # use your password
try:
fp = ClientCookie.urlopen(form.click())
fp.close()
except:
print "Please input correct username and password"
sys.exit(2)
return ClientCookie
def main():
#starttime = time.time()
print __doc__
p = optparse.OptionParser()
p.add_option('--id', '-i', default=None)
p.add_option('--area', '-a', default="oversea")
p.add_option('--username', '-u', default="shaozheng.wu")
p.add_option('--password', '-p', default="66369xxx")
options, arguments = p.parse_args()
if options.username != "shaozheng.wu" and options.password == "66369xxx":
options.password = getpass_getpass().rstrip()
urlLogined = login(options.username,options.password)
if options.id:
ids = options.id.split('+')
ids = set(ids) # make the ids not duplicates
for id in ids:
if not id.isdigit() or len(id) < 6 :
print "Please input valid build id"
print p.print_help()
sys.exit(1)
createBuildThread = lambda s:createThread(target = threadcode,args=(s,options.area,urlLogined))
buildThread = map(createBuildThread,ids)
map(waitChild,buildThread)
# createThread(target = threadcode,args=(id,options.area,urlLogined)).join()
else:
print p.print_help()
if __name__ == '__main__':
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment