Panoramio upload bot

pub.py

#!/usr/bin/python
# -*- coding: utf-8  -*-
"""
Script to upload images of panoramio to wikimedia commons.

by Shizhao 2014
 
"""
import urllib2,re, random
import datetime
from BeautifulSoup import BeautifulSoup
import upload,time, StringIO, hashlib, base64
import tempfile
import pywikibot
import pywikibot.data.api
from pywikibot import config
import sys    
from multiprocessing.dummy import Pool as ThreadPool 
reload(sys) # Python2.5 初始化后会删除 sys.setdefaultencoding 这个方法，我们需要重新载入     
sys.setdefaultencoding('utf-8')
site = pywikibot.Site(u'commons', u'commons')

def cleanUpTitle(title, site, author, project = u'panoramio'):
    ''' Clean up the title of a potential mediawiki page. Otherwise the title of
    the page might not be allowed by the software.
    
    from flickrripper.py

    '''
    maxBytes = 240 - len(project.encode('utf-8')) \
                    - len(author.encode('utf-8'))
    titleBytes = len(title.encode('utf-8'))
    if titleBytes > maxBytes:
        # maybe we cut more than needed, anyway we do it
        items = max(min(len(title), maxBytes / 4),
                    len(title) - titleBytes + maxBytes)
        title = title[:items]
    title = title.strip()
    title = re.sub(u"^ $", u"Untitled", title) 
    title = re.sub(u"[<{\\[]", u"(", title)
    title = re.sub(u"[>}\\]]", u")", title)
    title = re.sub(u"[ _]?\\(!\\)", u"", title)
    title = re.sub(u",:[ _]", u", ", title)
    title = re.sub(u"[;:][ _]", u", ", title)
    title = re.sub(u"[\t\n ]+", u" ", title)
    title = re.sub(u"[\r\n ]+", u" ", title)
    title = re.sub(u"[\n]+", u"", title)
    title = re.sub(u"[?!]([.\"]|$)", u"\\1", title)
    title = re.sub(u"[&#%?!]", u"^", title)
    title = re.sub(u"[;]", u",", title)
    title = re.sub(u"[/+\\\\:]", u"-", title)
    title = re.sub(u"--+", u"-", title)
    title = re.sub(u",,+", u",", title)
    title = re.sub(u"[-,^]([.]|$)", u"\\1", title)
    title = title.replace(u" ", u"_")
    if not site:
        site = pywikibot.Site(u'commons', u'commons')
    try:
        if pywikibot.Page(site, u'File:%s - %s.jpg' % (title, project)).exists():
            while True:
                if pywikibot.Page(site, u'File:%s - %s - %s.jpg' % (title, project, author)).exists():
                    i = 1
                    while True:
                        if (pywikibot.Page(site, u'File:%s - %s - %s (%d).jpg' % (title, project, author, i)).exists()):
                            i += 1
                        else:
                            return u'%s - %s - %s (%d).jpg' % (title, project, author, i)
                else:
                    return u'%s - %s - %s.jpg' % (title, project, author)
        else:
            return u'File:%s - %s.jpg' % (title, project)
    except AttributeError:
        return u'File:%s - %s.jpg' % (title, project)	

def checkcountry(cats):
    '''remove country category'''
    for countrycat in cats:
        if u'Countries' not in countrycat.title(withNamespace=False):
            country = False
        else:
            country = True
            break
    return country
    
def checkyear(category):
    '''remove year category'''
    if category.isdigit():
        checkyear = True
    else:
        checkyear = False
    return checkyear

def checkmetacategories(cats):
    '''remove Meta categories'''
    for metacat in cats:
        if u'Meta categories' not in metacat.title(withNamespace=False) or metacat.title(withNamespace=False) <> u'Topics':
            checkmeta = False
        else:
            checkmeta  = True
            break
    return checkmeta 

def checkmaincategory(categorypage):
    '''remove main category'''
    templates = categorypage.templates()
    maincattemp=[u'MetaCat', u'CatDiffuse', u'Categorise', u'CatCat', u'Disambig']
    for temp in templates:
        if temp.title(withNamespace=False) in maincattemp:
            checkmain = True
            break
        else:
            checkmain = False
    return checkmain
        
def buildDescription(Information, site, tags, Location):
    ''' Build the final description for the image.

    '''
    uploder = config.usernames['commons']['commons']
    description = Information
    catexists=len(tags)
    if tags:
        catexists=len(tags)
        catstext = u''
        for category in tags:
            try:
                categorypage=pywikibot.Page(site, u'Category:' + category)
                if categorypage.exists():
                    if categorypage.isCategoryRedirect():
                        cats=categorypage.getCategoryRedirectTarget().categories()
                        if checkcountry(cats) or checkyear(category) or checkmetacategories(cats) or checkmaincategory(categorypage):
                            catexists = catexists -1
                            pywikibot.output(u'remove category: ' + category)
                        else:
                            if u'[[Category:' + categorypage.getCategoryRedirectTarget().title(withNamespace=False) + ']]\n' not in catstext:
                                catstext = catstext + u'[[Category:' + categorypage.getCategoryRedirectTarget().title(withNamespace=False) + ']]\n'
                                pywikibot.output(u'RedirectTarget: '+categorypage.getCategoryRedirectTarget().title())
                            else:
                                catexists = catexists -1
                                pywikibot.output(u'remove category: ' + category + '. Dupe')
                    else:
                        cats=categorypage.categories()
                        if checkcountry(cats) or checkyear(category) or checkmetacategories(cats) or checkmaincategory(categorypage):
                            catexists = catexists -1
                            pywikibot.output(u'remove category: ' + category)
                        else:
                            catstext = catstext + u'[[Category:' + category + ']]\n'
                else:
                    catexists = catexists -1
                    pywikibot.output(u'[[Category:' + category + ']]' + u' not exists.')
            except:
                catexists = catexists -1
        if catexists == 0:
            if Location ==u'':
                description = description + u'{{subst:unc}}\n'
            else:
                description = description + u'{{subst:unc|geo=1}}\n'        
        else:
            description = description+u'{{subst:chc}}\n\n' + catstext
    else:
        if Location ==u'':
            description = description + u'{{subst:unc}}\n'
        else:
            description = description + u'{{subst:unc|geo=1}}\n'    
       
    uploder = config.usernames['commons']['commons']
    description = description + u'[[Category:Panoramio files uploaded by ' + uploder + ']]\n'
        
    return description
    
def downloadPhoto(photoUrl = ''):
    '''
    Download the photo and store it in a StrinIO.StringIO object.

    TODO: Add exception handling

    '''
    imageFile=urllib2.urlopen(photoUrl).read()
    return StringIO.StringIO(imageFile)

def findDuplicateImages(photo=None,
                        site=pywikibot.Site(u'commons', u'commons')):
    ''' Takes the photo, calculates the SHA1 hash and asks the mediawiki api
    for a list of duplicates.

    TODO: Add exception handling, fix site thing

    '''
    hashObject = hashlib.sha1()
    hashObject.update(photo.getvalue())
    return site.getFilesFromAnHash(base64.b16encode(hashObject.digest()))

start = time.time()
add=5
page = pywikibot.Page(site, u"User:Panoramio upload bot/status")
text = page.text
#r=int(text)+1 
r=int(text) 

for arg in pywikibot.handleArgs():
    if arg:
        if arg.startswith('-start:'):
            r = int(arg[7:])
        elif arg.startswith('-range:'):
            add = int(arg[7:])

def runputbot(i):
#for i in range(r,r+add):
    site = pywikibot.Site(u'commons', u'commons')
    url="http://www.panoramio.com/photo/%d" % i
    try:
        page = urllib2.urlopen(url)
        urlverify=True
    except (urllib2.HTTPError, urllib2.URLError), e:
        urlverify=False 
        print 'photo id %d' % i, e
        #sec=random.randint(1, 5)
        #pywikibot.output(u"Waiting for %d seconds." % sec)
        #time.sleep(sec)
    while urlverify:
        soup = BeautifulSoup(page)
        license = soup.find('li', attrs={'class' : re.compile("^license")})['class']
        #print license
        
        if license=="license by-sa" or license== "license by":
            pywikibot.output('photo id %d %s is OK! Ready upload Commons...' % (i, license))
            photo_url = "http://static.panoramio.com/photos/original/%d.jpg" % i
            #print photo_url

        #Should download the photo only once
            trying = True
            n=0
            while trying:           
                try:
                    photo = downloadPhoto(photo_url)
                    trying=False 
                except (urllib2.HTTPError,urllib2.URLError), e:
                    pywikibot.output(e.code + ' try....')
                    n=n+1
                    time.sleep(5*n*n)

        #Don't upload duplicate images, should add override option
            duplicates = findDuplicateImages(photo)
            if duplicates:
                pywikibot.output(u'Found duplicate image at %s' % duplicates.pop())
                urlverify=False 
            else:
                      
            #photo title
                title=soup.find("h1", { "id" : "photo-title" }).contents[0].strip().lstrip().rstrip(',')

                #author and author _url
                author=soup.find("a", { "rel" : "author" }).contents[0]
                author_url = soup.find("a", { "rel" : "author" })['href']
                author_url= 'http://www.panoramio.com' + author_url
                
                #clean filename
                filename=cleanUpTitle(title, site, author)
                pywikibot.output(filename,toStdout=True)
                #print str(filename.decode('utf-8'))        
                #tags
                tags=[]
                for tag in soup.findAll(attrs={'id' : re.compile("^tag_element")}):
                    tags.append(tag.a.contents[0].strip().lstrip().rstrip(','))
                try:
                    mapname=soup.find("div", { "id" : "map_info_name" }).a.contents[0]
                    if mapname not in tags:
                        tags.append(mapname)
                    print tags
                except AttributeError:
                    print tags                  
                
                #date
                try: 
                    date=soup.find("li", { "id" : "tech-details" }).findNext('ul').find(text=re.compile("^Taken on"))
                    formatdate=datetime.datetime.strptime(date,'Taken on %Y/%m/%d %X')
                    date = formatdate.strftime('%Y-%m-%d')
                    date=u'{{Taken on|%s}}' % date
                    print date
                except (AttributeError, TypeError):
                    date=soup.find("ul", { "id" : "details" }).li.contents[0].strip().lstrip().rstrip(',')
                    formatdate=datetime.datetime.strptime(date,'Uploaded on %B %d, %Y')
                    date = formatdate.strftime('%Y-%m-%d')
                    date =u'{{Original upload date|%s}}' % date
                    print 'take on not found.', date
                
                #geo
                try:
                    lat =soup.find("abbr", { "class" : "latitude" })['title']
                    lon = soup.find("abbr", { "class" : "longitude" })['title']
                    Location = u'{{Location|%s|%s|source:Panoramio}}' %(lat, lon)
                except TypeError:
                    Location = u''
                if license == "license by-sa":
                    licensetag = u'{{cc-by-sa-3.0|%s}}' % author
                elif license == "license by":
                    licensetag = u'{{cc-by-3.0|%s}}' % author
                reviewer = config.usernames['commons']['commons']  
                review = u'{{Panoramioreview|%s|{{subst:CURRENTYEAR}}-{{subst:CURRENTMONTH}}-{{subst:CURRENTDAY2}}}}' % reviewer
                
                #print soup.find("div", { "id" : "photo-description-formatted" }).contents[0]
                
                try:
                    photo_description = u''.join(unicode(item) for item in soup.find("div", { "id" : "photo-description-formatted" }).contents).strip().lstrip().rstrip(',')
                    if photo_description:
                        title=title+u'\n\n'+ photo_description
                    else:
                        pass
                except AttributeError:
                    pass
                Information=u'{{Information\n|description=%s\n|date=%s\n|source=%s\n|author=[%s %s]\n|permission=%s\n%s\n|other_versions=\n|other_fields=\n}}\n%s\n\n' % (title, date, url, author_url, author, licensetag, review, Location)
                #site = pywikibot.Site(u'commons', u'commons')
                Description = buildDescription(Information, site, tags, Location)
                #pywikibot.output(Description)

                bot = upload.UploadRobot(photo_url,
                                   description=Description,
                                   useFilename=filename,
                                   keepFilename=True,
                                   verifyDescription=False,
                                   ignoreWarning=True,uploadByUrl=True)
                uploadtoo = True
                n=0             
                while uploadtoo:    
                    try: 
                        bot.upload_image(debug=True)
                        urlverify=False
                        uploadtoo = False
                        #sec=random.randint(1, 10)
                        #pywikibot.output(u"Finished upload. Waiting for %d seconds." % sec)
                        #time.sleep(sec)
                    except e:
                        n=n+1
                        print e.code
                        #time.sleep(5*n*n)
        else:
            pywikibot.output('photo id %d %s is invalid! Ignore...' % (i, license))
            #sec=random.randint(1, 5)
            #pywikibot.output(u"Waiting for %d seconds." % sec)
            #time.sleep(sec)
            urlverify=False 
# Make the Pool of workers
pool = ThreadPool(4)
pool.map(runputbot, range(r,r+add))
#close the pool and wait for the work to finish 
pool.close() 
pool.join() 

print "Elapsed Time: %s" % (time.time() - start)	
		
#update upload status
i=r+add
statuspage = pywikibot.Page(site, u"User:Panoramio upload bot/status")
statuspage.text = u'%d' % i
statuspage.save(u"update upload status: %d" % i)