linedeco.py

__author__ = 'dodola'
#encoding: utf-8
import time
import urllib.request
import urllib
import threading
import contextlib
import queue
import string
import shutil
import re
import tempfile
import json
from urllib.error import URLError, HTTPError, ContentTooShortError
import os
from urllib.parse import (
    urlparse, urlsplit, urljoin, unwrap, quote, unquote,
    splittype, splithost, splitport, splituser, splitpasswd,
    splitattr, splitquery, splitvalue, splittag, to_bytes, urlunparse)

#http://api-linedeco.campmobile.com/service/common/getComponent.json
#{"regionCode":"cn","pageNo":2,"componentSeq":684,"language":"zh_CN"}

def myurlretrieve(url, filename=None, reporthook=None, data=None):
    _url_tempfiles=[]
    url_type, path = splittype(url)

    #user_agent = 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0)'
    #headers = {'User-Agent': user_agent}
    req = urllib.request.Request(url, data);

    with contextlib.closing(urllib.request.urlopen(req)) as fp:
        headers = fp.info()

        if url_type == "file" and not filename:
            return os.path.normpath(path), headers

        # Handle temporary file setup.
        if filename:
            tfp = open(filename, 'wb')
        else:
            tfp = tempfile.NamedTemporaryFile(delete=False)
            filename = tfp.name
            _url_tempfiles.append(filename)

        with tfp:
            result = filename, headers
            bs = 1024 * 8
            size = -1
            read = 0
            blocknum = 0
            if "content-length" in headers:
                size = int(headers["Content-Length"])

            if reporthook:
                reporthook(blocknum, 0, size)

            while True:
                block = fp.read(bs)
                if not block:
                    break
                read += len(block)
                tfp.write(block)
                blocknum += 1
                if reporthook:
                    reporthook(blocknum, len(block), size)

    if size >= 0 and read < size:
        raise ContentTooShortError(
            "retrieval incomplete: got only %i out of %i bytes"
            % (read, size), result)

    return result
def validateTitle(title):
    rstr = r"[\/\\\:\*\?\"\<\>\|]"  # '/\:*?"<>|'
    new_title = re.sub(rstr, "", title)
    return new_title

componentID={461:"linephoto",684:"editor"}

def downloadComponent(pid):
    for i in range(1,337):
        print("开始下载第"+str(i)+"页")
        values=json.dumps({"regionCode":"cn","pageNo":i,"componentSeq":pid,"language":"zh_CN"})
        user_agent = 'LINEdeco/1.0.3;android/4.2.1;resolution/xxhdpi;MI 3'
        headers = { 'User-Agent' : user_agent,"Content-Type": "application/json;charset=UTF-8"}
        req=urllib.request.Request("http://api-linedeco.campmobile.com/service/common/getComponent.json",values.encode("utf-8"),headers)
        res=urllib.request.urlopen(req)
        jsonData=json.loads(str(res.read(),"utf8"))
        print(jsonData)
        for data in jsonData["itemList"]:
            #获取下载链接
            detailvalues=json.dumps({"wallpaperSeq":data["wallpaperSeq"],"beforeSeq":3271,"regionCode":"cn","language":"zh_CN"})
            detailreq=urllib.request.Request("http://api-linedeco.campmobile.com/service/wallpaper/getDetail.json",detailvalues.encode("utf-8"),headers)
            detailres=urllib.request.urlopen(detailreq)
            detailData=json.loads(str(detailres.read(),"utf8"))
            print(detailData)
            myurlretrieve(detailData["downloadUrl"],"g:/linedeco/"+componentID[pid]+"/"+validateTitle(data["displayName"])+".jpg")
        # if(not jsonData["hasMore"]):
        #     break

def downloadTop():
    for i in range(1,9000):
        print("开始下载第"+str(i)+"页")
        values=json.dumps({"pageSize":30,"regionCode":"cn","pageNo":i,"language":"zh_CN"})
        user_agent = 'LINEdeco/1.0.3;android/4.2.1;resolution/xxhdpi;MI 3'
        headers = { 'User-Agent' : user_agent,"Content-Type": "application/json;charset=UTF-8"}
        req=urllib.request.Request("http://api-linedeco.campmobile.com/service/theme/getTopList.json",values.encode("utf-8"),headers)
        res=urllib.request.urlopen(req)
        jsonData=json.loads(str(res.read(),"utf8"))
        print(jsonData)
        for data in jsonData["themeList"]:
            #获取下载链接
            detailvalues=json.dumps({"themeSeq":data["themeSeq"],"beforeSeq":660,"regionCode":"cn","language":"zh_CN"})
            detailreq=urllib.request.Request("http://api-linedeco.campmobile.com/service/theme/getDetail.json",detailvalues.encode("utf-8"),headers)
            detailres=urllib.request.urlopen(detailreq)
            detailData=json.loads(str(detailres.read(),"utf8"))
            savePath=validateTitle(detailData["displayName"])
            rapath="g:/linedeco/icontop/"+savePath+"/"
            if not os.path.exists(rapath):
                os.makedirs(rapath)
            else:
                continue
            for icodata in detailData["iconList"]:
                filename=rapath+"/"+validateTitle(icodata["iconId"])+".png"
                print(filename)
                myurlretrieve(icodata["downloadUrl"],filename)

        # if(not jsonData["hasMore"]):
        #     break

def downloadiconTop():
    for i in range(1,9000):
        print("开始下载第"+str(i)+"页")
        values=json.dumps({"pageSize":30,"regionCode":"cn","pageNo":i,"language":"zh_CN"})
        user_agent = 'LINEdeco/1.0.3;android/4.2.1;resolution/xxhdpi;MI 3'
        headers = { 'User-Agent' : user_agent,"Content-Type": "application/json;charset=UTF-8"}
        req=urllib.request.Request("http://api-linedeco.campmobile.com/service/icon/getTopList.json",values.encode("utf-8"),headers)
        res=urllib.request.urlopen(req)
        jsonData=json.loads(str(res.read(),"utf8"))
        print(jsonData)
        for data in jsonData["iconList"]:
            #获取下载链接
            # detailvalues=json.dumps({"iconSeq":data["iconSeq"],"beforeSeq":660,"regionCode":"cn","language":"zh_CN"})
            # detailreq=urllib.request.Request("http://api-linedeco.campmobile.com/service/theme/getDetail.json",detailvalues.encode("utf-8"),headers)
            # detailres=urllib.request.urlopen(detailreq)
            # detailData=json.loads(str(detailres.read(),"utf8"))
            rapath="g:/linedeco/icontops/"
            if not os.path.exists(rapath):
                os.makedirs(rapath)

            filename=rapath+validateTitle(data["iconId"]+str(data["iconSeq"]))+".png"
            print(filename)
            myurlretrieve(data["path"],filename)

downloadiconTop()