Bad Ideal

pycontw2017_talk_bad.py

#-*- coding: utf-8 -*-
#!/usr/bin/python

import sys
import re
import logging
import time
import os
import ConfigParser
import codecs
from ConfigParser import SafeConfigParser
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.wait import WebDriverWait
from selenium.common.exceptions import NoSuchElementException
from pyvirtualdisplay import Display


# solve:UnicodeDecodeError: 'ascii' codec can't decode byte 0x?? in position 1: ordinal not in range(128)
reload(sys)  
sys.setdefaultencoding('utf8')

logger = logging.getLogger(__name__)
#logger.propagate = False

if os.name == "nt": 
    INI = "\\SETTING.ini"
else:
    INI = "/SETTING.ini" 

candidates = ['SETTING.ini']
section_VER = "VER"
section_value_NEW = "NEW"
section_value_OLD = "OLD"
section_candidate = [section_VER]
section_value_candidate = [section_value_NEW, section_value_OLD]

path = os.getcwd()
FAILCount = 0
content_ini = ""

def pkill(process_name):
    try:
        killed = os.system('taskkill /F /IM ' + process_name)
        time.sleep(1)
    except Exception, e:
        killed = 0
    logger.info("kill return code = " + str(killed))

def filename(ext=".xlsx", paths=path):
    for fname in os.listdir(paths):
        extension = os.path.splitext(fname)
        if extension[1] == ext:
            logger.info(os.path.join(paths, fname))
            return fname
    return None

def ini_file(filename, inifile=INI):
    """
    Update SETTING.ini.
    Here can check the DRP latest 2 files
    """
    # check SETTING.INI has exist or not.
    filename_old = ""
    parser = SafeConfigParser()
    found = parser.read(inifile)
    missing = set(candidates) - set(found)
    
    #logger.info('Found config files: '+str(sorted(found)))
    #logger.info('Missing Files: '+str(sorted(missing)))

    if found == [] or missing == [inifile]:
        logger.info('ini file is Empty!')
        with open(inifile, 'wb') as fp:
            fp.write("[VER]\n")
            fp.write(section_value_NEW+" = \n")
            fp.write(section_value_OLD+" = \n")
            fp.write("[VERWEB]\n")
            fp.write(section_value_NEW+" = \n")
            fp.write(section_value_OLD+" = ")
        fp.close()
        # write new value
        config = ConfigParser.ConfigParser()
        config.optionxform = str
        config.read(inifile)
        config.set(section_VER, section_value_NEW, filename) #要修改Key 的 Value
        config.write(open(inifile, 'wb'))
        return 1
    else:
        logger.info('Exist '+str(inifile))
        for section in section_candidate:
            if parser.has_section(section):
                logger.info('%s section exists: %s' % (section, parser.has_section(section)))
                for candidate in section_value_candidate:
                    logger.info('%s.%-12s  : %s' % (section, candidate, parser.has_option(section, candidate)))
                    config = ConfigParser.ConfigParser()
                    config.optionxform = str
                    config.read(inifile)
                    if str(candidate).upper() == section_value_NEW: #//Section value = new
                        if str(config.get(section, section_value_NEW)) != filename and filename != None:
                            filename_old = config.get(section, section_value_NEW)
                            logger.info("Modify section:"+str(section)+" "+str(section_value_NEW)+"'s value")
                            config.set(section, section_value_NEW, filename) #要修改Key 的 Value
                            config.write(open(inifile, 'wb'))
                        else:
                            return 254      
                    if str(candidate).upper() == section_value_OLD and filename_old != "":
                        logger.info("Modify section:"+str(section)+" "+str(section_value_OLD)+"'s value")
                        config.set(section, section_value_OLD, filename_old) #要修改Key 的 Value
                        config.write(open(inifile, 'wb'))
            else:
                print 'missing section, %s section exists: %s' % (section, parser.has_section(section))
                return 2
        return 0

def createini(inifile):
    with open(inifile, 'wb') as fp:
        fp.write("[VER]\n")
        fp.write(section_value_NEW+" = \n")
        fp.write(section_value_OLD+" = \n")
        fp.write("[VERWEB]\n")
        fp.write(section_value_NEW+" = \n")
        fp.write(section_value_OLD+" = ")
    fp.close()

def readini(inifile, mode="OTHER"):
    """
    return the ini file name
    """
    section_VER = "VERWEB"
    section_value_NEW = "NEW"
    section_value_OLD = "OLD"
    section_candidate = [section_VER]
    section_value_candidate = [section_value_NEW, section_value_OLD]

    config = ConfigParser.ConfigParser()
    config.optionxform = str
    config.read(inifile)
    
    if mode.upper() == "NEW":
        logger.info(str(config.get(section_VER, section_value_NEW)))
        return str(config.get(section_VER, section_value_NEW))
    elif mode.upper() == "OLD":
        logger.info(str(config.get(section_VER, section_value_OLD)))
        return str(config.get(section_VER, section_value_OLD))
    else:
        return 255

def updateini(inifile, filename, mode="OTHER"):
    """
    return the ini file name
    """
    section_VER = "VERWEB"
    section_value_NEW = "NEW"
    section_value_OLD = "OLD"
    section_candidate = [section_VER]
    section_value_candidate = [section_value_NEW, section_value_OLD]

    config = ConfigParser.ConfigParser()
    config.optionxform = str
    config.read(inifile)
    
    if mode.upper() == "NEW":
        config.set(section_VER, section_value_NEW, filename) #要修改Key 的 Value
        config.write(open(inifile, 'wb'))
        logger.info(str(config.get(section_VER, section_value_NEW)))
        return str(config.get(section_VER, section_value_NEW))
    elif mode.upper() == "OLD":
        config.set(section_VER, section_value_OLD, filename) #要修改Key 的 Value
        config.write(open(inifile, 'wb'))
        logger.info(str(config.get(section_VER, section_value_OLD)))
        return str(config.get(section_VER, section_value_OLD))
    else:
        return 255

def web_kit(link, ini_path=path, download_path=path, count=0):
    c = count
    # hidden the display and using vritual screen 
    if os.name != "nt": 
        display = Display(visible=0, size=(1024, 786))
        display.start()
    if os.name == "nt": 
        chromedriver = path + "\chromedriver\chromedriver_win32.exe"
        proxy = 'proxy8.local:80'
    else:
        chromedriver = path + "/chromedriver/chromedriver"
        proxy = '127.0.0.1:3128'
    logger.info(chromedriver)
    
    # setting Chrome option
    os.environ["webdriver.chrome.driver"] = chromedriver
    prefs = {"download.default_directory" : download_path}
    chrome_options = Options()
    chrome_options.add_experimental_option("prefs", prefs)
    chrome_options.add_argument('--proxy-server=http://%s' % proxy)
    driver = webdriver.Chrome(chromedriver, chrome_options=chrome_options)
    wait = WebDriverWait(driver, 100)

    if not os.path.exists(ini_path+INI):
        logger.info("Create SETTING.INI file.")
        createini(inifile=ini_path+INI)
    
    try:
        if c == 0:
            logger.info("Link to Nimbus.")
            driver.get("{}")
            content = wait.until(lambda driver: driver.find_element_by_id("accountname"),"link to nimbus fail")

            logger.info("Input the user account and password.")
            wait.until(lambda driver: driver.find_element_by_id("accountname"), "FAIL to find account")
            driver.find_element_by_id("accountname").send_keys("{}")
            driver.find_element_by_id("accountpassword").send_keys("{}")
            driver.find_element_by_id("continueFieldbutton").click()
        time.sleep(5)
        driver.get(link)
        time.sleep(5)
        
        # 確認是否有成功連接到BM頁面
        try:
            #c += 1
            while bool(driver.find_element_by_id("project-list")):
                logger.info("Link to BM again " + link + ", time = " + str(c) + ".")
                c += 1
                driver.get(link)
                time.sleep(5)
        except Exception, e:
            logger.info("Link to BM Success.")

        logger.info("Waitting for Logged status.")
        content = wait.until(lambda driver: driver.find_element_by_xpath
                            ("(//div[@id='topnav']/ul[@id='user-box'])"), 
                            "Find the element user_box fail")
        time.sleep(10)
        Request_count = 0
        Check_Leave = 0
        while 1:
            content = driver.find_element_by_xpath("//div[@id='topnav']/ul[@id='user-box']").text
            content = content.decode("utf-8").split("\n")
            logger.info(repr(content[1]) + ".")

            # 帳號列出現怪怪字元
            if content[1] == u"What's New": return 255
           
            # BM版本號
            content = driver.find_element_by_xpath("//div[@id='build-controller-container'] \
                            /div[@id='sheet-actions-container']/ul[@id='sheet-tabs']").text
            logger.info(repr(content) + ".")
            
            if content == "": 
                logger.info("Version Status Null, Retry.")
            elif content.decode("utf-8").split("\n")[0] == "Current":
                logger.info("Version Number Empty, Rerty.")
            else:
                Check_Leave = 0
                # Switch to right now version.
                if Request_count == 0:
                    # 先確認版本是否有變
                    """
                    file_now = filename(paths=ini_path)
                    if file_now != None:
                        file_now = str(file_now).split("(")
                        file_now = str(file_now[2]).split(")")
                        file_now = str(file_now[0]).replace("/", "")

                        content = content.decode("utf-8").split("\n")
                        content = str(content[0]).split("-")
                        content = str(content[0]).split("(")
                        content = str(content[1]).split(")")
                        content = str(content[0]).replace("/", "")
                    """
                    file_now = readini(inifile=ini_path+INI, mode="NEW")
                    if file_now != "":
                        content = content.decode("utf-8").split("\n")
                        content = str(content[0])
                        logger.info("Version Now:" + str(file_now))
                        logger.info("Nimbus:" + str(content))
                        # update ini
                        updateini(inifile=ini_path+INI, filename=content, mode="NEW")
                        updateini(inifile=ini_path+INI, filename=file_now, mode="OLD")
                        # check version. and replace the version not digital
                        #file_now = re.sub(r'\D', "", file_now[0])
                        #content = re.sub(r'\D', "", content[0])
                        #if str(content[0]).upper() == str(file_now[0]).upper():
                        if str(file_now).upper() == str(content).upper():
                            logger.info("BM Version not change.")
                            return 254
                    if file_now == "":
                        content = content.decode("utf-8").split("\n")
                        content = str(content[0])
                        logger.info("File now = '', write: " + content)
                        updateini(inifile=ini_path+INI, filename=content, mode="NEW")

                    # Switch to History Version
                    logger.info("Switch to History Version.")
                    el = driver.find_element_by_id("tab-revisions")
                    webdriver.ActionChains(driver).move_to_element(el).click(el).perform()
                elif Request_count > 0:
                    '''
                    content = content.decode("utf-8").split("\n")
                    content = str(content[0]).split("-")
                    content = str(content[0]).split("(")
                    content = str(content[1]).split(")")
                    '''
                    content = content.decode("utf-8").split("\n")
                    logger.info("Version= " + repr(content[0]) + ".")
                    if Request_count == 2: break
                Request_count = Request_count + 1
            
            time.sleep(10)
            if Check_Leave == 10: return 255
            Check_Leave = Check_Leave + 1
        wait_button = WebDriverWait(driver, 40)

        # Delete old excel NOT remove in autodownload
        #old_f = filename(paths=download_path)
        #if old_f != None: os.remove(os.path.join(download_path, old_f))
        #time.sleep(1)

        # 確定export是否存在
        logger.info("Check The Export Button.")
        content = wait.until(lambda driver: driver.find_element_by_xpath
                            ("//div[@id='sheet-actions-container'] \
                            /ul[@id='sheet-actions']"), "Find export button fail.")
        try:
            """
            Logical: 
            Step1. click "SHARE", and while to show the "export" then click. if not shoe retry 3 times.
            """
            found = False
            count = 0
            while not found:
                try:
                    logger.info("Click the SHARE and Click.")
                    driver.find_element_by_xpath("//div[@id='sheet-actions-container'] \
                                                /ul[@id='sheet-actions']/li[3]").click()
                    time.sleep(3)
                    link = driver.find_element_by_link_text("Export")
                    driver.find_element_by_link_text("Export").click()
                    found = True
                    time.sleep(6)
                except NoSuchElementException:
                    logger.info("Retry.")
                    time.sleep(2)
                count += 1
                if count == 3: raise ValueError(u"Click SHARE Button FAIL.")

            while 1:
                logger.info("Check Download Button.")
                content = wait_button.until(lambda driver: driver.find_element_by_xpath(
                                        "//button[@class='btn ng-binding btn-primary']"),
                                        "Click Download and Export Button PASS.")
                if bool(content) == True:
                    time.sleep(3)
                    driver.find_element_by_xpath(
                                "//button[@class='btn ng-binding btn-primary']").click()
                    logger.info("Retry Click Export Button.")
                time.sleep(1)
        except ValueError, e:
            raise e
        except Exception, e:
            #logger.info(e)
            #logger.info(str(type(e.args)))
            logger.info("Continue to start Download.")
        
        # click the download and continue the final download
        logger.info("Click the Download Button.")
        content = wait.until(lambda driver: driver.find_element_by_xpath(
                    "//button[@class='btn btn-primary btn-lg btn-block']"),
                    "Find download button fail.")
        driver.find_element_by_xpath("//button[@class='btn btn-primary btn-lg btn-block']").click()
        time.sleep(30)
        download_count = 0
        while filename(paths=download_path) == None:
            if download_count >= 10: 
                raise KeyError(u"Download Fail.")
            logger.info("Waiting Download...")
            time.sleep(10)
        logger.info("Download Success.") 
        _ret = 0  
    except Exception, e:
        logger.info("FAILED Messages:"+ str(e) + str(e.args))
        _ret = 255
    finally:
        new_f = filename(paths=download_path)
        ret_init = ini_file(new_f, inifile=ini_path+INI)
        if os.name != "nt": 
            display.stop()
            driver.quit()
        else:
            driver.close()
        logger.info("FINISH Nimbus Download Process.")
    
    if ret_init == 0:
        return _ret
    else:
        return ret_init
    
if __name__ == '__main__':  
    """
    Return Code Description 
    description:
        ret = 1, 2 ini_file()'s error
        ret = 255 web_kit()'s error
    """
    #logging.basicConfig(filename='craw_Nimbus_DRP.log', level=logging.INFO, format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p')
    logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p')

    f = "C:\Users\yichieh_chen\Desktop\OneDrive\SourceCode\python\Nimbus_DRP_20150630\MLBDRP\Kirin\SETTING.ini"
    Rn = readini(inifile=f, mode="NEW")
    Ro = readini(inifile=f, mode="OLD")
    print Rn, Ro

    R = updateini(inifile=f, filename=Rn, mode="OLD")
    print R
    sys.exit()