Automate Keyword Profiling with AudienceView - Requires and existing Crosstab Report

hitwise-audienceview-keyword-research.py

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import Select
import time
import pandas as pd
import os

keyword = [
    'healthy eating',
    'healthy recipes',
    'keto diet',
]

download_folder = r"C:\Users\paul.Shapiro\AudienceView"
output_file = r"C:\Users\paul.Shapiro\AudienceView\keyword_profiles.csv"
usern = ""
passw = ""

def get_keyword_profile(keyword):
    print("Starting process for " + keyword)
    time.sleep(10)
    print("Waiting 10 seconds")
    driver.find_element_by_css_selector('#new-crosstab > div.ng-scope > div > super-ribbon > div > div > div > div.actions-container > div > div:nth-child(1) > button > div > ng-include > svg > path').click()
    driver.find_element_by_css_selector('#editor > crosstab-drop-area:nth-child(3) > div > div.crosstab-drop-area.dropAreaExpanded > ul > div > div > li > div.adr-notification-seg-element-border.ng-scope > div > img').click()
    print("Click edit button")
    time.sleep(2)
    driver.find_element_by_css_selector('#new-crosstab > string-lightbox > div > div > div.bottomFrame > div > div > div > p').click()
    search = driver.find_element_by_id("strsearchinput")
    print("entering keyword: " + keyword)
    search.send_keys(keyword)
    print("wait 10 seconds")
    time.sleep(10)
    print("Find and click exact match keyword: " + keyword)
    driver.find_element_by_xpath("//*[text()='" + keyword + "']").click()
    driver.maximize_window()
    driver.find_element_by_css_selector('#new-crosstab > string-lightbox > div > div > div.optionsFrame.container-fluid > div > span > button').click()
    print("Generate new report")
    driver.find_element_by_css_selector('#new-crosstab > div.ng-scope > div > super-ribbon > div > div > div > div.actions-container > div > div:nth-child(2) > button > div > ng-include > svg').click()
    print("Wait 60 seconds")
    time.sleep(60)
    print("Iniate export")
    try:
        driver.find_element_by_css_selector('#new-crosstab > div.ng-scope > div > super-ribbon > div > div > div > div.actions-container > div > div:nth-child(5) > button > div > ng-include > svg > path').click()
    except e:
        print("Waiting another 60 seconds")
        time.sleep(60)
        driver.find_element_by_css_selector('#new-crosstab > div.ng-scope > div > super-ribbon > div > div > div > div.actions-container > div > div:nth-child(5) > button > div > ng-include > svg > path').click()
    print("Wait 2 seconds")
    time.sleep(2)
    print("Find select box")
    select = Select(driver.find_element_by_css_selector('body > div.modal.fade.ng-isolate-scope.ssda-modal.in > div > div > div.modal-body.ng-scope > form > div:nth-child(1) > select'))
    print("Select CSV")          
    select.select_by_value('1')
    print("Click export button")
    driver.find_element_by_css_selector('body > div.modal.fade.ng-isolate-scope.ssda-modal.in > div > div > div.modal-footer.ng-scope > button.ssda-button.ssda-button-orange').click()
    print("Wait for text to appear")
    print("Wait 20 seconds")
    time.sleep(20)
    print("Downloading")
    driver.find_element_by_xpath('//a[@download]').click()
    print("Waiting 2 seconds")
    time.sleep(2)
    print("Clicking cancel")
    driver.find_element_by_css_selector('body > div.modal.fade.ng-isolate-scope.ssda-modal.in > div > div > div.modal-footer.ng-scope > button.ssda-button.ssda-button-default').click()
    print("Waiting 10 seconds")
    time.sleep(10)
    print("Download complete for " + keyword)

chrome_options = webdriver.ChromeOptions()
prefs = {'download.default_directory' : download_folder }
# chrome_options.add_argument("--headless")
chrome_options.add_experimental_option('prefs', prefs)

driver = webdriver.Chrome(chrome_options=chrome_options)
driver.get("https://audience-view.com/")
print("Launching login page.")
username = driver.find_element_by_name("emailID")
username.clear()
username.send_keys(usern)
print("Inserting username")
password = driver.find_element_by_name("password")
password.clear()
password.send_keys(passw)
print("Inserting password")
password.send_keys(Keys.RETURN)
print("Logging in")
print("Waiting 2 seconds")
time.sleep(2)
print("Going to crosstab")
driver.get("https://audience-view.com/#/crosstab/32617?tab=0")

for x in keyword:
    get_keyword_profile(x)

driver.close()

last_column = 0
df = pd.read_csv(download_folder + "/crosstab.csv", usecols = [*range(0, 4)])
for file in os.listdir("./"):
    if file.endswith(".csv"):
        if not file.endswith("crosstab.csv"):
            if last_column > 0:
                df = merge
            new = pd.read_csv(file, usecols=[3])
            merge = pd.concat([df, new], axis=1)
            last_column += 1
merge.to_csv(output_file)