selenium_cafe_migrations: Migration from Tabelog to Google Maps

.selenium_googlemaps.py

from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as ec
from selenium.webdriver.common.by import By

import os
import sys
import time


def main(driver_path):
    # Open Chrome with google account
    URL = 'https://www.google.com/accounts?hl=ja-JP'
    GOOGLE_USERNAME = os.environ.get('GOOGLE_USERNAME', None)
    GOOGLE_PASSWORD = os.environ.get('GOOGLE_PASSWORD', None)

    driver = webdriver.Chrome(executable_path=driver_path)
    driver.get(URL)
    WAIT_TIME = 30

    if (GOOGLE_USERNAME is not None) and (GOOGLE_PASSWORD is not None):
        # Get X-PATH for Username
        xpath_username = '//*[@id="identifierNext"]'
        WebDriverWait(driver, WAIT_TIME).until(ec.presence_of_element_located((By.XPATH, xpath_username)))
        # Enter username(email address)
        driver.find_element_by_name("identifier").send_keys(GOOGLE_USERNAME)
        driver.find_element_by_xpath(xpath_username).click()

        # Get X-Path for Password
        xpath_password = '//*[@id="passwordNext"]'
        WebDriverWait(driver, WAIT_TIME).until(ec.presence_of_element_located((By.XPATH, xpath_password)))
        # Enter password
        driver.find_element_by_name("password").send_keys(GOOGLE_PASSWORD)

        time.sleep(1)
        driver.find_element_by_xpath(xpath_password).click()
    else:
        driver.quit()
        print('>>> Error, Password for Google account not set.')
        print('Set environmental variables with commands below, before running programs.')
        print()
        print('\texport GOOGLE_USERNAME=\'YOUR_GOOGLE_USERNAME\'')
        print('\texport GOOGLE_PASSWORD=\'YOUR_GOOGLE_PASSWORD\'')
        print()
        sys.exit(1)


if __name__ == "__main__":
    WEB_DRIVER_PATH = os.environ.get('WEB_DRIVER_PATH', None)

    if WEB_DRIVER_PATH is None:
        print('>>> Error, PATH of Chrome webdriver not set.')
        print('Set environmental variables with commands below, before running programs.')
        print()
        print('\texport WEB_DRIVER_PATH=\'PATH_TO_YOUR_WEB_DRIVER_BINARY\'')
        print()
        sys.exit(1)
    else:
        print('>>> Starting to program !!')
        main(driver_path=WEB_DRIVER_PATH)

.selenium_tabelog.py

import csv
import pyperclip
import os
import sys

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException

FACEBOOK_ID = os.environ.get('FACEBOOK_ID', None)
FACEBOOK_PASSWORD = os.environ.get('FACEBOOK_PASSWORD', None)
CHROME_DRIVER_PATH = os.environ.get('CHROME_DRIVER_PATH', None)

if (FACEBOOK_ID == None) or (FACEBOOK_PASSWORD == None) or (CHROME_DRIVER_PATH == None):
    print(">>> Error, missing environmental variable config.")
    sys.exit(1)

driver = webdriver.Chrome(executable_path=CHROME_DRIVER_PATH)
driver.get('https://tabelog.com/')
# Login with Facebook account
driver.find_element_by_class_name('js-open-login-modal').click()
driver.find_element_by_class_name('p-login-panel__btn--facebook').click()
driver.find_element_by_id('email').send_keys(FACEBOOK_ID)
driver.find_element_by_id('pass').send_keys(FACEBOOK_PASSWORD)
driver.find_element_by_id('loginbutton').click()

driver.find_element_by_class_name('p-user-menu__target--hozon').click()

restaurants = []
while True:
    WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CLASS_NAME, 'js-copy-restaurant-info-to-clipboard')))
    clipboard_buttons = driver.find_elements_by_class_name('js-copy-restaurant-info-to-clipboard')
    for button in clipboard_buttons:
        button.click()
        data = pyperclip.paste().split('\n')
        if len(data) < 4:
            data.insert(1, '')
        map_url = 'https://www.google.co.jp/maps/search/' + ' '.join(data[0:3])
        data.append(map_url)
        restaurants.append(data)
    try:
        driver.find_element_by_class_name('c-pagination__arrow--next').click()
    except NoSuchElementException:
        break

with open('result.csv', 'wt', encoding='utf_8_sig') as f:
    writer = csv.writer(f, quoting=csv.QUOTE_ALL)
    writer.writerows(restaurants)

print('>>> Done.')
driver.quit()

requirements.txt

pyperclip==1.8.0
selenium==3.141.0
urllib3==1.26.5

~README.md

Selenium Cafe Migrations

Overviews

TBA

Environments

All the Python programs in this sub-directory are expected to run correctly under the environments below:

Components	Version
OS	10.15.4 (Catalina)
Python	3.6.4 with pyenv
Selenium	3.141.0
Google Chrome	81.0.4044.122 Official Build (64-bit)

---

Installation & Setup

Installation

Firstly it's required to install selenium package.
If you use virtualenv, create your private virtual environment first, and install the selenium pacakge with pip.

# If you use virtual env: run `source ENV_NAME/bin/activate` first
pip install selenium

Setup webdriver

Secondary, we need to install Chrome webdriver to your enviroment, where the program runs.
We can install the packages with package manager like brew, or download from the sites https://sites.google.com/a/chromium.org/chromedriver/downloads .

brew cask install chromedriver

Also, since the Python scripts will try to fetch environmental variables in the codes,
we need to set environmental variables of selenium chrome driver.
For example of bash, follow the commands below:

export SELENIUM_PATH='PATH_TO_YOUR_CHROME_DRIVER_BINARY'
# For example with your parameters
# export SELENIUM_PATH='$HOME/chromedriver'