scraping.py

import argparse
import time
from selenium import webdriver

URL = 'https://www.japannetbank.co.jp/service/payment/web_all/csv_download.html'
DOWNLOAD_PATH = '/PATH/TO/DOWNLOAD'

def enable_download_in_headless_chrome(driver, download_dir):
    driver.command_executor._commands["send_command"] = ("POST", '/session/$sessionId/chromium/send_command')
    params = {'cmd': 'Page.setDownloadBehavior', 'params': {'behavior': 'allow', 'downloadPath': download_dir}}
    driver.execute("send_command", params)


def main():
    argp = argparse.ArgumentParser()
    argp.add_argument('--headless', action='store', default=False)

    args = argp.parse_args()

    options = webdriver.ChromeOptions()

    if args.headless:
        options.add_argument('--headless')
        options.add_argument('--no-sandbox')
        options.add_argument('--disable-gpu')

    # set download path
    prefs = {"download.default_directory": DOWNLOAD_PATH, "download.prompt_for_download": False}
    options.add_experimental_option("prefs", prefs)
    driver = webdriver.Chrome(chrome_options=options)

    enable_download_in_headless_chrome(driver, DOWNLOAD_PATH)

    driver.get(URL)
    driver.execute_script("document.getElementsByTagName('a')[0].removeAttribute('target')")
    driver.execute_script("document.getElementsByTagName('a')[0].click()")
    time.sleep(3)

    driver.close()
    driver.quit()


main()