gistfile1.txt

import pandas as pd
from selenium import webdriver
from bs4 import BeautifulSoup as soup
import time
import os

url = 'http://finra-markets.morningstar.com/BondCenter/Results.jsp'
driver = webdriver.Chrome()
driver.get(url)
time.sleep(5)
driver.find_element_by_xpath('//*[@id="ms-agreement"]/input').click()
time.sleep(2)
driver.find_element_by_xpath('/html/body/div[2]/div[2]/div[2]/div[4]/form/a').click()
time.sleep(2)
driver.find_element_by_xpath('//*[@id="ms-finra-advanced-search-form"]/div[2]/input[2]').click()

lines = []
next_button = True

while next_button:
    try:
        time.sleep(2)
        html = driver.page_source
        page_soup = soup(html, "html.parser")
        containers = page_soup.find_all("div", {"class": "rtq-grid-row"})
        for container in containers[1:]:
            sub_container = container.find_all('div', {'class': 'rtq-grid-cell'})
            data = [item.text for item in sub_container]
            lines.append(data[1:])
        try:
            driver.find_element_by_xpath('//*[@id="ms-finra-search-results"]/div/div[3]/div[2]/div[2]/div/a[2]').click()
        except:
            next_button = False
    except:
        break

headers = ['Issuer Name', 'Symbol', 'Callable', 'Sub-Product Type', 'Coupon', 'Maturity', "Moody's®", 'S&P', 'Price', 'Yield']
df = pd.DataFrame(lines, columns=headers)
df.to_excel('data.xlsx')
os.startfile('data.xlsx')