Scrape Youtube Trending Videos

scrape_youtube_trending_videos.py

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from bs4 import BeautifulSoup

BASE_URL = 'http://www.youtube.com'

path_to_chromedriver = '/Users/nebula/Projects/test-scrape-youtube-explore/chromedriver' # change path as needed
browser = webdriver.Chrome(executable_path = path_to_chromedriver)

url = 'https://www.youtube.com/feed/trending'
browser.get(url)
browser.implicitly_wait(30)

body = browser.find_element(by=By.TAG_NAME, value='body') #This is a dummy element
soup = BeautifulSoup(browser.page_source, features="html.parser")

all_videos = soup.find_all('ytd-video-renderer')
all_videos_scraped = []
for video in all_videos:
    video_a = video.find('a', 'yt-simple-endpoint style-scope ytd-video-renderer')
    video_link = video_a['href']
    video_title = video_a['title']
    video_channel_data = video.find('yt-formatted-string', 'style-scope ytd-channel-name complex-string')
    video_channel_a = video_channel_data.find('a', 'yt-simple-endpoint style-scope yt-formatted-string')
    video_channel_name = video_channel_a.text
    video_channel_link = video_channel_a['href']

    video_views = video.find_all('span', 'style-scope ytd-video-meta-block')[0].text

    all_videos_scraped.append({
        'title': video_title,
        'link':  BASE_URL + video_link,
        'channel': {
            'name': video_channel_name,
            'link': BASE_URL + video_channel_link
        },
        'views': video_views
    })

print(all_videos_scraped)

browser.close()