Scrape Twitter user tweets by chrome scrolling. Can bypass Twitter API limit and works for protected accounts you have access to.

twitter_scroll_scrape.py

import time
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By

MAX_STOPS = 5
user = "zimonitrome"

BROWSER = webdriver.Chrome()

wait = WebDriverWait(BROWSER, 500)
def wait_for_correct_current_url(desired_url):
    wait.until(
        lambda driver: driver.current_url == desired_url)

if __name__ == "__main__":
    # First login
    BROWSER.get("https://twitter.com/login")
    wait_for_correct_current_url("https://twitter.com/home")

    # Then get user
    BROWSER.get(f"https://twitter.com/{user}") # Twitter username goes here
    time.sleep(2)

    current_height = BROWSER.execute_script("return document.body.scrollHeight")

    stops = 0
    
    while True:
        BROWSER.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        time.sleep(2)
        
        new_height = BROWSER.execute_script("return document.body.scrollHeight")
        
        if new_height == current_height:
            stops += 1
            if stops == MAX_STOPS:
                break
            time.sleep(5)
            continue
        else:
            current_height = new_height
            stops = 0

        # tweets = BROWSER.find_elements_by_class_name("tweet")
        times = BROWSER.find_elements(By.TAG_NAME, 'time')
        tweets = BROWSER.find_elements(By.CSS_SELECTOR, '[data-testid="tweetText"]')

        print("Tweets:")
        with open("./saved.txt", "a+", encoding="utf-8") as file:
            for t, tweet in zip(times, tweets):
                try:
                    tt = t.get_attribute("innerText")
                    ttt = tweet.get_attribute("innerText")
                    tttt = tt + "\n" + ttt + "\n" + "-"*20 + "\n\n"
                    file.write(tttt)
                    print(tttt)
                except:
                    pass

    BROWSER.quit()