Rise of Skywalker rotten score

crawlerRottenTomatoesRiseOfSkywalker.py

from selenium import webdriver
from scrapy import Spider
import time

#url = 'https://www.rottentomatoes.com/m/empire_strikes_back/reviews'
url = 'https://www.rottentomatoes.com/m/star_wars_the_rise_of_skywalker/reviews'

class SkywalkaSpider(Spider):
    name = 'Skywalka'
    allowed_domains = [url]
    start_urls = [url+'?type=user']

    def __init__(self):
        self.driver = webdriver.Firefox()
        #full half empty
        self.results = {"votes":{(0,0,5): 0,
                                 (0,1,4): 0,
                                 (1,0,4): 0,
                                 (1,1,3): 0,
                                 (2,0,3): 0,
                                 (2,1,2): 0,
                                 (3,0,2): 0,
                                 (3,1,1): 0,
                                 (4,0,1): 0,
                                 (4,1,0): 0,
                                 (5,0,0): 0,
                                 },
                        "voters": 0}

    def parse(self, response):
        self.driver.get(SkywalkaSpider.start_urls[0])

        forceEnd = False

        while True:
            try:
                time.sleep(0.35)
                votes = self.driver.find_elements_by_class_name('star-display')

                for vote in votes:
                    empty  = len(vote.find_elements_by_class_name('star-display__empty'))
                    half   = len(vote.find_elements_by_class_name('star-display__half'))
                    filled = len(vote.find_elements_by_class_name('star-display__filled'))
                    key = (filled, half, empty)
                    self.results["votes"][key] += 1
                    self.results["voters"] += 1
                if forceEnd:
                    break
            except Exception as ex:
                print("Exception", ex)
                pass
            try:
                next = self.driver.find_element_by_class_name('prev-next-paging__button-right')
                next.click()
            except:
                break

        import pickle
        with open("dump.pickle", "wb") as file:
            pickle.dump(self.results, file)

        self.driver.close()




if __name__ == "__main__":
    from scrapy.crawler import CrawlerProcess


    process = CrawlerProcess()
    process.crawl(SkywalkaSpider)
    process.start()

    import pickle
    with open("dump.pickle", "rb") as file:
        results = pickle.load(file)

    #My previous result
    #results = {'votes': {(0, 0, 5): 0, (0, 1, 4): 679, (1, 0, 4): 413, (1, 1, 3): 237, (2, 0, 3): 381, (2, 1, 2): 208, (3, 0, 2): 378, (3, 1, 1): 451, (4, 0, 1): 1222, (4, 1, 0): 761, (5, 0, 0): 4016}, 'voters': 8746}

    x = list(range(11)) # 0 to 10
    y = list(results["votes"].values())
    import matplotlib.pyplot as plt
    import statistics

    scores = []
    for (voters, score) in zip(x, y):
        scores += [voters*score/sum(y)]

    totalScore = sum(scores)
    stdDev = statistics.stdev(scores)

    plt.bar(x, y)
    plt.title(url)
    plt.ylabel("# of votes")
    plt.xlabel("Score (votes=%.0f, avg=%.2f, stddev=%.2f)" % (results["voters"], totalScore, stdDev))
    plt.show()
    print()