Skip to content

Instantly share code, notes, and snippets.

@seagatesoft
seagatesoft / middlewares.py
Last active January 3, 2024 20:58
An example of RotateUserAgentMiddleware
from random import choice
from scrapy import signals
from scrapy.exceptions import NotConfigured
class RotateUserAgentMiddleware(object):
"""Rotate user-agent for each request."""
def __init__(self, user_agents):
self.enabled = False
self.user_agents = user_agents