Parth-Vader

from twisted.web.server import Site, NOT_DONE_YET
from twisted.web.resource import Resource
from twisted.internet import reactor, endpoints

counter = 0
import time

class Home(Resource):
    isLeaf = True
    

Parth-Vader

from twisted.web.server import Site, NOT_DONE_YET
from twisted.web.resource import Resource
from twisted.internet import reactor, endpoints
from twisted.internet.task import deferLater

def getarg(request, name, default=None, type=None):
    if name in request.args:
        value = request.args[name][0]
        if type is not None:
            value = type(value)

Parth-Vader

from twisted.internet import reactor, endpoints
from twisted.web.server import Site
from twisted.web.static import File
from twisted.web.resource import Resource
import time
import os

class Home(Resource):
    isLeaf = False
    

Parth-Vader

# To be placed inside scrapy folder
git fetch;
if git status | grep -q "Your branch is behind 'origin/master'"; then  
    
    git merge;
    
    cd ../scrapy-bench; # Change folder scrapy-bench
    
    tox -- bookworm;
    tox -- xpathbench;

Parth-Vader

#!/usr/bin/env python
# -*- coding: utf-8 -*-
import re
import datetime

from six.moves.urllib.parse import urlparse
import click
import scrapy
from scrapy.http import Request, HtmlResponse
from scrapy.linkextractors import LinkExtractor

Parth-Vader

from scrapy.crawler import CrawlerProcess
from scrapy.crawler import CrawlerRunner
from twisted.internet import reactor
from scrapy.utils.project import get_project_settings
from scrapy.utils.log import configure_logging
process = CrawlerProcess(get_project_settings())

# 'followall' is the name of one of the spiders of the project.
#process.crawl('followall')