Skip to content

Instantly share code, notes, and snippets.

from scrapy import project, signals
from scrapy.conf import settings
from scrapy.crawler import CrawlerProcess
from scrapy.xlib.pydispatch import dispatcher
from multiprocessing.queues import Queue
import multiprocessing
class CrawlerWorker(multiprocessing.Process):
def __init__(self, spider, result_queue):
#!/bin/bash
# store the current dir
CUR_DIR=$(pwd)
# Let the person running the script know what's going on.
echo "\n\033[1mPulling in latest changes for all repositories...\033[0m\n"
# Find all git repositories and update it to the master latest revision
for i in $(find . -name ".git" | cut -c 3-); do