This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from scrapy import project, signals | |
from scrapy.conf import settings | |
from scrapy.crawler import CrawlerProcess | |
from scrapy.xlib.pydispatch import dispatcher | |
from multiprocessing.queues import Queue | |
import multiprocessing | |
class CrawlerWorker(multiprocessing.Process): | |
def __init__(self, spider, result_queue): |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
# store the current dir | |
CUR_DIR=$(pwd) | |
# Let the person running the script know what's going on. | |
echo "\n\033[1mPulling in latest changes for all repositories...\033[0m\n" | |
# Find all git repositories and update it to the master latest revision | |
for i in $(find . -name ".git" | cut -c 3-); do |