This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# main.py: | |
from project.spiders.log_test import TestSpider as EstiloMASpider | |
from scrapy.xlib.pydispatch import dispatcher | |
from scrapy.crawler import Crawler | |
from twisted.internet import reactor | |
from scrapy.utils.project import get_project_settings | |
from scrapy import log, signals |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from PyQt4.QtCore import QUrl | |
from PyQt4.QtGui import QApplication | |
from PyQt4.QtWebKit import QWebPage | |
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor | |
from scrapy.spider import BaseSpider | |
from scrapy.http import HtmlResponse | |
class Render(QWebPage): | |
def __init__(self, url): |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
class PartialResponse(object): | |
""" Downloader middleware to only return the first n bytes | |
""" | |
def process_response(self, request, response, spider): | |
max_size = getattr(spider, 'response_max_size', 0) | |
if max_size and len(response.body) > max_size: | |
h = response.headers.copy() | |
h['Content-Length'] = max_size | |
response = response.replace( | |
body=response.body.encode('utf-8')[:max_size], |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<?php | |
/** | |
* generic debug printer | |
* | |
* Because I didn't like having to pass two arguments to a debug printer, namely the | |
* evaluated and un-evaluated expressions, like: $baker->bread and "baker.bread", i.e., | |
* I only wanted to pass the un-evaluated string and let the print routine do the | |
* evaluating. This script does that proceduraly, i.e., not in a function, so expression | |
* scope is not changed. | |
* |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/python | |
# Google Places Search | |
# | |
# Use the Google Places API to text search for the supplied keywords and output | |
# the first result to standard out. | |
import sys | |
import json | |
import argparse |
NewerOlder