Skip to content

Instantly share code, notes, and snippets.

@8maki
8maki / file0.py
Created March 21, 2014 10:46
ScrapyでItem Pipelineにsettingsを渡す方法 ref: http://qiita.com/8maki/items/7d994fb730d2a6089e45
class MySQLStorePipeline(object):
@classmethod
def from_settings(cls, settings):
return cls(settings.get('DB_SETTING'))
def __init__(self, db_settings):
db.init_session(db_settings)
def process_item(self, item, spider):
@8maki
8maki / file0.py
Created March 21, 2014 08:39
ScrapyをScriptから実行する際のスクリプト ref: http://qiita.com/8maki/items/62b39130c116d3844c1c
spider = FollowAllSpider(domain='scrapinghub.com')
settings = get_project_settings()
crawler = Crawler(settings)
crawler.signals.connect(reactor.stop, signal=signals.spider_closed)
crawler.configure()
crawler.crawl(spider)
crawler.start()
log.start()
reactor.run()
@8maki
8maki / isNumber.js
Created February 27, 2012 11:08
isNumber
function isNumber(str) {
return !isNaN(+str)
}
@8maki
8maki / is_dict.py
Created December 31, 2011 16:50
is_dict
aaa = {}
isinstance(aaa, dict)
@8maki
8maki / sort_dirlist_by_ctime.py
Created November 21, 2011 02:31
sort_dirlist_by_ctime
dir_names.sort(cmp=lambda x,y: cmp(os.path.getctime('%s/%s' % (base_dir, x)), os.path.getctime('%s/%s' % (base_dir, y)))
@8maki
8maki / sort_list_by_value.py
Created November 21, 2011 02:29
sort_list_by_value
list.sort(cmp=lambda x,y: cmp(int(x), int(y)))
@8maki
8maki / sort_list.py
Created November 8, 2011 02:32
sort_list
list.sort( cmp=lambda x, y: cmp(x, y), reverse=True )
re.sub(u'([ぁ-ゞァ-ヾ一-龠0−9!?]+)', lambda m: urllib2.quote(m.group(0).encode('utf8')), k)
@8maki
8maki / convert_datetime_to_unixtime.py
Created October 5, 2011 10:06
convert_datetime_to_unixtime
def convert_datetime_to_unixtime(d)
time.mktime(d.timetuple())
@8maki
8maki / sort_dict_by_value.py
Created October 4, 2011 11:52
sort_dict_by_value
for k, v in sorted(d.items(), key=lambda x:x[1]):
print k, v