tumblrの全postをmongoDBに突っ込むアレ ※ クロール中にpostすると, バグります.

tumblr.py

# -*- coding: utf-8 -*-
import requests
from pymongo import Connection

con = Connection("localhost", 27017)
db = con["tumblr"]
url = "http://api.tumblr.com/v2/blog/tumblr.non117.com/posts/"
params = {"api_key" : ""}

def fetch(offset):
    params.update({"offset":offset})
    json = requests.get(url, params=params).json()
    posts = json["response"]["posts"]
    if posts == []:
        return -1
    db.posts.insert(posts)
    return len(posts)

if __name__ == "__main__":
    offset = 0
    while offset != -1:
        offset += fetch(offset)
        print offset
    con.close()