Nodejsを使った山手線の家賃平均を求めるスクリプト

readme.md

npm install

REQUIRE PhantomJS

node scraping.js > result.json

package.json

{
  "name": "yamanote-tintai",
  "version": "1.0.0",
  "description": "",
  "main": "index.js",
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 1"
  },
  "author": "",
  "license": "ISC",
  "dependencies": {
    "async": "^1.5.0",
    "phantom": "^0.8.0"
  }
}

result.json

{
	"yamanote": [
		{
			"name": "西日暮里",
			"rent": 24667
		},
		{
			"name": "大塚",
			"rent": 27667
		},
		{
			"name": "池袋",
			"rent": 31333
		},
		{
			"name": "日暮里",
			"rent": 31667
		},
		{
			"name": "御徒町",
			"rent": 32000
		},
		{
			"name": "駒込",
			"rent": 33000
		},
		{
			"name": "上野",
			"rent": 33667
		},
		{
			"name": "巣鴨",
			"rent": 34667
		},
		{
			"name": "田端",
			"rent": 35000
		},
		{
			"name": "高田馬場",
			"rent": 37000
		},
		{
			"name": "恵比寿",
			"rent": 39000
		},
		{
			"name": "目白",
			"rent": 39000
		},
		{
			"name": "渋谷",
			"rent": 40333
		},
		{
			"name": "秋葉原",
			"rent": 41333
		},
		{
			"name": "大崎",
			"rent": 42000
		},
		{
			"name": "新大久保",
			"rent": 42333
		},
		{
			"name": "新宿",
			"rent": 42333
		},
		{
			"name": "鶯谷",
			"rent": 44667
		},
		{
			"name": "五反田",
			"rent": 47667
		},
		{
			"name": "目黒",
			"rent": 48333
		},
		{
			"name": "神田",
			"rent": 49333
		},
		{
			"name": "品川",
			"rent": 50667
		},
		{
			"name": "田町",
			"rent": 51000
		},
		{
			"name": "代々木",
			"rent": 55667
		},
		{
			"name": "浜松町",
			"rent": 56333
		},
		{
			"name": "原宿",
			"rent": 63333
		},
		{
			"name": "東京",
			"rent": 64667
		},
		{
			"name": "新橋",
			"rent": 74000
		},
		{
			"name": "有楽町",
			"rent": 104333
		}
	]
}

scraping.js

var phantom = require('phantom');
var async = require('async');
var result = {yamanote: []};// 取得したデータを格納

// まずは山手線の駅一覧を取得
phantom.create(function (ph) {
	ph.createPage(function (page) {
		page.open("http://suumo.jp/chintai/tokyo/en_yamanotesen/", function (status) {
			page.evaluate(function () {
				var list = [];
				$(".searchitem label").each(function(i,station){
					var id = $(station).attr("for");
					var name = $(station).find("span:first").text();
					list.push({id: id, name: name});
				});
				return list;
			},
			// ここから各駅の平均家賃収集
			function (stationlist) {
				async.eachSeries(stationlist, function(data, next) {
					this.data = data;
					async.waterfall([
						// 1. phantom.create
						function (callback) {
							phantom.create({parameters: {'load-images': 'no'}},function (ph) {
								this.ph = ph;
								callback(null);
							})
						},
						// 2. ph.createPage
						function (callback) {
							this.ph.createPage(function(page){
								this.page = page;
								callback(null);
							});
						},
						// 3. page.open
						function (callback) {
							this.page.open("http://suumo.jp/chintai/tokyo/en_yamanotesen/", function (status) {
								callback(null);
							});
						},
						// 4. config
						function (callback) {
							this.page.evaluate(
								function(data){
									$('input[name=et]').val(['15']); // 徒歩15分以内
									document.getElementById(data.id).checked = true; // 駅指定
									document.getElementById("tc4").checked = true; // エアコン
									document.getElementById("tc0").checked = true; // バストイレ別
									return document.getElementsByClassName("js-ensenSearchBtn")[0].click();
								},
								function(){
									setTimeout(function(){callback(null)}, 3000); // 3秒ルール
								},
								this.data
							)
						},
						// 5. calc
						function (callback) {
							this.page.evaluate(
								function(){
									var sum=0;
									$('.detailbox-property-point').each(function(){
										sum += parseInt($(this).text(), 10);
									});
									return sum / $('.detailbox-property-point').length;
								},
								function(average){
									average = Math.round(average*10000);
									var obj = {name: this.data.name, rent: average};
									result.yamanote.push(obj);
									callback(null);
								}
							);
						}
						// 6. screenshot
						// ,
						// function(callback){
						// 	this.page.render(data.id+'.png');
						// 	callback(null);
						// }
					], function (err) {
							this.ph.exit();
							next();
					});
				}, function complete(err) {
					result.yamanote.sort(function(a,b){
						if(a.rent<b.rent) return -1;
						if(a.rent > b.rent) return 1;
						return 0;
					});
					console.log(JSON.stringify(result, null, '\t'));
					ph.exit();
				});
			});
		});
	});
});