Skip to content

Instantly share code, notes, and snippets.

@Azazeo
Azazeo / crawler.py
Created January 13, 2014 09:49
Crawler and data extractor for airbnb.com website
MAX_THREADS = 5
delay = 0.5
import psycopg2
import re
import sys
import time
import threading
import urllib2
import urlparse
@Azazeo
Azazeo / gist:8397359
Created January 13, 2014 09:51
Custom csv-file processor
class Room():
@classmethod
def _process_calendar(self, s):
return [[x.split(':')[0],x.split(':')[1]] for x in s.replace('"','').replace(' ','').replace('{','').replace('}','').split(',')]
def __init__(self, s):
a = s[0:-1].split('\t')
self.url = a[0]
self.date_parsed = a[1]
self.bedrooms = a[9]