Skip to content

Instantly share code, notes, and snippets.

@koddsson
Created December 19, 2019 12:44
Show Gist options
  • Save koddsson/4368ba60cf517c300214b4fbe80248e8 to your computer and use it in GitHub Desktop.
Save koddsson/4368ba60cf517c300214b4fbe80248e8 to your computer and use it in GitHub Desktop.
A simple app that scrapes rightmove for flats and sends them to a Telegram chat.
#!/usr/bin/env node
const cheerio = require('cheerio')
const fetch = require('node-fetch')
const loki = require('lokijs')
const telegram = require('telegram-bot-api')
const api = new telegram({ token: process.env.TELEGRAM_API });
const db = new loki('rightmoveFile', {
autoload: true,
autoloadCallback : databaseInitialize,
autosave: true,
autosaveInterval: 4000
})
// implement the autoloadback referenced in loki constructor
function databaseInitialize() {
let propertiesDB = db.getCollection('properties')
if (propertiesDB === null) {
propertiesDB = db.addCollection('properties', { indices: 'id' })
}
}
// TODO: Build up URL.
const url = 'PASTE THE RIGHTMOVE RESULTS PAGE URL HERE'
fetch(url).then(response => response.text()).then(async html => {
const $ = cheerio.load(html)
const script = $('script').eq(2)
const json = script.html().replace('window.jsonModel =', '')
const data = JSON.parse(json)
const propertiesDB = db.getCollection('properties')
const properties = data.properties
.filter(property => property.heading !== 'Featured Property') // Filter out ads
.map(property => {
return {
id: property.id,
url: `https://rightmove.co.uk${property.propertyUrl}`,
date: property.addedOrReduced
}
})
.filter(property => propertiesDB.find({'id': property.id}).length === 0)
.map(property => {
propertiesDB.insert(property)
return property
})
for (const property of properties.reverse()) {
await api.sendMessage({
chat_id: 'PASTE YOUR CHAT ID HERE',
text: property.url
})
}
db.close()
})
{
"name": "rightmove-scraper",
"version": "0.0.1",
"description": "Simple Rightmove scraper that messages you flats on Telegram",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"keywords": [],
"author": "",
"license": "ISC",
"dependencies": {
"cheerio": "^1.0.0-rc.3",
"lokijs": "^1.5.7",
"node-fetch": "^2.6.0",
"telegram-bot-api": "^1.3.3"
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment