Created
December 19, 2019 12:44
-
-
Save koddsson/4368ba60cf517c300214b4fbe80248e8 to your computer and use it in GitHub Desktop.
A simple app that scrapes rightmove for flats and sends them to a Telegram chat.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env node | |
const cheerio = require('cheerio') | |
const fetch = require('node-fetch') | |
const loki = require('lokijs') | |
const telegram = require('telegram-bot-api') | |
const api = new telegram({ token: process.env.TELEGRAM_API }); | |
const db = new loki('rightmoveFile', { | |
autoload: true, | |
autoloadCallback : databaseInitialize, | |
autosave: true, | |
autosaveInterval: 4000 | |
}) | |
// implement the autoloadback referenced in loki constructor | |
function databaseInitialize() { | |
let propertiesDB = db.getCollection('properties') | |
if (propertiesDB === null) { | |
propertiesDB = db.addCollection('properties', { indices: 'id' }) | |
} | |
} | |
// TODO: Build up URL. | |
const url = 'PASTE THE RIGHTMOVE RESULTS PAGE URL HERE' | |
fetch(url).then(response => response.text()).then(async html => { | |
const $ = cheerio.load(html) | |
const script = $('script').eq(2) | |
const json = script.html().replace('window.jsonModel =', '') | |
const data = JSON.parse(json) | |
const propertiesDB = db.getCollection('properties') | |
const properties = data.properties | |
.filter(property => property.heading !== 'Featured Property') // Filter out ads | |
.map(property => { | |
return { | |
id: property.id, | |
url: `https://rightmove.co.uk${property.propertyUrl}`, | |
date: property.addedOrReduced | |
} | |
}) | |
.filter(property => propertiesDB.find({'id': property.id}).length === 0) | |
.map(property => { | |
propertiesDB.insert(property) | |
return property | |
}) | |
for (const property of properties.reverse()) { | |
await api.sendMessage({ | |
chat_id: 'PASTE YOUR CHAT ID HERE', | |
text: property.url | |
}) | |
} | |
db.close() | |
}) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"name": "rightmove-scraper", | |
"version": "0.0.1", | |
"description": "Simple Rightmove scraper that messages you flats on Telegram", | |
"main": "index.js", | |
"scripts": { | |
"test": "echo \"Error: no test specified\" && exit 1" | |
}, | |
"keywords": [], | |
"author": "", | |
"license": "ISC", | |
"dependencies": { | |
"cheerio": "^1.0.0-rc.3", | |
"lokijs": "^1.5.7", | |
"node-fetch": "^2.6.0", | |
"telegram-bot-api": "^1.3.3" | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment