Skip to content

Instantly share code, notes, and snippets.

@chasestarr
Created May 22, 2019 16:34
Show Gist options
  • Save chasestarr/d81e92b238adad6e74cb111adaa6246a to your computer and use it in GitHub Desktop.
Save chasestarr/d81e92b238adad6e74cb111adaa6246a to your computer and use it in GitHub Desktop.

env vars:

SLACK_WEBHOOK_URL
CRAIGSLIST_SEARCH_URL=https://sfbay.craigslist.org/search/sfc/apa?query=glen+park&hasPic=1&postedToday=1&availabilityMode=0&sale_date=all+dates
SLACK_ICON_URL
const p = require('puppeteer');
const request = require('request-promise-native');
async function postToSlack(listing) {
const options = {
method: 'POST',
uri: process.env.SLACK_WEBHOOK_URL,
body: {
icon_url: process.env.SLACK_ICON_URL,
attachments: [
{
title: listing.title,
title_link: listing.href,
fallback: listing.title,
thumb_url: listing.img,
},
],
},
headers: {
'Content-type': 'application/json',
},
json: true,
};
try {
await request(options);
} catch (error) {
console.error(`failed to post ${listing.href} to slack.`);
console.log(error);
}
}
async function getDetails(listing) {
const img = await listing.$eval('img', img => img.getAttribute('src'));
const titleElement = await listing.$('.result-title');
const title = await (await titleElement.getProperty('textContent')).jsonValue();
const href = await (await titleElement.getProperty('href')).jsonValue();
return { img, title, href };
}
async function main() {
const browser = await p.launch({
args: ['--no-sandbox'],
defaultViewport: { width: 1024, height: 720 },
});
const page = await browser.newPage();
await page.goto(process.env.CRAIGSLIST_SEARCH_URL);
const rows = await page.$$('.rows>*');
const headingIndex = rows.findIndex(r => r._remoteObject.description === 'h4.ban.nearby');
const relevant = rows.slice(0, headingIndex);
const listings = await Promise.all(relevant.map(getDetails));
await Promise.all(listings.map(postToSlack));
await browser.close();
}
exports.main = main;
{
"name": "glen-park-homes",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"keywords": [],
"author": "",
"license": "ISC",
"dependencies": {
"@google-cloud/pubsub": "^0.24.1",
"puppeteer": "^1.12.2",
"request": "^2.88.0",
"request-promise-native": "^1.0.5"
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment