Skip to content

Instantly share code, notes, and snippets.

@nnnikolay
Created February 23, 2017 15:02
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save nnnikolay/4ca6646dfb4cb458ff3fcaf519e8977d to your computer and use it in GitHub Desktop.
Save nnnikolay/4ca6646dfb4cb458ff3fcaf519e8977d to your computer and use it in GitHub Desktop.
With this module it's possible to crawl google places API in a stream
import stream from 'stream'
import https from 'https'
export default class googleServiceStream extends stream.Transform {
/**
* Initialize the stream
*/
constructor() {
super({objectMode: true})
}
/**
* Take data and call Googl Service API
* to fetch all related data
* @param {data} JSON string (from another stream)
* @param {String} encoding The encoding of the passed data chunk
* @param {Function} done A callback function for async operations
*/
_transform(data, encoding, done) {
try {
data = JSON.parse(data)
fetchFromGoogleServices.call(this, null, data, done)
} catch (e) {
done()
}
}
}
// Google API Response for each item looks like this:
// {
// geometry: [
// Object
// ],
// icon: 'https: //maps.gstatic.com/mapfiles/place_api/icons/generic_business-71.png',
// id: '', //<---- deprected don't use it,
// name: '',
// place_id: '',
// reference: '', //<---- deprected don't use it
// scope: 'GOOGLE',
// types: [
// Object
// ],
// vicinity: ''
// }
//
// place_id — a textual identifier that uniquely identifies a place.
// To retrieve information about the place, pass this identifier in the placeId field of a Places API request.
// For more information about place IDs, see the place ID overview (https://developers.google.com/places/web-service/place-id).
function fetchFromGoogleServices(nextPageToken = null, placeObject = null, done = null) {
// request google service API with specific lat/lon
let url = 'https://maps.googleapis.com/maps/api/place/nearbysearch/json?key=' + process.env.GOOGLE_PLACES_API_KEY + '&'
const options = (nextPageToken === null)
? [
'type=funeral_home',
'rankby=prominence',
'radius=50000',
'location=' + placeObject['Latitude'] + ',' + placeObject['Longitude']
].join('&')
: 'pagetoken=' + nextPageToken
url = url + options
https.get(url, (res) => {
let rawData = '', parsedData = {}
res.setEncoding('utf8')
res.on('data', (chunk) => rawData += chunk)
res.on('end', () => {
try {
parsedData = JSON.parse(rawData)
console.log(parsedData.results.length)
} catch (e) {
throw e.message
}
// there are more pages in the result
if (parsedData.next_page_token !== undefined) {
// transform it
parsedData.results.map((r) => {
this.push(JSON.stringify(transformIt(r)))
}, this)
// call the next page only in 2 seconds !! Google limitation
setTimeout(function() {
fetchFromGoogleServices.call(this, parsedData.next_page_token, null, done)
}.bind(this), 2000)
} else {
try {
parsedData.results.map((r) => {
this.push(JSON.stringify(transformIt(r)))
}, this)
done()
} catch (e) {
console.log(e)
}
}
})
}).on('error', (e) => {
console.error(e)
})
}
function transformIt(data) {
return {
id: data.place_id,
name: data.name,
address: data.vicinity,
lat: data.geometry.location.lat,
lng: data.geometry.location.lng
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment