Skip to content

Instantly share code, notes, and snippets.

@Necmttn
Last active June 12, 2017 09:00
Show Gist options
  • Save Necmttn/765beff6bcc5795709b56a94149b1b94 to your computer and use it in GitHub Desktop.
Save Necmttn/765beff6bcc5795709b56a94149b1b94 to your computer and use it in GitHub Desktop.
Instagram download picture.
var fs = require("fs")
var request = require("request")
module.exports = function(url, dest, cb) {
var file = fs.createWriteStream(dest)
var sendReq = request.get(url)
// verify response code
sendReq.on("response", function(response) {
if (response.statusCode !== 200) {
return cb("Response status was " + response.statusCode)
}
})
// check for request errors
sendReq.on("error", function(err) {
fs.unlink(dest)
if (cb) {
return cb(err.message)
}
})
sendReq.pipe(file)
file.on("finish", function() {
file.close(cb) // close() is async, call cb after close completes.
})
file.on("error", function(err) {
// Handle errors
fs.unlink(dest)
// Delete the file async. (But we don't check the result)
if (cb) {
return cb(err.message)
}
})
}
const fs = require("fs")
const request = require("request")
const mkdirp = require("mkdirp")
const ProgressBar = require("progress")
const { get } = require("lodash")
const download = require("./utils/download-file")
const username = process.argv[2]
if (!username) {
console.log(
`
You didn't supply an Instagram username!
Run this command like:
node scrape.js INSTAGRAM_USERNAME
`
)
process.exit()
}
// Convert timestamp to ISO 8601.
const toISO8601 = timestamp => new Date(timestamp * 1000).toJSON()
// Create the progress bar
const bar = new ProgressBar(
`Downloading instagram posts [:bar] :current/:total :elapsed secs :percent`,
{
total: 0,
width: 30,
}
)
// Create the images directory
mkdirp.sync(`./data/images`)
let posts = []
// Write json
const saveJSON = _ =>
fs.writeFileSync(`./data/posts.json`, JSON.stringify(posts, "", 2))
const getPosts = maxId => {
let url = `https://www.instagram.com/${username}/media`
if (maxId) url += `?max_id=${maxId}`
request(url, { encoding: `utf8` }, (err, res, body) => {
if (err) console.log(`error: ${err}`)
body = JSON.parse(body)
// Parse posts
let lastId
body.items
.filter(item => item.type === `image`)
.map(item => {
// Parse item to a simple object
return {
id: get(item, `id`),
code: get(item, `code`),
username: get(item, `user.username`),
avatar: get(item, `user.profile_picture`),
time: toISO8601(get(item, `created_time`)),
type: get(item, `type`),
likes: get(item, `likes.count`),
comment: get(item, `comments.count`),
text: get(item, `caption.text`),
media: get(item, `images.standard_resolution.url`, ``).replace(
`/s640x640`,
``
),
image: `images/${item.code}.jpg`,
}
})
.forEach(item => {
if (posts.length >= 100) return
// Download image locally and update progress bar
bar.total++
download(item.media, `./data/images/${item.code}.jpg`, _ => bar.tick())
// Add item to posts
posts.push(item)
// Save lastId for next request
lastId = item.id
})
if (posts.length < 100 && get(body, `more_available`)) getPosts(lastId)
else saveJSON()
})
}
getPosts()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment