Skip to content

Instantly share code, notes, and snippets.

@AlexanderDzhoganov
Created December 17, 2019 13:45
Show Gist options
  • Save AlexanderDzhoganov/c1fcbf942acc1afb8177e4f74fc4351d to your computer and use it in GitHub Desktop.
Save AlexanderDzhoganov/c1fcbf942acc1afb8177e4f74fc4351d to your computer and use it in GitHub Desktop.
import fs from 'fs'
import path from 'path'
import qs from 'querystring'
import httpsProxyAgent from 'https-proxy-agent'
import axios from 'axios'
const outDir = './images/'
const baseUrl = 'https://www.deviantart.com/_napi/da-browse/api/tags'
const userAgent = 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11'
let proxies = [
'http://95.216.240.148:808',
'http://95.216.156.195:443',
'http://5.1.53.46:8080',
'http://144.217.74.219:3128',
'http://178.62.232.215:8080',
'http://167.114.247.229:9999',
'http://136.243.47.220:3128',
'http://51.158.106.54:8811',
'http://51.158.123.35:8811',
'http://163.172.189.32:8811',
'http://51.158.68.133:8811',
'http://51.158.111.229:8811'
]
const tags = [
'games',
'gaming',
'videogame',
'leagueoflegends',
'paintings',
'digitalart'
]
function pageUrl(tag, offset) {
const query = qs.stringify({
init: 'false',
order: 'popular-all-time',
limit: '120',
tag,
offset
})
return `${baseUrl}?${query}`
}
function createProxyAgent() {
const proxy = proxies[Math.floor(Math.random() * proxies.length)]
console.log('using proxy url', proxy)
return httpsProxyAgent(proxy)
}
async function apiRequest(url) {
console.log(url)
const response = await axios(url, {
headers: {
'User-Agent': userAgent
},
json: true,
httpsAgent: createProxyAgent(),
timeout: 5000
})
return response.data
}
async function downloadImage(id, url, ext) {
console.log('downloading image', id, url)
let response
try {
response = await axios(url, {
headers: {
'User-Agent': userAgent
},
responseType: 'stream'
})
} catch (err) {
console.log('failed', id)
return
}
const imgPath = path.resolve(outDir, `${id}${ext}`)
if (fs.existsSync(imgPath)) {
return
}
const stream = fs.createWriteStream(imgPath)
response.data.pipe(stream)
return new Promise((resolve, reject) => {
stream.on('finish', resolve)
stream.on('error', reject)
}).then(() => {
console.log('success', id)
}).catch(err => {
console.log('error', id)
return
})
}
function fetchImages(page) {
return Promise.all(page.deviations
.filter(deviation => deviation.isDownloadable)
.filter(deviation => deviation.media && deviation.media.token && deviation.media.token.length !== 0)
.map(deviation => {
const {baseUri, prettyName, token, types} = deviation.media
let url = baseUri
if (types && types.length !== 0) {
let maxWidth = 0
let urlPart = ''
for (const type of types) {
if (!type.c) {
continue
}
if (type.w > maxWidth) {
maxWidth = type.w
urlPart = type.c.replace('<prettyName>', prettyName)
}
}
if (maxWidth !== 0) {
url = `${url}/${urlPart}`
}
}
url = `${url}?token=${token[0]}`
const ext = path.extname(baseUri)
return downloadImage(deviation.deviationId, url, ext)
}))
}
async function run() {
let offset = 0
let tag = tags[0]
while (true) {
let page
try {
page = await apiRequest(pageUrl(tag, offset))
} catch (err) {
console.log('failed, retrying')
continue
}
await fetchImages(page)
offset = page.nextOffset
if (!offset) {
tag = tags[0]
tags.shift()
offset = 0
console.log('using tag', tag)
}
console.log('current offset', offset)
}
}
run()
.then(() => console.log('all done'))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment