Inkbunny favorites crawler

README.md

Inkbunny favorites downloader

Author: Hoya Kim

Prerequirements

• Node.js 6 or higher
• latest npm package manager
• Large enough free space of storage

How to use it

• Download sync.js
• Place sync.js in an empty folder
• Open terminal, type 'npm init'
• After init(just type enter several times) type 'npm i lodash axios path'
• 'mkdir data' folder(or edit dataDir variable as you want)
• Place credential.json and last_submission_id.txt into the data folder
• Edit 2 files, Do NOT set 0 or negative number into the last_submission_id.txt file. Program will never stops.
• last_submission_id is your last favorite submission id. Submission id is like this: https://inkbunny.net/s/1648903
• Run 'nodejs sync.js', Downloading will takes very long time.
• Favorites saved into the each Artist's name folder.

credential.json

{
  "username": "Hoya82",
  "password": "pa$$w0rd"
}

last_submission_id.txt

1648903

sync.js

'use strict'

const fs = require('fs')
const path = require('path')
const _ = require('lodash')
const axios = require('axios')

let sid = ''
let user_id = 0
const siteUrl = 'https://inkbunny.net'

const dataDir = './data'
const credential = JSON.parse(fs.readFileSync(path.join(dataDir, 'credential.json'), 'utf-8'))

function delay(t) {
  return new Promise(done => {
    setTimeout(() => { done() }, t)
  })
}

async function download(url, path, options) {
  // axios image download with response type "stream"
  const response = await axios({
    method: 'GET',
    url: url,
    responseType: 'stream',
    headers: options.headers
  })

  // pipe the result stream into a file on disc
  response.data.pipe(fs.createWriteStream(path))

  // return a promise and resolve when download finishes
  return new Promise((resolve, reject) => {
    response.data.on('end', () => {
      resolve()
    })

    response.data.on('error', () => {
      reject()
    })
  })
}

async function getSubmission(submission_id) {
  let submission = {}
  while(true) {
    let res = await axios.get(siteUrl + '/api_submissions.php', {
      headers: {},
      params: {
        sid: sid,
        submission_ids: submission_id,
        output_mode: 'json',
        sort_keywords_by: 'alphabetical',
        show_description: 'yes',
        show_description_bbcode_parsed: 'yes',
        show_writing: 'yes',
        show_writing_bbcode_parsed: 'yes',
        show_pools: 'yes'
      }
    })

    if(res.status != 200) {
      await delay(60000)
      continue
    } else {
      submission = _.get(res.data, 'submissions[0]', {})
      break
    }
  }

  let artist = submission.username

  if(!fs.existsSync(path.join(dataDir, artist))) {
    fs.mkdirSync(path.join(dataDir, artist))
    fs.writeFileSync(path.join(dataDir, artist,'index.json'), JSON.stringify({ submissions: {} }, null, 2))
  }

  let usermeta = JSON.parse(fs.readFileSync(path.join(dataDir, artist, 'index.json')).toString())
  usermeta.submissions[submission.submission_id] = submission
  fs.writeFileSync(path.join(dataDir, artist, 'index.json'), JSON.stringify(usermeta, null, 2))

  if(typeof(submission.files) != typeof([])) {
    console.error(typeof(submission.files))
    return false
  }
  let urls = submission.files.map((f) => f.file_url_full)

  for(let url of urls) {
    let filename = url.substr(url.lastIndexOf('/') + 1)

    // Download only local file not exists
    if(!fs.existsSync(path.join(dataDir, artist, filename))) {
      let retry = 10
      while(retry > 0) {
        try {
          await download(encodeURI(url), path.join(dataDir, artist, filename), {})
          console.log({artist, filename})
          break
        } catch(e) {
          retry--;
          console.error('Cannot find: '+encodeURI(url))
          await delay(10000)
        }
      }
    } else {
      return true
    }
  }

  return false
}


;(async () => {
  let token = await axios.get(siteUrl + '/api_login.php', {
    headers: {},
    params: {
      username: credential.username,
      password: credential.password,
      output_mode: 'json'
    }
  })

  if(token.status == 200) {
    sid = token.data.sid
    user_id = token.data.user_id

    // console.log({sid, user_id})
  } else {
    console.error(token.data)
    return
  }

  // Fetch the first page(mode 1)
  let favList = await axios.get(siteUrl + '/api_search.php', {
    headers: {},
    params: {
      sid: sid,
      get_rid: 'yes',
      output_mode: 'json',
      submission_ids_only: 'yes',
      submissions_per_page: 100,
      favs_user_id: user_id,
      orderby: 'fav_datetime'
    }
  })

  let page = 0
  let rid = null
  let latest_submission_id = 0
  let last_submission_id = 0
  try {
    last_submission_id = parseInt(fs.readFileSync(path.join(dataDir, 'last_submission_id.txt')).toString())
    console.log("Fetch from " + last_submission_id)
  } catch(e) {
    // Do nothing
  }

  if(favList.status == 200) {
    page = favList.data.page // Usually 1
    rid = favList.data.rid
    latest_submission_id = _.get(favList, "data.submissions[0].submission_id", 0)

    for(let s of favList.data.submissions) {
      if(s.submission_id <= last_submission_id) {
        console.log("Latest submission " + latest_submission_id)
        fs.writeFileSync(path.join(dataDir, 'last_submission_id.txt'), latest_submission_id)
        return
      }
      await getSubmission(s.submission_id)
    }
  }

  page++
  while(true) {
    await delay(1000)

    // Countinuous search(mode 2)
    favList = await axios.get(siteUrl + '/api_search.php', {
      headers: {},
      params: {
        sid: sid,
        rid: rid,
        output_mode: 'json',
        submission_ids_only: 'yes',
        submissions_per_page: 100,
        favs_user_id: user_id,
        page: page
      }
    })

    if(favList.status != 200) {
      await delay(5000)
      continue
    }

    for(let s of favList.data.submissions) {
      if(s.submission_id <= last_submission_id) {
        console.log("Latest submission " + latest_submission_id)
        fs.writeFileSync(path.join(dataDir, 'last_submission_id.txt'), latest_submission_id)
        return
      }
      await getSubmission(s.submission_id)
    }

    page++
  }
})();