Scrapes all the download urls of color sublime themes

colorsublime.js

const R = require('ramda')
const O = require('observable-air')
const axios = require('axios')
const { JSDOM } = require('jsdom')

const makeRequest = url =>
  axios.get(url).then(response => response.data).then(html => new JSDOM(html))
const fetchDOM = url => O.multicast(O.fromPromise(() => makeRequest(url)))
const fromArray = R.compose(O.fromArray, Array.from)
const findE = R.curry((sel, dom) =>
  fromArray(dom.window.document.querySelectorAll(sel))
)
const findDownloadURLS = R.compose(
  O.map(R.prop('href')),
  O.join,
  O.map(findE('.download'))
)
const findPages = R.compose(
  O.filter(R.test(/http/)),
  O.map(R.prop('href')),
  O.join,
  O.map(findE('.pagination a'))
)

const unique = (def, fn) => {
  const set = new Set()
  return p => {
    if (set.has(p)) return def()
    set.add(p)
    return fn(p)
  }
}

const main = unique(O.empty, url => {
  const dom$ = fetchDOM(url)
  const paginationURL$ = findPages(dom$)
  const downloadURL$ = findDownloadURLS(dom$)
  return O.merge(downloadURL$, O.flatMap(main, paginationURL$))
})