Scraping & reading Versum pixel data

_1_scraping.js

// Fetches minted tokens from Versum, starting from the most recent -> older
//
// You can try this out in a REST app or curl'ing by calling a GET to
// https://versum.xyz/api/public/feed?offset=0&limit=10&maxtime=1649840640000

// Returns a json object with the token data, creator wallet & info, and urls to the images on IPFS
// Use query string params 'offset' and 'limit' to paginate data

//...

async fetchLatestTokens(offset = 0, limit = 10) {

  const now = moment.utc()
  const floored = now.startOf(`minute`)
  const timestamp = floored.valueOf()

  const params = {
    offset: offset,
    limit: limit,
    maxtime: timestamp
  }

  const url = this._parseRouteUrl(this.feedRoute, params)

  const response = await fetch(url)
  const result = await response.json()

  if(!result || !result.hasOwnProperty(`tokens`))
    throw new Error(`VersumScraper.fetchLatestTokens: Error, invalid json result or no .tokens property available.`)

  return result.tokens
}
  
//...

/*
{
    "tokens": [
        {
            "id": "67d05941eb5b50b0647d3842bf958a403fb776ca4d52bd18e6ebb84d5a1be34b",
            "attributes": [],
            "artifact_uri": "ipfs://QmVYosGzWK3V19QZd8KwwSCt9i7oRUqw6G2u8G9YuaWRBm",
            "max_per_address": 0,
            "creators": [
                {
                    "wallet": {
                        "address": "tz1VJwMU99Mm3tRRVpStbHk8P1MJj2Y52Yz8",
                        "banned_hen": false,
                        "banned_versum": false,
                        "banned_versum_on_chain": false,
                        "user": [
                            {
                                "user": {
                                    "id": "tz1VJwMU99Mm3tRRVpStbHk8P1MJj2Y52Yz8",
                                    "name": "vanguardesign",
                                    ....
                                    ....
            "display_uri": "ipfs://QmWGJGJHnvuGcJC3X4jDw9mhZH7396tAYtHBQatF3kRNrQ",
            "formats": [
                {
                    "uri": "ipfs://QmVYosGzWK3V19QZd8KwwSCt9i7oRUqw6G2u8G9YuaWRBm",
                    "duration": "00:00:04.000000",
                    "fileName": "original.mp4",
                    "fileSize": 2694180,
                    "mimeType": "video/mp4",
                    "dimensions": {
                        "unit": "px",
                        "value": "1080x1080"
                    }
                },
                {
                    "uri": "ipfs://QmZ4L51xArKS7qHJzmUULrt7VvvUqZ5LxeLFBXDccHeiGS",
                    "duration": "00:00:04.000000",
                    "fileName": "full_video_720h.mp4",
                    "fileSize": 1574861,
                    "mimeType": "video/mp4",
                    "dimensions": {
                        "unit": "px",
                        "value": "720x720"
                    }
                },
                {
                    "uri": "ipfs://QmUQnNzKwFAPwGKSGMC92zSQjAi1ksaNkpUnfhehG7geNH",
                    "duration": "00:00:04.000000",
                    "fileName": "preview_video_480w.mp4",
                    "fileSize": 902016,
                    "mimeType": "video/mp4",
                    "dimensions": {
                    ...
                    
*/

_2_read_ipfs.js

// You can then get for example the 'display_uri' property from the JSON object and download the image from IPFS
// I strongly suggest using JIMP from npm registry https://www.npmjs.com/package/jimp
// So you can do easy image manipulation and don't have to be bothered by MIME types & other image related technical details.

//...

async downloadImage(urlOrHash) {
  const hash = urlOrHash.includes(`ipfs://`)
    ? urlOrHash.substring(7)
    : urlOrHash
      
  const url = `https://api.ipfsbrowser.com/ipfs/get.php?hash=${hash}`

  return Jimp.read(url)
}

//...

// This returns a jimp image object that you can use to manipulate, resize, ...
// e.g.: 
//
//    const image = await downloadImage(token.display_uri);
//    image.scaleToFit(1080, 1080)     // resizes and scales the image to fit in a 1080x1080 resolution

_3_measure_monochrome.js

// This code reads the raw pixel data from the JIMP image and tries to determine if the image is monochrome
// You could try to extract a color palette from the image and include it in your local cache of the Versum db.
// Like: imagexyz has color palette: #ff0000 #ee0011 #dd0022 ...
//
// threshold is the 'fault' margin, if the image its monochromacity value is below this threshold, the image probably is monochromatic.

//...

_isMonochromatic(image, threshold = 0.015) {
  return this._measureMonochromaticity(image) < threshold
}


_measureMonochromaticity(image) {
  const pixels = image.bitmap.data
  const width = image.bitmap.width
  const height = image.bitmap.height

  let diff = 0
  for(let i = 0; i < pixels.length; i += 4) {

    const r = pixels[i]
    const g = pixels[i + 1]
    const b = pixels[i + 2]

    const rg = Math.abs(r - g)
    const rb = Math.abs(r - b)
    const gb = Math.abs(g - b)

    diff += rg + rb + gb
  }

  const result = diff / (width * height) / (255 * 3)

  return result
}

//...