Jupyter utils.js

utils.js

/*


Gist: https://gist.github.com/mfyz/1dd5bb1145e809fecb60df15661045b3


## Get/Update Local Copy:
curl https://gist.githubusercontent.com/mfyz/1dd5bb1145e809fecb60df15661045b3/raw/utils.js > utils.js


## Dependencies (add/change database dependency package for sequelize)
npm install svg-chartist dotenv sequelize mysql2


*/

require('dotenv').config()
const chartistSvg = require('svg-chartist')
const Sequelize = require('sequelize') // Disable this line if no db conn needed
const papa = require('papaparse')
const fs = require('fs')

const sum = (MyArr) => MyArr.reduce((a, b) => a + b, 0)
const avg = (MyArr) => MyArr.reduce(function(p,c,i){return p+(c-p)/(i+1)},0)

const sortBy = (data, column, reverse) => {
  data.sort((a, b) => {
    const retTrue = reverse ? 1 : -1
    const retFalse = reverse ? -1 : 1
    if (a[column] < b[column]) return retTrue
    if (a[column] > b[column]) return retFalse
    return 0
  })
  return data
}


const clone = (obj) => {
  try { return JSON.parse(JSON.stringify(obj)) }
  catch (e) { return obj }
}


class tableCell {
  constructor (value, opts) {
    this.value = value
    this.opts = opts || {}
  }

  toString () {
    const { style } = this.opts
    return `<td${style ? ` style="${style}"` : ''}>${this.value}</td>`
  }
}

const table = (data, opts) => {
  data = clone(data)
  let { cellRenderer, fields, top, slice, sort, sortReverse } = (opts ? opts : {})
  if (fields) fields = fields.split(',')
  else fields = Object.keys(data[0])
  // Normalize data object to array
  if (data[0] === undefined) {
    data = Object.keys(data).map((key) => {
      return {
        key,
        ...(typeof data[key] == 'object' ? data[key] : { value: data[key] })
      }
    })
  }
  // Sort
  if (sort) {
    if (sort.substring(0,1) == '-') { sort = sort.substring(1); sortReverse = true }
    data = sortBy(data, sort, sortReverse)
  }
  // Slice (or top)
  if (top || slice) {
    if (top && !slice) slice = '0,' + top
    slice = slice.split(',')
    data = data.slice(slice[0], slice[1])
  }
  // Start constructing table
  let table = `<table cellpadding="5" border="1">`
  // Header Row
  table += `<tr style="background-color: rgba(150, 150, 150, 0.3); font-weight: bold;">`
  for (field of fields) {
    table += `<td align="left">${field}</td>`
  }
  table += '</tr>'
  // Rows
  let value = ''
  let i = 0
  for (row of data) {
    table += '<tr>'
    for (field of fields) {
      if (cellRenderer) value = cellRenderer(value, field, i, row)
      if (typeof row[field] === 'tableCell') { // TODO: this will be broken with the clone() at the top
        table += row[field].toString()
      }
      else {
        value = row[field]
        if (!value) value = ''
        table += `<td align="left">${value}</td>`
      }
    }
    table += '</tr>'
    i++
  }
  table += '</table>'
  return table
}


const chartPie = (data, labels, opts) => {
  data = clone(data)

  // versions = Object.keys(engineVersionsData)
  // const values = []
  // const labels = []
  // versions.map((d) => {
  //   values.push(engineVersionsData[d])
  //   labels.push(d + '.x')
  //   return d
  // })
  
  // // render chart
  // chartData = {
  //   labels: labels,
  //   series: values
  // }
  // chartOpts = {
  //   options: {
  //     width: 700,
  //     height: 400,
  //     chartPadding: 30,
  //     labelOffset: 100,
  //     labelDirection: 'explode',
  //   },
  //   labelInterpolationFnc: function(value) {
  //     return value[0]
  //   }
  // }
  // html = await chartistSvg('pie', chartData, chartOpts)
}


const chartBar = (data, opts) => {
  data = clone(data)

  let { labelField, valueField, labelRenderer } = (opts ? opts : {})
  let values = []
  let labels = []
  if (labelField && valueField) {
    data.map((r) => {
      labels.push(r[labelField])
      values.push(r[valueField])
    })
  }
  else if (data[0] !== undefined) {
    // Simple array with key and value or 0, 1
    for (r of data) {
      if (r['key'] !== undefined) {
        labels.push(r['key'])
        values.push(r['value'])
      }
      else {
        labels.push(r[0])
        values.push(r[1])
      }
    }
  }
  else { // Simple key value object
    Object.keys(data).map((l) => {
      labels.push(l)
      values.push(data[l])
    })
  }
  if (labelRenderer) labels = labels.map((l) => labelRenderer(l))
  // console.log('--> labels', labels)
  // console.log('--> values', values)

  // Render chart
  chartData = {
    labels: labels,
    series: [
      values
    ]
  }

  const { options, ...chartProps } = (opts ? opts : {})
  chartOptions = {
    options: {
      width: 700,
      height: 350,
      axisX: {
        showLabel: true,
        showGrid: false,
      },
      ...options
    },
    title: {
      height: 50,
      fill: "blue"
    },
    onDraw: function (data) {
      if(data.type === 'bar') {
        data.element.attr({
          style: 'stroke-width: 30px'
        });
      }
    },
    ...chartProps
  }

  return chartistSvg('bar', chartData, chartOptions)
}

const chartBarStack = (data, fields, opts) => { // stack multiple field values
  
}

const chartLine = (data, fields, opts) => { // line chart from multiple field values
  
}

const chartHist = (data, field, opts) => { // automatic distribution calc
  
}

const chartTimeBar = (data, field, opts) => { // automatic bar from time series
  // opts.period -> day/week/month
}

// chartTimeBarStack
// chartTimeLine


/* ============ Database methods using sequelize ============ */

const db_connect = async ({ dsn, dialect, logging }) => {
  const db = new Sequelize(dsn ? dsn : process.env.DB_DSN, {
    dialect: dialect ? dialect : 'mysql',
    logging: logging ? logging : false
  })
  await db.authenticate();
  return db
}

const db_query = async (db, sql, values, type) => {
  return await db.query(sql, {
    type: type ? type : Sequelize.QueryTypes.SELECT,
    ...(values ? {
      replacements: values
    } : {})
  })
}
const db_select = db_query
const db_insert = async (db, sql, values) => db_update(db, sql, values, Sequelize.QueryTypes.INSERT)
const db_update = async (db, sql, values) => db_update(db, sql, values, Sequelize.QueryTypes.UPDATE)
const db_delete = async (db, sql, values) => db_update(db, sql, values, Sequelize.QueryTypes.DELETE)


const readCsvString = (str) => {
  const csv = papa.parse(str, { header: true })
  return csv.data
}

const readCsvFile = (filePath) => {
  const csvString = fs.readFileSync(filePath, { encoding:'utf8' })
  return readCsvString(csvString)
}

const convertCsv = (data, fields) => {
  if (fields === undefined) {
    fields = Object.keys(data[0])
  }
  else if (typeof fields === 'string') { 
    fields = fields.split(',')
    fields.map(f => f.trim())
  }

  const csv = papa.unparse({
    fields,
    data
  })
  return csv
}

const writeCsv = (filepath, data, fields) => {
  const csv = convertCsv(data, fields)
  fs.writeFileSync(filepath, csv)
  return true
}


module.exports = {
  readCsvString,
  readCsvFile,
  convertCsv,
  writeCsv,
  clone,
  sum,
  avg,
  sortBy,
  tableCell,
  table,
  chartBar,
  db: {
    connect: db_connect,
    query: db_query,
    select: db_select,
    insert: db_insert,
    update: db_update,
    delete: db_delete,
  }
}