sos_lib_segments.js

if(!Memory.__segindex) {
  Memory.__segindex = {
    'index': {},
    'savelog': {},
    'buffer': {},
    'ttls': {},
    'clear': [],
    'critical': [],
    'last': 100
  }
}


var cache = {}

var sos_lib_segments = {

  maxMemory: 100*1024,
  maxActiveSegments: 10,

  // Start at 19- leave the first 20 for manual assignment
  min: 19,
  max: 99,

  // *If possible* Leave this many segments available after processing.
  free: 3,

  hasSegment: function (label) {
    return !!Memory.__segindex.index[label]
  },

  saveObject: function (label, data) {
    this.saveString(label, JSON.stringify(data))
  },

  saveString: function (label, string) {
    var needed_segments = Math.ceil(string.length/this.maxMemory)
    var ids = this.getIndexByLabel(label, true)
    if(!!RawMemory.segments) {
      var availableSegments = Object.keys(RawMemory.segments)
    } else {
      var availableSegments = []
    }

    if(ids.length > needed_segments) {
      // Mark unused segments for cleaning.
      var unneeded = ids.slice(needed_segments, ids.length)
      Memory.__segindex.clear = Memory.__segindex.clear.concat(unneeded)
      ids = ids.slice(0, needed_segments)
    } else if (ids.length < needed_segments) {
      var diff = needed_segments - ids.length
      for(var i = 0; i<diff; i++) {
        var id = this.getNextId()
        if(!id && id !== 0) {
          return ERR_FULL
        }
        ids.push(id)
      }
    }

    Memory.__segindex.index[label] = {'ids':ids}

    for(var i = 0; i < needed_segments; i++) {
      var start = i * this.maxMemory
      var end = start + this.maxMemory // will end *one before* this value
      var chunk = string.slice(start, end)
      var id = ids[i]
      Memory.__segindex.savelog[id] = Game.time
      if(!cache[Game.time]) {
        cache[Game.time] = {}
      }
      cache[Game.time][label] = chunk
      if(availableSegments.indexOf(id) < 0) {
        Memory.__segindex.buffer[id] = chunk
      } else {
        RawMemory.segments[id] = chunk
      }
    }
  },

  getObject: function (label) {
    var stringdata = this.getString(label)
    if(typeof stringdata == 'string') {
      if(stringdata.length <= 0) {
        return {}
      } else {
        var start = Game.cpu.getUsed()
        var data = JSON.parse(stringdata)
        var parseTime = Game.cpu.getUsed() - start
        Logger.log('Segment ' + label + ' parse time: ' + parseTime + ' with length ' + stringdata.length, LOG_WARN)
        Stats.addStat('segments.' + label, {
          'label': label,
          'parseTime': parseTime,
          'length': stringdata.length
        }, true)
        return data
      }
    }
    if(!stringdata || stringdata < 0) {
      return stringdata
    }
    return ERR_NOT_FOUND
  },

  getString: function (label, ttl=3) {
    var ids = this.getIndexByLabel(label, true)
    var datastring = ''
    for(var id of ids) {
      this.requestSegment(id, ttl)
      if(datastring === false || datastring < 0) {
        continue
      }

      if(typeof Memory.__segindex.buffer[id] == 'string') {
        datastring += Memory.__segindex.buffer[id]
        continue
      }

      if(!!RawMemory.segments && typeof RawMemory.segments[id] == 'string') {
        datastring += RawMemory.segments[id]
        continue
      }

      var saveTick = Memory.__segindex.savelog[id]
      if(!!cache[saveTick] && typeof cache[saveTick][id] == 'string') {
        datastring += cache[saveTick][id]
        continue
      }

      datastring = ERR_BUSY
    }

    return datastring
  },

  clear: function (label) {
    var ids = this.getIndexByLabel(label)
    Memory.__segindex.clear = Memory.__segindex.clear.concat(ids)
    delete Memory.__segindex.index[label]
  },

  reserveSegments: function (label, count=1) {
    var ids = this.getIndexByLabel(label)

    if(!ids) {
      ids = []
    }

    if(ids.length > count) {
      var unneeded = ids.slice(count, ids.length)
      Memory.__segindex.clear = Memory.__segindex.clear.concat(unneeded)
      ids = ids.slice(0, count)
    } else if(ids.length < count) {
      var diff = count - ids.length
      for(var i = 0; i<diff; i++) {
        var id = this.getNextId()
        if(!id && id !== 0) {
          return ERR_FULL
        }
        ids.push(id)
      }
    }

    Memory.__segindex.index[label].ids = ids
    return ids
  },

  requestSegment: function (index, ttl=5) {
    Memory.__segindex.ttls[index] = ttl
  },

  unrequestSegment: function (index) {
    delete Memory.__segindex.ttls[index]
  },

  markCritical: function (label) {
    if(Memory.__segindex.critical.indexOf(label) < 0) {
      Memory.__segindex.critical.push(label)
    }
  },

  unmarkCritical: function (label) {
    var index = Memory.__segindex.critical.indexOf(label)
    if(index >= 0) {
      Memory.__segindex.critical.splice(index, 1)
    }
  },

  getAvailableSegments: function () {
    if(!RawMemory.segments) {
      return []
    }
    var availableSegments = Object.keys(RawMemory.segments).map(Number)
    availableSegments = _.filter(availableSegments, function(a){
      return Number.isInteger(a)
    })
    return availableSegments
  },

  moveToGlobalCache: function () {

    // On a server without memory segments so there is nothing to move.
    if(!RawMemory.setActiveSegments) {
      return
    }

    // Shift segments out of RawSegments so more segments can be saved.
    var availableSegments = this.getAvailableSegments()
    for (var id of availableSegments) {
      // Out of management range.
      if(id < this.min || id > this.max) {
        continue
      }

      // Hasn't been saved yet so don't remove it.
      if(!Memory.__segindex.savelog[id]) {
        continue
      }

      // Something may be trying to clear it- just leave blank.
      if(RawMemory.segments[id] === '') {
        continue
      }

      // Something is trying to clear it- leave for end of tick processing.
      if(Memory.__segindex.clear.includes(id)) {
        continue
      }

      var saveTick = Memory.__segindex.savelog[id]
      if(!cache[saveTick]) {
        cache[saveTick] = {}
      }
      cache[saveTick][id] = RawMemory.segments[id]
      delete RawMemory.segments[id]
    }
  },

  process: function () {
    var availableSegments = this.getAvailableSegments()

    // Build list of critical segments from label.
    var critical = []
    for(var critical_label of Memory.__segindex.critical) {
      var ids = this.getIndexByLabel(critical_label)
      critical = critical.concat(ids)
    }

    // Create list of segments to ask for, starting with the critical ones.
    if(critical.length > this.maxActiveSegments) {
      var segments = _.shuffle(critical.concat([])).slice(0,this.maxActiveSegments)
    } else {
      var segments = critical.concat([])
    }

    // clean available segments
    for(var index in Memory.__segindex.clear) {
      var id = Memory.__segindex.clear[index]
      if(Memory.__segindex.buffer[id]) {
        delete Memory.__segindex.buffer[id]
      }
      if(availableSegments.indexOf(id) >= 0) {
        RawMemory.segments[id] = ''
        delete Memory.__segindex.clear[index]
      }
    }
    Memory.__segindex.clear = _.filter(Memory.__segindex.clear, function(a){
      return Number.isInteger(a)
    })

    // On a server without memory segments, so just keep things in buffer
    // and don't attempt to request real segments.
    if(!RawMemory.setActiveSegments) {
      return
    }

    // Flush buffer where possible, add to request where not

    // Get number of usable segments remaining.
    var currentsegments = this.getAvailableSegments().length
    var usablesegments = 10 - (currentsegments + this.free)

    // Move data from the buffers to segments.
    for(var index in Memory.__segindex.buffer) {
      var id = Number(index)

      // Is the segment active?
      if(availableSegments.indexOf(id) >= 0) {
        RawMemory.segments[id] = Memory.__segindex.buffer[index]
        delete Memory.__segindex.buffer[index]

      // Are there enough free segments to send data?
      } else if(usablesegments > 0) {
        RawMemory.segments[id] = Memory.__segindex.buffer[index]
        delete Memory.__segindex.buffer[index]
        usablesegments--

      // Are there enough free segments on the next tick to reserve some?
      } else if(segments.length < this.maxActiveSegments) {
        segments.push(id)
      }
    }

    for(var index in Memory.__segindex.clear) {
      var id = Number(index)
      if(usablesegments < 1) {
        break
      }
      RawMemory.segments[id] = ''
      delete Memory.__segindex.clear[index]
      usablesegments--
    }

    // Cache all segments in global to make reconstruction easier.
    for (var id of availableSegments) {
      var saveTick = Memory.__segindex.savelog[id]
      if(!saveTick) {
        saveTick = Game.time
        Memory.__segindex.savelog[id] = Game.time
      }
      if(!cache[saveTick]) {
        cache[saveTick] = {}
      }
      cache[saveTick][id] = RawMemory.segments[id]
    }

    // Add requested segments
    if(segments.length < this.maxActiveSegments) {
      var diff = this.maxActiveSegments - segments.length
      var reqs = Object.keys(Memory.__segindex.ttls)
      if(reqs.length > diff) {
        reqs.sort(function(a,b){
          return this.ttl[a] - this.ttl[b]
        }.bind({'ttl':Memory.__segindex.ttls}))
      }
      for(var index of reqs) {
        Memory.__segindex.ttls[index]--
        if(Memory.__segindex.ttls[index] <= 0) {
          delete Memory.__segindex.ttls[index]
          continue
        }
        segments.push(index)
        diff--
        if(diff <= 0) {
          break
        }
      }
    }

    // Add segments which needs to be cleared
    // Replace this by just injecting blind, as that works now
    if(segments.length < this.maxActiveSegments) {
      var diff = this.maxActiveSegments - segments.length
      var clear = Memory.__segindex.clear.concat({})
      if(clear.length > diff) {
        clear = clear.slice(0,diff)
      }
      segments = segments.concat(clear)
    }

    segments = _.filter(_.uniq(segments.map(Number)),function(a){return Number.isInteger(a)})
    if(segments.length > 0) {
      RawMemory.setActiveSegments(segments)
    }
  },

  getNextId: function() {
    var current = Memory.__segindex.last
    if(!current || current > this.max) {
      current = this.min
    }
    var start = current

    var inUse = []
    for(var label in Memory.__segindex.index) {
      inUse = inUse.concat(Memory.__segindex.index[label].ids)
    }
    inUse = inUse.concat(_.values(Memory.__segindex.clear))

    while(true) {
      if(inUse.indexOf(current) < 0) {
        Memory.__segindex.last = +current + +1
        return current
      }
      current++
      if(current > this.max) {
        current = this.min
      }
      if(current == start) {
        return ERR_FULL
      }
    }
  },

  getIndexByLabel: function(label, autoassign=true) {
    if(!!Memory.__segindex.index[label] && !!Memory.__segindex.index[label].ids) {
      return Memory.__segindex.index[label].ids
    }

    if(autoassign) {
      var id = this.getNextId()
      Memory.__segindex.index[label] = {'ids':[id]}
      return Memory.__segindex.index[label].ids.map(Number)
    } else {
      return ERR_NOT_FOUND
    }
  },

}


module.exports = sos_lib_segments