Skip to content

Instantly share code, notes, and snippets.

@ebrensi
Last active Oct 29, 2020
Embed
What would you like to do?
Some mods to jakearchibald/idb-keyval that make it faster

idb-keyval

idb-keyval is a tiny TypeScript library by Jake Archibald that simplifies usage of IndexedDB as a client-side key-value store. His repo includes a few JavaScript builds for different use cases.

idb-keyval-alt.js

I submitted this as a pull-request (see #91) but it appears not much is happening with that repo right now, so here is the essential part of my pull request for anyone to use.

The original idb-keyval opens a new transaction for every (get, set, or del) operation, which is inefficient for performing lots of opertations. One alternative is to use a more full-featured library like IDB or Dexie, which feature batch operations that perform several in the same transaction. This mod maintains the original idb-keyval API, but does batch operations behind the scene and results in much better performance. The way it works is by keeping a buffer of operations, and then performing them all as a batch if no new operations are added after BUFFER_TIMEOUT=60 milliseconds, or the buffer reaches MAX_TRANSACTION_SIZE=200 operations.

I have only included es6-module source code here, which works great if you're using a bundler. If you are unfamiliar with es6 modules, and doing it old-school (including script tags in HTML) then leave a comment and I'll try to help you include it in your project.

I have been using this in a development version of my app (https://www.heatflask.com) for caching map tiles, and it works great. If you know of any use cases where this might fail, please let me know in the comments.

const BUFFER_TIMEOUT = 60
const MAX_TRANSACTION_SIZE = 200
export class Store {
constructor(dbName, storeName, keyPath) {
this.storeName = storeName
this._dbp = this._initialize(dbName, storeName, keyPath)
}
_initialize(dbName, storeName, keyPath, version) {
return new Promise((resolve, reject) => {
let openreq = indexedDB.open(dbName, version),
db
const self = this
openreq.onerror = onerror
openreq.onupgradeneeded = onupgradeneeded
openreq.onsuccess = onsuccess
function onupgradeneeded(e) {
// console.log(`"${dbName}" onupgradeneeded: adding "${storeName}"`, e);
db = e.target.result
if (keyPath === undefined) db.createObjectStore(storeName)
else db.createObjectStore(storeName, { keyPath: keyPath })
}
function onerror(event) {
console.log(`"${dbName}" error`, event.target.error)
reject(event)
}
function onsuccess(event) {
// console.log(`"${dbName}" success`, event)
db = event.target.result
db.onversionchange = onversionchange
if (db.objectStoreNames.contains(storeName)) resolve(db)
else {
// console.log(`"${storeName}" not in "${dbName}". attempting upgrade...`)
upgrade()
}
}
function onversionchange() {
// console.log(`"${dbName}" versionchange`);
upgrade()
}
function upgrade() {
const v = +db.version
db.close()
self._dbp = self._initialize(dbName, storeName, keyPath, v + 1)
self._dbp.then((db) => resolve(db))
}
})
}
_withIDBStore(type, callback) {
return this._dbp.then(
(db) =>
new Promise((resolve, reject) => {
const transaction = db.transaction(this.storeName, type)
transaction.oncomplete = () => resolve()
transaction.onabort = transaction.onerror = () =>
reject(transaction.error)
callback(transaction.objectStore(this.storeName))
})
)
}
close() {
return this._dbp.then((db) => db.close())
}
}
const pendingTransactions = new Map
function doBulkGet(store) {
// const t0 = Date.now()
// let count = 0
return store
._withIDBStore("readonly", (thisTransactionObjectStore) => {
// We do one transaction involving queries.count operations
const queries = pendingTransactions.get(store)
if (!queries || !queries.count) return
const getQueries = queries.get
// clear this set of pending transactions so no one will add to it
queries.get = {}
queries.count.get = 0
// make all the get requests
for (const [key, {resolve, reject}] of Object.entries(getQueries)) {
const req = thisTransactionObjectStore.get(key)
req.onsuccess = (e) => resolve(e.target.result)
req.onerror = (e) => reject(e)
// count++
}
})
.then(() => {
// console.log(`${store.storeName}: got ${count} in ${Date.now()-t0}ms`)
})
}
function doBulkPutDel(store) {
// const t0 = Date.now()
// let delCount = 0
// let putCount = 0
return store
._withIDBStore("readwrite", (thisTransactionObjectStore) => {
// We do one transaction involving queries.count operations
const queries = pendingTransactions.get(store)
if (!queries || !queries.count) return
const putQueries = queries.put
const delQueries = queries.del
// delete this set of pending transactions so no one will add to them
queries.put = {}
queries.del = {}
queries.count.put = 0
queries.count.del = 0
// make all put and del requests
for (const [key, {value, resolve, reject}] of Object.entries(putQueries)) {
const req = thisTransactionObjectStore.put(value, key)
req.onsuccess = () => resolve()
req.onerror = (e) => reject(e)
// putCount++
}
for (const [key, {resolve, reject}] of Object.entries(delQueries)) {
const req = thisTransactionObjectStore.delete(key)
req.onsuccess = () => resolve()
req.onerror = (e) => reject(e)
// delCount++
}
})
.then(() => {
// console.log(`${store.storeName}: put ${putCount}, del ${delCount} in ${Date.now()-t0}ms`)
})
}
// Rather than opening a new transaction for each get operation, we
// store the operations into a buffer, and perform a batch operation in
// one transaction when nothing has been added to the buffer for
function addOp(store, op, key, value) {
return new Promise((resolve, reject) => {
if (!pendingTransactions.has(store)) {
pendingTransactions.set(store,
{get: {}, put: {}, del: {}, count: {get: 0, put: 0, del: 0}})
}
const queries = pendingTransactions.get(store)
queries[op][key] = {value, resolve, reject}
const myNum = queries.count[op]++
/*
* Here we set a short timeout to allow for more ops to be added.
* when we come back, if nothing has been added then we go ahead and
* perform a bulk transaction.
*/
setTimeout(() => {
if (op === "get") {
const getCount = queries.count.get
if (getCount > MAX_TRANSACTION_SIZE || getCount === myNum + 1) {
doBulkGet(store)
}
return
}
const putDelCount = queries.count.put + queries.count.del
if (putDelCount > MAX_TRANSACTION_SIZE || queries.count[op] === myNum + 1) {
doBulkPutDel(store)
}
}, BUFFER_TIMEOUT)
})
}
export function get(key, store) {
return addOp(store, "get", key)
}
export function set(key, value, store) {
return addOp(store, "put", key, value)
}
export function del(key, store) {
return addOp(store, "del", key)
}
export function clear(store) {
return store._withIDBStore("readwrite", (store) => {
store.clear()
})
}
export function keys(store) {
const keys = []
return store
._withIDBStore("readonly", (store) => {
// This would be store.getAllKeys(), but it isn't supported by Edge or Safari.
// And openKeyCursor isn't supported by Safari.
;(store.openKeyCursor || store.openCursor).call(
store
).onsuccess = function () {
if (!this.result) return
keys.push(this.result.key)
this.result.continue()
}
})
.then(() => keys)
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment