Created
March 14, 2018 23:08
-
-
Save pwnall/2221afba0ce0eeff9a328ca833014b52 to your computer and use it in GitHub Desktop.
Workaround for OOM when deleting a large IndexedDB database
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// Opens a connection to an IndexedDB database. | |
// | |
// @param [String] name the IDB database name | |
// @return [Promise<IDBDatabase>] a connection to the opened database | |
function OpenDatabase(name) { | |
return new Promise((resolve, reject) => { | |
const request = indexedDB.open(name); | |
request.onsuccess = () => { resolve(request.result); }; | |
request.onerror = () => { reject(request.error); }; | |
}); | |
} | |
// Internal function for reading and/or deleting data from IndexedDB. | |
// | |
// Must be called in a readwrite transaction whose scope includes the object | |
// store(s) owning the given source. Guarantees that the transaction will still | |
// be alive when the callback is called. | |
// | |
// @param [IDBObjectStore | IDBIndex] source the IDB object store or index whose | |
// values are getting deleted | |
// @param [Object] options | |
// @option options [Number] limit stop after reading this many values | |
// @option options [Boolean] deleteValues delete the read values if true | |
// @param [function(?Error, ?Number)] callback called when done; in case of | |
// success, the callback is passed the number of items that were read (and | |
// potentially deleted) | |
// @return undefined | |
function DoIterate(source, options, callback) { | |
let items_read = 0; | |
let items_left = | |
('limit' in options) ? options.limit : Number.MAX_SAFE_INTEGER; | |
const request = | |
options.deleteValues ? source.openCursor() : source.openKeyCursor(); | |
request.onsuccess = () => { | |
const cursor = request.result; | |
if (!cursor) { | |
callback(null, items_read); | |
return; | |
} | |
const delete_request = options.deleteValues ? cursor.delete() : {}; | |
delete_request.onsuccess = () => { | |
items_left -= 1; items_read += 1; | |
if (items_left === 0) { | |
callback(null, items_read); | |
return; | |
} | |
cursor.continue(); | |
}; | |
delete_request.onerror = () => { callback(request.error); }; | |
if (!options.deleteValues) | |
delete_request.onsuccess(); | |
}; | |
request.onerror = () => { callback(request.error); }; | |
} | |
// Deletes the first N items in an IDB data source and iterates over the rest. | |
// | |
// Must be called in a readwrite transaction whose scope includes the object | |
// store(s) owning the given source. Guarantees that the transaction will still | |
// be alive when the callback is called. | |
// | |
// @param [IDBObjectStore | IDBIndex] source the IDB object store or index whose | |
// values are getting deleted | |
// @param [Object] options | |
// @option options [Number] limit stop after reading this many values | |
// @option options [Boolean] deleteValues delete the read values if true | |
// @param [function(?Error, ?Boolean)] callback called when done; in case of | |
// success, the callback is passed "true" if at least one item was deleted, | |
// or "false" if no item was deleted | |
// @return undefined | |
function IterateAndDelete(source, options, callback) { | |
DoIterate(source, { limit: options.limit, deleteValues: true }, | |
(error, deleted_count) => { | |
if (error) { | |
callback(error); | |
return; | |
} | |
const has_deleted_items = (deleted_count !== 0); | |
DoIterate(source, { deleteValues: false }, (error, _) => { | |
if (error) { | |
callback(error); | |
return; | |
} | |
callback(null, has_deleted_items); | |
}); | |
}); | |
} | |
// Deletes the first N keys in each index and iterates over all the indexes. | |
// | |
// Must be called in a readwrite transaction whose scope includes the object | |
// store(s) owning the given indexes. Guarantees that the transaction will still | |
// be alive when the callback is called. | |
// | |
// @param [Array<IDBIndex>] indexes the indexes to iterate over; to ensure | |
// tombstone cleanup, this set must contain all the indexes of a object | |
// store | |
// @param [Object] options | |
// @option options [Number] limit the number of keys to delete from each index | |
// @param [function(Error, Boolean)] callback called when done; in case of | |
// success, | |
function DeleteIndexKeys(indexes, options, callback) { | |
let current_index = -1; | |
let has_deleted_items = false; | |
const Callback = (error, has_deleted_items_in_index) => { | |
has_deleted_items = has_deleted_items || has_deleted_items_in_index; | |
current_index += 1; | |
if (current_index === indexes.length) { | |
callback(null, has_deleted_items); | |
return; | |
} | |
IterateAndDelete(indexes[current_index], options, Callback); | |
}; | |
Callback(null, false); | |
} | |
// Deletes N items from an object store. | |
// | |
// This is intended to be called repeatedly when deleting an object store, to | |
// avoid generating a single IndexedDB transaction that's too large to fit in | |
// memory. | |
// | |
// @param [IDBDatabase] database the database owning the object store | |
// @param [string] object_store_name the name of the target object store | |
// @param [Object] options | |
// @option options [number] limit the maximum number of items to delete | |
// @option options [Boolean] drain_indexes if true, deletes items via indexes; | |
// otherwise, deletes item by iterating over the object store | |
// @return [Promise<Boolean>] resolves with true if at least one item was | |
// deleted, and false otherwise | |
function DrainObjectStore(database, object_store_name, options) { | |
return new Promise((resolve, reject) => { | |
let has_deleted_items = false; | |
const transaction = database.transaction([object_store_name], 'readwrite'); | |
transaction.oncomplete = () => { resolve(has_deleted_items); }; | |
transaction.onerror = () => { reject(transaction.error); }; | |
const object_store = transaction.objectStore(object_store_name); | |
const indexes = Array.from(object_store.indexNames).map( | |
(name) => object_store.index(name)); | |
const per_index_limit = | |
(options.drain_indexes && indexes.length !== 0) ? | |
options.limit / indexes.length : 0; | |
DeleteIndexKeys( | |
indexes, { limit: per_index_limit }, (error, call_deleted_items) => { | |
if (error) { | |
reject(error); | |
return; | |
} | |
has_deleted_items = call_deleted_items; | |
if (has_deleted_items) { | |
// Let the transaction commit. Enough work was done. | |
return; | |
} | |
DoIterate( | |
object_store, { limit: options.limit }, (error, deleted_count) => { | |
if (error) { | |
reject(error); | |
return; | |
} | |
has_deleted_items = (deleted_count !== 0); | |
// Let the transaction commit. | |
}); | |
}); | |
}); | |
} | |
// Deletes the contents of an IndexedDB object store. | |
// | |
// The deletion is done in many small transactions, to avoid generating a single | |
// IndexedDB transaction that's too large to fit in memory. Does not delete the | |
// object store itself. | |
// | |
// @param [IDBDatabase] database the database owning the object store | |
// @param [string] object_store_name the name of the target object store | |
// @param [Object] options | |
// @option options [number] step_limit maximum number of items to delete in a | |
// single transaction | |
// @return [Promise<>] resolves when the data inside the object store is deleted | |
async function DeleteObjectStore(database, object_store_name, options) { | |
while (true) { | |
const seen_items = await DrainObjectStore( | |
database, object_store_name, | |
{ limit: options.step_limit, drain_indexes: true }); | |
if (!seen_items) | |
break; | |
} | |
// Delete any items that may not be listed in indexes. | |
while (true) { | |
const seen_items = await DrainObjectStore( | |
database, object_store_name, | |
{ limit: options.step_limit, drain_indexes: false }); | |
if (!seen_items) | |
break; | |
} | |
} | |
// Deletes the contents of an IndexedDB database. | |
// | |
// The deletion is done in many small transactions, to avoid generating a single | |
// IndexedDB transaction that's too large to fit in memory. Does not delete the | |
// database's object stores or indexes. | |
// | |
// @param [IDBDatabase] database the database to be deleted | |
// @return [Promise<>] resolves when the data inside the database is deleted | |
async function DeleteDatabase(database, options) { | |
for (let object_store_name of database.objectStoreNames) | |
await DeleteObjectStore(database, object_store_name, options); | |
} | |
// Driver for testing the code above. | |
(async () => { | |
const database = await OpenDatabase('messages0'); | |
await DeleteDatabase(database, { step_limit: 10 }); | |
database.close(); | |
console.log('Database drained'); | |
})(); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment