Skip to content

Instantly share code, notes, and snippets.

@pwnall
Created March 14, 2018 23:08
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save pwnall/2221afba0ce0eeff9a328ca833014b52 to your computer and use it in GitHub Desktop.
Save pwnall/2221afba0ce0eeff9a328ca833014b52 to your computer and use it in GitHub Desktop.
Workaround for OOM when deleting a large IndexedDB database
// Opens a connection to an IndexedDB database.
//
// @param [String] name the IDB database name
// @return [Promise<IDBDatabase>] a connection to the opened database
function OpenDatabase(name) {
return new Promise((resolve, reject) => {
const request = indexedDB.open(name);
request.onsuccess = () => { resolve(request.result); };
request.onerror = () => { reject(request.error); };
});
}
// Internal function for reading and/or deleting data from IndexedDB.
//
// Must be called in a readwrite transaction whose scope includes the object
// store(s) owning the given source. Guarantees that the transaction will still
// be alive when the callback is called.
//
// @param [IDBObjectStore | IDBIndex] source the IDB object store or index whose
// values are getting deleted
// @param [Object] options
// @option options [Number] limit stop after reading this many values
// @option options [Boolean] deleteValues delete the read values if true
// @param [function(?Error, ?Number)] callback called when done; in case of
// success, the callback is passed the number of items that were read (and
// potentially deleted)
// @return undefined
function DoIterate(source, options, callback) {
let items_read = 0;
let items_left =
('limit' in options) ? options.limit : Number.MAX_SAFE_INTEGER;
const request =
options.deleteValues ? source.openCursor() : source.openKeyCursor();
request.onsuccess = () => {
const cursor = request.result;
if (!cursor) {
callback(null, items_read);
return;
}
const delete_request = options.deleteValues ? cursor.delete() : {};
delete_request.onsuccess = () => {
items_left -= 1; items_read += 1;
if (items_left === 0) {
callback(null, items_read);
return;
}
cursor.continue();
};
delete_request.onerror = () => { callback(request.error); };
if (!options.deleteValues)
delete_request.onsuccess();
};
request.onerror = () => { callback(request.error); };
}
// Deletes the first N items in an IDB data source and iterates over the rest.
//
// Must be called in a readwrite transaction whose scope includes the object
// store(s) owning the given source. Guarantees that the transaction will still
// be alive when the callback is called.
//
// @param [IDBObjectStore | IDBIndex] source the IDB object store or index whose
// values are getting deleted
// @param [Object] options
// @option options [Number] limit stop after reading this many values
// @option options [Boolean] deleteValues delete the read values if true
// @param [function(?Error, ?Boolean)] callback called when done; in case of
// success, the callback is passed "true" if at least one item was deleted,
// or "false" if no item was deleted
// @return undefined
function IterateAndDelete(source, options, callback) {
DoIterate(source, { limit: options.limit, deleteValues: true },
(error, deleted_count) => {
if (error) {
callback(error);
return;
}
const has_deleted_items = (deleted_count !== 0);
DoIterate(source, { deleteValues: false }, (error, _) => {
if (error) {
callback(error);
return;
}
callback(null, has_deleted_items);
});
});
}
// Deletes the first N keys in each index and iterates over all the indexes.
//
// Must be called in a readwrite transaction whose scope includes the object
// store(s) owning the given indexes. Guarantees that the transaction will still
// be alive when the callback is called.
//
// @param [Array<IDBIndex>] indexes the indexes to iterate over; to ensure
// tombstone cleanup, this set must contain all the indexes of a object
// store
// @param [Object] options
// @option options [Number] limit the number of keys to delete from each index
// @param [function(Error, Boolean)] callback called when done; in case of
// success,
function DeleteIndexKeys(indexes, options, callback) {
let current_index = -1;
let has_deleted_items = false;
const Callback = (error, has_deleted_items_in_index) => {
has_deleted_items = has_deleted_items || has_deleted_items_in_index;
current_index += 1;
if (current_index === indexes.length) {
callback(null, has_deleted_items);
return;
}
IterateAndDelete(indexes[current_index], options, Callback);
};
Callback(null, false);
}
// Deletes N items from an object store.
//
// This is intended to be called repeatedly when deleting an object store, to
// avoid generating a single IndexedDB transaction that's too large to fit in
// memory.
//
// @param [IDBDatabase] database the database owning the object store
// @param [string] object_store_name the name of the target object store
// @param [Object] options
// @option options [number] limit the maximum number of items to delete
// @option options [Boolean] drain_indexes if true, deletes items via indexes;
// otherwise, deletes item by iterating over the object store
// @return [Promise<Boolean>] resolves with true if at least one item was
// deleted, and false otherwise
function DrainObjectStore(database, object_store_name, options) {
return new Promise((resolve, reject) => {
let has_deleted_items = false;
const transaction = database.transaction([object_store_name], 'readwrite');
transaction.oncomplete = () => { resolve(has_deleted_items); };
transaction.onerror = () => { reject(transaction.error); };
const object_store = transaction.objectStore(object_store_name);
const indexes = Array.from(object_store.indexNames).map(
(name) => object_store.index(name));
const per_index_limit =
(options.drain_indexes && indexes.length !== 0) ?
options.limit / indexes.length : 0;
DeleteIndexKeys(
indexes, { limit: per_index_limit }, (error, call_deleted_items) => {
if (error) {
reject(error);
return;
}
has_deleted_items = call_deleted_items;
if (has_deleted_items) {
// Let the transaction commit. Enough work was done.
return;
}
DoIterate(
object_store, { limit: options.limit }, (error, deleted_count) => {
if (error) {
reject(error);
return;
}
has_deleted_items = (deleted_count !== 0);
// Let the transaction commit.
});
});
});
}
// Deletes the contents of an IndexedDB object store.
//
// The deletion is done in many small transactions, to avoid generating a single
// IndexedDB transaction that's too large to fit in memory. Does not delete the
// object store itself.
//
// @param [IDBDatabase] database the database owning the object store
// @param [string] object_store_name the name of the target object store
// @param [Object] options
// @option options [number] step_limit maximum number of items to delete in a
// single transaction
// @return [Promise<>] resolves when the data inside the object store is deleted
async function DeleteObjectStore(database, object_store_name, options) {
while (true) {
const seen_items = await DrainObjectStore(
database, object_store_name,
{ limit: options.step_limit, drain_indexes: true });
if (!seen_items)
break;
}
// Delete any items that may not be listed in indexes.
while (true) {
const seen_items = await DrainObjectStore(
database, object_store_name,
{ limit: options.step_limit, drain_indexes: false });
if (!seen_items)
break;
}
}
// Deletes the contents of an IndexedDB database.
//
// The deletion is done in many small transactions, to avoid generating a single
// IndexedDB transaction that's too large to fit in memory. Does not delete the
// database's object stores or indexes.
//
// @param [IDBDatabase] database the database to be deleted
// @return [Promise<>] resolves when the data inside the database is deleted
async function DeleteDatabase(database, options) {
for (let object_store_name of database.objectStoreNames)
await DeleteObjectStore(database, object_store_name, options);
}
// Driver for testing the code above.
(async () => {
const database = await OpenDatabase('messages0');
await DeleteDatabase(database, { step_limit: 10 });
database.close();
console.log('Database drained');
})();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment