Dexie TrackedTransaction addon

README.md

This is an addon for Dexie v3+ that adds a new db.trackedTransaction() method, using DBCore middleware.

trackedTransaction() is called just like transaction(), but takes an additional report function that is called with a list of TrackedChange records once the transaction is ready to commit (but before it actually commits).

See additional docs in the module declaration, and examples in the test file.

trackedTransaction.test.ts

import Dexie, {TrackedChange} from 'dexie';
import {TrackedTransaction} from './trackedTransaction';


describe(`trackedTransaction`, () => {
  class DexieTest extends Dexie {
    table1: Dexie.Table<object, number>;
    table2: Dexie.Table<object>;
    log: Dexie.Table<object, number>;

    constructor() {
      super('test', {addons: [TrackedTransaction]});

      this.version(1)
        .stores({
          table1: 'id', // inbound primary key
          table2: '', // non-inbound primary key
          log: '++id', // autoIncrement primary key
        });

      this.table1 = this.table('table1');
      this.table2 = this.table('table2');
      this.log = this.table('log');
    }
  }

  let db: DexieTest;
  beforeEach(() => {
    db = new DexieTest();
  });

  // Tracking basics

  describe(`tracking`, () => {

    it(`tracks changes in a transaction`, async (done) => {
      await db.trackedTransaction('rw', [db.table1, db.table2],
        async () => {
          await db.table1.add({id: 1, a: '1'});
          await db.table1.put({id: 1, a: '2'});
          await db.table2.put({b: '3'}, 'theKey'); // external key
          await db.table1.delete(1);
        },
        (changes) => {
          expect(changes).toEqual<TrackedChange[]>([
            {tableName: 'table1', type: 'add', keys: [1], before: [undefined], after: [{id: 1, a: '1'}]},
            {tableName: 'table1', type: 'put', keys: [1], before: [{id: 1, a: '1'}], after: [{id: 1, a: '2'}]},
            {tableName: 'table2', type: 'put', keys: ['theKey'], before: [undefined], after: [{b: '3'}]},
            {tableName: 'table1', type: 'delete', keys: [1], before: [{id: 1, a: '2'}], after: [undefined]},
          ]);
          done();
        });
    });

    it(`tracks changes in bulk operations`, async(done) => {
      await db.trackedTransaction('rw', [db.table1, db.table2],
        async () => {
          await db.table1.bulkAdd([{id: 1}, {id: 2}, {id: 3}]);
          await db.table1.bulkPut([{id: 2, a: '2'}, {id: 4, a: '4'}]);
          await db.table2.bulkPut([{b: '1'}, {b: '3'}], ['key1', 'key3']); // external key
          await db.table1.bulkDelete([1, 3]);
        },
        async (changes) => {
          expect(changes).toEqual<TrackedChange[]>([
            {tableName: 'table1', type: 'add', keys: [1, 2, 3],
              before: [undefined, undefined, undefined], after: [{id: 1}, {id: 2}, {id: 3}]},
            {tableName: 'table1', type: 'put', keys: [2, 4],
              before: [{id: 2}, undefined], after: [{id: 2, a: '2'}, {id: 4, a: '4'}]},
            {tableName: 'table2', type: 'put', keys: ['key1', 'key3'],
              before: [undefined, undefined], after: [{b: '1'}, {b: '3'}]},
            {tableName: 'table1', type: 'delete', keys: [1, 3],
              before: [{id: 1}, {id: 3}], after: [undefined, undefined]},
          ]);
          await expect(db.table1.toArray()).resolves
            .toEqual([{id: 2, a: '2'}, {id: 4, a: '4'}]);
          done();
        });
    });

    it(`allows additional (untracked) operations in report`, async () => {
      await db.trackedTransaction('rw', [db.table1, db.log],
        () => db.table1.add({id: 1}),
        async (changes) => {
          expect(changes).toBeArrayOfSize(1);
          // Add to log table. Note that this table must be included in the transaction.
          // Also, note that although this table has an autoIncrement primary key, it's
          // OK to add to it here (because changes are no longer being tracked).
          await db.log.add({time: 'now', ...changes[0]});
          expect(changes).toBeArrayOfSize(1); // previous line didn't push any change
        });
      await expect(await db.log.toArray()).toEqual([{
        id: 1, time: 'now', tableName: 'table1', type: 'add',
        keys: [1], before: [undefined], after: [{id: 1}]
      }]);
    });

    it(`correctly handles Collection.modify`, async (done) => {
      // This is really just checking that Collection.modify(fn)
      // turns into one of the DBCore mutations we already handle.
      await db.table1.bulkAdd([{id: 1, a: 1}, {id: 2, a: 2}]);
      await db.trackedTransaction('rw', [db.table1],
        () => db.table1.toCollection().modify(
          (obj: {a: number}) => obj.a += 1),
        (changes) => {
          expect(changes).toEqual<TrackedChange[]>([{
            tableName: 'table1', type: 'put', keys: [1, 2],
            before: [{id: 1, a: 1}, {id: 2, a: 2}],
            after: [{id: 1, a: 2}, {id: 2, a: 3}]
          }]);
          done();
        });
    });

    it(`handles autoIncrement primary keys`, async (done) => {
      await db.trackedTransaction('rw', [db.log],
        async () => {
          const key1 = await db.log.add({a: 1});
          await db.log.put({id: key1, a: 2});
          await db.log.bulkPut([{id: key1, a: 3}, {b: 1}]);
          await db.log.delete(key1);
        },
        (changes) => {
          expect(changes).toEqual<TrackedChange[]>([
            {tableName: 'log', type: 'add', keys: [1], before: [undefined], after: [{id: 1, a: 1}]},
            {tableName: 'log', type: 'put', keys: [1], before: [{id: 1, a: 1}], after: [{id: 1, a: 2}]},
            {tableName: 'log', type: 'put', keys: [1, 2],
              before: [{id: 1, a: 2}, undefined], after: [{id: 1, a: 3}, {id: 2, b: 1}]},
            {tableName: 'log', type: 'delete', keys: [1], before: [{id: 1, a: 3}], after: [undefined]},
          ]);
          done();
        });
    });

  });

  // Failures and rollbacks
  describe(`failures and rollbacks`, () => {

    it(`doesn't report failures handled within the transaction scope`, async (done) => {
      await db.trackedTransaction('rw', [db.table1],
        async () => {
          await db.table1.add({id: 1});
          try {
            await db.table1.add({id: 1, error: true}); // this causes error...
          } catch (err) { // ... but catching it allows transaction to continue
            await db.table1.put({id: 1, caught: true});
          }
          try {
            // Only the middle of these three items causes a failure...
            await db.table1.bulkAdd([{id: 2}, {id: 1, bulkError: true}, {id: 3}]);
          } catch (err) {
            // ... and if you catch it, the other two adds will succeed
          }
        },
        async (changes) => {
          await expect(await db.table1.toArray()).toEqual([{id: 1, caught: true}, {id: 2}, {id: 3}]);
          expect(changes).toEqual<TrackedChange[]>([
            {tableName: 'table1', type: 'add', keys: [1], before: [undefined], after: [{id: 1}]},
            {tableName: 'table1', type: 'put', keys: [1], before: [{id: 1}], after: [{id: 1, caught: true}]},
            {tableName: 'table1', type: 'add', keys: [2, 3], before: [undefined, undefined], after: [{id: 2}, {id: 3}]},
          ]);
          done();
        });
    });

    it(`doesn't report any changes when scope fails`, async () => {
      const report = jest.fn();
      await db.table1.add({id: 1});
      const op = db.trackedTransaction('rw', [db.table1],
        () => db.table1.add({id: 1}), // add duplicate should fail
        report);
      await expect(op).rejects.toThrow(Dexie.ConstraintError);
      expect(report).not.toHaveBeenCalled();
    });

    it(`rolls back scope changes when report errors`, async () => {
      await db.table1.add({id: 1, updated: 'no'});
      const op = db.trackedTransaction('rw', [db.table1],
        () => db.table1.put({id: 1, updated: 'yes'}),
        async () => {
          // Updated within the transaction
          await expect(db.table1.get(1)).resolves.toEqual({id: 1, updated: 'yes'});
          throw new Error("Fail in report");
        });
      await expect(op).rejects.toThrow("Fail in report");
      // Update from scope rolled back:
      await expect(db.table1.get(1)).resolves.toEqual({id: 1, updated: 'no'});
    });

    it(`rolls back scope changes when report fails`, async () => {
      // This is a slight variation on the previous test,
      // causing an IDB violation in report, rather than explicitly throwing.
      await db.table1.add({id: 1, updated: 'no'});
      const op = db.trackedTransaction('rw', [db.table1],
        () => db.table1.put({id: 1, updated: 'yes'}),
        () => db.table1.add({id: 1})); // constraint error
      await expect(op).rejects.toThrow(Dexie.ConstraintError);
      // Update from scope rolled back:
      await expect(db.table1.get(1)).resolves.toEqual({id: 1, updated: 'no'});
    });

  });

  // Nesting
  describe(`nesting`, () => {

    it(`tracks nested transactions`, async (done) => {
      await db.trackedTransaction('rw', [db.table1],
        async () => {
          await db.table1.add({id: 1});
          await db.transaction('rw', db.table1, async() => {
            await db.table1.put({id: 1, a: 'A'});
            await db.table1.put({id: 2});
          });
          await db.table1.delete(1);
        },
        (changes) => {
          expect(changes).toEqual<TrackedChange[]>([
            {tableName: 'table1', type: 'add', keys: [1], before: [undefined], after: [{id: 1}]},
            {tableName: 'table1', type: 'put', keys: [1], before: [{id: 1}], after: [{id: 1, a: 'A'}]},
            {tableName: 'table1', type: 'put', keys: [2], before: [undefined], after: [{id: 2}]},
            {tableName: 'table1', type: 'delete', keys: [1], before: [{id: 1, a: 'A'}], after: [undefined]},
          ]);
          done();
        });
    });

    it(`can be nested within a regular transaction`, async (done) => {
      await db.transaction('rw', [db.table1, db.log], async () => {
        let changes: ReadonlyArray<TrackedChange> = [];
        await db.trackedTransaction('rw', [db.table1],
          async () => db.table1.add({id: 1}),
          (chg) => { changes = chg; });
        await db.log.add({log: true});
        expect(changes).toEqual<TrackedChange[]>([
          {tableName: 'table1', type: 'add', keys: [1], before: [undefined], after: [{id: 1}]},
        ]);
        done();
      });
    });

    it(`cannot be nested within itself`, async () => {
      const op = db.trackedTransaction('rw', [db.table1],
        async () => {
          await db.table1.add({id: 1});
          await db.trackedTransaction('rw', [db.table1],
            () => db.table1.add({id: 2}),
            (_changes) => null);
        },
        (_changes) => null);
      await expect(op).rejects.toThrow("trackedTransaction may not be nested");
    });

  });

  // Limitations (most of these could be fixed if needed)
  describe(`known limitations`, () => {

    it(`doesn't currently apply mapToClass to reported values`, async (done) => {
      // You can use mapToClass with trackedTransaction, but the before/after values
      // in your changes callback won't be mapped
      class FancyObject {
        constructor(readonly id: number) {}
      }
      db.table1.mapToClass(FancyObject);

      await db.trackedTransaction('rw', [db.table1],
        async () => {
          await db.table1.add(new FancyObject(22));
          await expect(await db.table1.get(22)).toBeInstanceOf(FancyObject); // mapToClass is working
          await db.table1.put({id: 22, modified: true});
          await db.table1.put(new FancyObject(15));
          await db.table1.delete(22);
        },
        async (changes) => {
          expect(changes).toBeArrayOfSize(4);

          expect(changes[0].after[0]).toEqual({id: 22});
          expect(changes[0].after[0]).not.toBeInstanceOf(FancyObject);

          expect(changes[1].before[0]).toEqual({id: 22});
          expect(changes[1].before[0]).not.toBeInstanceOf(FancyObject);
          expect(changes[1].after[0]).toEqual({id: 22, modified: true});
          expect(changes[1].after[0]).not.toBeInstanceOf(FancyObject);

          expect(changes[2].after[0]).toEqual({id: 15});
          expect(changes[2].after[0]).not.toBeInstanceOf(FancyObject);

          expect(changes[3].before[0]).toEqual({id: 22, modified: true});
          expect(changes[3].before[0]).not.toBeInstanceOf(FancyObject);

          done();
        });
    });

    it(`doesn't currently support deleteRange`, async () => {
      // deleteRange is the underlying operation used for Table.clear() and
      // Table.where(primaryKey).FILTER(...).delete() for the FILTERs above,
      // below, between, and inAnyRange.
      const op = db.trackedTransaction('rw', [db.table1],
        () => db.table1.where('id').above(0).delete(),
        (_changes) => null);
      await expect(op).rejects.toThrow(
        "trackedTransaction cannot handle deleteRange operation (on table 'table1')");
    });

  });

  it(`doesn't cross the streams on concurrent transactions`, async () => {
    // Transactions in different tables (with "non-overlapping scopes")
    // may run concurrently.

    // https://dexie.org/docs/Dexie/Dexie.waitFor()#sample-1
    function sleep(ms: number) {
      return new Dexie.Promise(resolve => setTimeout(resolve, ms));
    }

    let changes1: ReadonlyArray<TrackedChange> = [];
    let changes2: ReadonlyArray<TrackedChange> = [];
    const sequence: string[] = []; // track operation order
    const tx1 = db.trackedTransaction('rw', [db.table1],
      async () => {
        sequence.push('tx1: adding 1');
        await db.table1.add({id: 1});
        sequence.push('tx1: added 1');
        // await Dexie.waitFor(tx2); // doesn't work - deadlock
        await Dexie.waitFor(sleep(10));
        sequence.push('tx1: adding 2');
        await db.table1.add({id: 2});
        sequence.push('tx1: added 2');
      },
      changes => { changes1 = changes; });
    const tx2 = db.trackedTransaction('rw', [db.table2],
      async () => {
        sequence.push('tx2: start');
        await Dexie.waitFor(sleep(5));
        sequence.push('tx2: adding 3');
        await db.table2.add({}, 3);
        sequence.push('tx2: added 3');
      },
      changes => { changes2 = changes; });

    await Dexie.Promise.all([tx1, tx2]);
    // Make sure the changes were not scrambled between transactions:
    expect(changes1).toEqual([
      {tableName: 'table1', type: 'add', keys: [1], before: [undefined], after: [{id: 1}]},
      {tableName: 'table1', type: 'add', keys: [2], before: [undefined], after: [{id: 2}]},
    ]);
    expect(changes2).toEqual([
      {tableName: 'table2', type: 'add', keys: [3], before: [undefined], after: [{}]},
    ]);

    // Check that the transactions ran concurrently:
    expect(sequence).toEqual([
      "tx1: adding 1",
      "tx2: start",
      "tx1: added 1",
      // This is what we're really expecting, but fake-indexeddb
      // is for some reason delaying tx2 requests until tx1 finishes:
      //   "tx2: adding 3",
      //   "tx2: added 3",
      "tx1: adding 2",
      "tx1: added 2",
      "tx2: adding 3",
      "tx2: added 3",
    ]);
  });

});

trackedTransaction.ts

/**
 * Dexie add-on that provides DB-level change tracking.
 */

import Dexie, {
  DBCore,
  DBCoreAddRequest,
  DBCoreDeleteRequest,
  DBCoreIndex,
  DBCoreMutateRequest,
  DBCoreMutateResponse,
  DBCorePutRequest,
  DBCoreTable,
  Middleware,
  PromiseExtended,
  TrackedChange, // (this is imported from the ambient declaration below)
  TrackedChangeList,
  Transaction,
  TransactionMode,
} from 'dexie';


declare module 'dexie' {
  export interface Dexie {
    /**
     * Like Dexie.transaction, but also accumulates a list of changes made
     * within the transaction scope function. Once that is complete--but
     * before the transaction is committed--calls report with the list
     * of changes.
     *
     * Report may perform additional DB operations within the transaction
     * (such as recording changes to a log table). Changes during report are
     * not tracked. If report fails the entire transaction will roll back.
     *
     * The report function has the same "transaction scope" (zone)
     * considerations as the scope function. (See Dexie.transaction.)
     *
     * Provided by the TrackedTransaction Dexie add-on.
     */
    trackedTransaction<U>(
      mode: TransactionMode,
      tables: Dexie.Table[] | string[],
      scope: (trans: Transaction) => PromiseLike<U> | U,
      report: (changes: ReadonlyArray<TrackedChange>, trans: Transaction) => PromiseLike<any> | any,
    ): PromiseExtended<U>;
  }


  export interface TrackedChange<T = any, Key = any> {
    tableName: string;

    /**
     * Fundamental type of the transaction
     * (a subset of DBCoreMutateRequest.type)
     */
    type: 'add' | 'put' | 'delete';

    /**
     * List of keys affected by this change.
     * (keys.length === before.length === after.length, always.)
     */
    keys: Key[];

    /**
     * Values stored at each corresponding key, before the change.
     * For 'add', will be list of `undefined`.
     */
    before: T[] | undefined[];

    /**
     * Values stored at each corresponding key, after the change.
     * For 'delete', will be list of `undefined`.
     * Does *not* include any auto-generated keys.
     */
    after: T[] | undefined[];
  }

  export type TrackedChangeList = Array<TrackedChange>;

  export interface Transaction {
    /**
     * Within a trackedTransaction scope (but not report), this list of
     * changes accumulated so far. Undefined outside trackedTransaction scope.
     *
     * (Should only be modified by the TrackedTransaction add-on.)
     */
    trackedChanges?: TrackedChangeList;
  }

}


/**
 * Dexie add-on that provides DB-level change tracking
 * via a new Dexie.trackedTransaction function (see docs above).
 */
export const TrackedTransaction = (db: Dexie) => {
  db.use(trackedTransactionMiddleware);

  db.trackedTransaction = function <U>(
    mode: TransactionMode,
    tables: Dexie.Table[], // | string[],
    scope: (trans: Transaction) => PromiseLike<U> | U,
    report: (changes: ReadonlyArray<TrackedChange>, trans: Transaction) => PromiseLike<any> | any,
  ): PromiseExtended<U> {
    return db.transaction<U>(mode, tables, async (tx) => {
      if (getTrackedChangeList(tx)) {
        throw new Error(`trackedTransaction may not be nested`);
      }
      tx.trackedChanges = []; // enable trackedTransactionMiddleware for this transaction
      const result = await scope(tx);
      const changes = tx.trackedChanges;
      delete tx.trackedChanges; // don't track changes during report
      await report(changes, tx);
      return result;
    });
  };
  return db;
}


// Dexie's built-in hooks middleware is helpful in figuring out how to implement this:
// https://github.com/dfahlander/Dexie.js/blob/v3.0.1/src/hooks/hooks-middleware.ts

const trackedTransactionMiddleware: Middleware<DBCore> = {
  stack: 'dbcore',
  name: 'TrackedTransactionMiddleware',
  create: (dbCore: DBCore) => ({
    ...dbCore,
    table(tableName: string): DBCoreTable {
      const dbCoreTable = dbCore.table(tableName);
      return {
        ...dbCoreTable,
        async mutate(req: DBCoreMutateRequest): Promise<DBCoreMutateResponse> {
          const trackedChangeList = getTrackedChangeList(Dexie.currentTransaction); // ??? or req.trans?
          if (trackedChangeList === undefined) {
            // Not tracking; no need to do all the extra work below.
            return dbCoreTable.mutate(req);
          }
          if (req.type !== 'add' && req.type !== 'put' && req.type !== 'delete') {
            // The only other type (as of now) is deleteRange.
            // If you needed deleteRange tracking, you could probably enumerate
            // the existing keys in the range somehow, and then continue as below.
            throw new Error(`trackedTransaction cannot handle ${req.type} operation (on table '${tableName}')`);
          }

          // ??? Does this need to run in a special promise (like hooks middleware uses)?
          //     (PSD.trans as Transaction)._promise('readwrite', () => { code below })
          const {primaryKey} = dbCoreTable.schema;
          let keys = getEffectiveKeys(primaryKey, req);
          let before = await getExistingValues(dbCoreTable, req, keys);

          if (primaryKey.autoIncrement && (req.type === 'add' || req.type === 'put')) {
            req = {...req, wantResults: true};
          }
          const response = await dbCoreTable.mutate(req);
          if (primaryKey.autoIncrement && (req.type === 'add' || req.type === 'put')) {
            if (response.results === undefined) {
              throw new Error(`autoIncrement keys not returned from ${req.type} mutation`);
            }
            keys = response.results;
          }
          // This could be optimized (just use undefined[] for delete ops,
          // use req.values for add/put if not primaryKey.autoIncrement, etc.)
          let after = await dbCoreTable.getMany({trans: req.trans, keys});

          const {failures, numFailures} = response;
          if (numFailures > 0) {
            // Remove failed items
            keys = keys.filter((_key, i) => !failures[i]);
            before = before.filter((_obj, i) => !failures[i]);
            after = after.filter((_obj, i) => !failures[i]);
          }
          if (keys.length > 0) {
            trackedChangeList.push({tableName, type: req.type, keys, before, after});
          }
          return response;
        }
      };
    }
  })
};


function getTrackedChangeList(trans: Transaction): TrackedChangeList | undefined {
  let tx: Transaction | undefined = trans;
  while (tx) {
    if (tx.trackedChanges !== undefined) {
      return tx.trackedChanges;
    }
    tx = tx.parent;
  }
  return undefined;
}


// These DBCore helpers aren't exported by Dexie; borrowed directly from:
// https://github.com/dfahlander/Dexie.js/blob/v3.0.1/src/dbcore/get-effective-keys.ts

function getEffectiveKeys(
  primaryKey: DBCoreIndex,
  req: (Pick<DBCoreAddRequest | DBCorePutRequest, "type" | "values"> & { keys?: any[] })
    | Pick<DBCoreDeleteRequest, "keys" | "type">
) {
  if (req.type === 'delete') return req.keys;
  return req.keys || req.values.map(primaryKey.extractKey)
}

function getExistingValues(
  table: DBCoreTable,
  req: DBCoreAddRequest | DBCorePutRequest | DBCoreDeleteRequest,
  effectiveKeys: any[]
) {
  return req.type === 'add' ? Promise.resolve(new Array(req.values.length).fill(undefined)) :
    table.getMany({trans: req.trans, keys: effectiveKeys});
}