Dedup data in view counter datastore

dedup.js

async function go() {
	const { Datastore } = require("@google-cloud/datastore");

	const ds = new Datastore();

	const query = ds.createQuery('view');

	const [entities] = await ds.runQuery(query);

    const uniqueURLs = {};
    const originalKeys = [];

    console.log(`total number of original entities: ${entities.length}`);
    const entityViews = entities.reduce((previous, current) => {
        return previous + current.count;
    }, 0);
    console.log(`total original entity views: ${entityViews}`);

    entities.forEach(entity => {
        const keyName = entity[ds.KEY].name;
        const url = new URL(keyName);
        const unique = `${url.origin}${url.pathname}`;

        if (!uniqueURLs[unique]) {
            uniqueURLs[unique] = entity.count
        } else {
            uniqueURLs[unique] += entity.count
        }

        originalKeys.push(entity[ds.KEY]);
    });

    const newEntities = Object.keys(uniqueURLs).map(keyName => {
        return {
            key: ds.key(['view', keyName]),
            data: { count: uniqueURLs[keyName] }
        }
    });

    // console.dir(newEntities);
    // console.dir(uniqueURLs);
    console.log(`total number of new entities: ${newEntities.length}`);
    const uniqueViews = newEntities.reduce((previous, current) => {
        return previous + current.data.count;
    }, 0);
    console.log(`total new entity views: ${uniqueViews}`);

    console.log('deleting original entities');
    await ds.delete(originalKeys.slice(0, 499));
    await ds.delete(originalKeys.slice(500, 999));
    await ds.delete(originalKeys.slice(1000));

    console.log('adding new entities');
    await ds.upsert(newEntities.slice(0, 499));
    await ds.upsert(newEntities.slice(500));
}

go();