test.js

const _ = require('lodash');
const {generatePlayers} = require('./generatePlayers');
const {rateMatching} = require('./rateTeams');
const {k_combinations} = require('./combinations');
const combinatorics = require('js-combinatorics');
const neo4j = require('neo4j-driver').v1
const Promise = require('bluebird');

const driver = neo4j.driver("bolt://localhost:7687", neo4j.auth.basic("neo4j", "test"));
(async () => {
    const session = await driver.session();
    await session.run(`MATCH (n) DETACH DELETE n`);
    console.log('pruned data')
    await session.run(`CREATE CONSTRAINT ON (player:Player) ASSERT player.userId IS UNIQUE`);
    const players = generatePlayers(100000);
    console.log(`generated ${players.length} players`)

    async function playersToNeo4j() {
        const chunkedPlayers = _.chunk(players, 50);
        // await Promise.map(players, async (player) => {
        //     return session.run(`CREATE (:Player { userId: '${player.id}'})`)
        //         .catch((err) => {
        //             if (!(err.code === "Neo.ClientError.Schema.ConstraintValidationFailed")) {
        //                 throw err;
        //             }
        //         }).then(() => {
        //             console.log('finished insert')
        //         })
        // }, {
        //     concurrency: 10
        // });
        await Promise.map(chunkedPlayers, async (chunkOfPlayers) => {
            return session.run([
                `CREATE `,
                chunkOfPlayers
                    .map((player) => ` (:Player { userId: '${player.id}'})`)
                    .join(', ')
            ].join(''))
                .catch((err) => {
                    if (!(err.code === "Neo.ClientError.Schema.ConstraintValidationFailed")) {
                        throw err;
                    }
                }).then(() => {
                    console.log('finished insert')
                })
        }, {
            concurrency: 3
        })
    }

    await playersToNeo4j();
    const chunkSize = 10;
    const chunkedPlayers = _.chunk(players, chunkSize);
    await Promise.each(chunkedPlayers, (chunkOfPlayers, index) => {
        const queries = chunkOfPlayers.map((player) => {
            const likeQueries = player.likes.map((likedPlayer) => {
                return [
                    `MATCH (a:Player),(b:Player)`,
                    `WHERE a.userId = '${player.id}' AND b.userId = '${likedPlayer}'`,
                    `CREATE (a)-[r:Likes]->(b)`,
                    `RETURN type(r), r.name`
                ].join('\n')
            });
            const dislikeQueries = player.dislikes.map((likedPlayer) => {
                return [
                    `MATCH (a:Player),(b:Player)`,
                    `WHERE a.userId = '${player.id}' AND b.userId = '${likedPlayer}'`,
                    `CREATE (a)-[r:Dislikes]->(b)`,
                    `RETURN null`
                ].join('\n')
            });
            return [...likeQueries, ...dislikeQueries];
        });
        return session.writeTransaction(function (transaction) {
            _.flatten(queries).forEach((query) => {
                transaction.run(query);
            });
        }).then(() => {
            console.log(`finished relationships for Player # ${index * chunkSize} @ ${Math.floor(index * chunkSize / players.length * 100)}%`)
        })
    })
    process.exit(0)

    // await Promise.map(players, async (player) => {
    //     const likePromises = player.likes.map((likedPlayer) => {
    //         const query = [
    //             `MATCH (a:Player),(b:Player)`,
    //             `WHERE a.userId = '${player.id}' AND b.userId = '${likedPlayer}'`,
    //             `CREATE (a)-[r:Likes { rel: a.name + '->' + b.name }]->(b)`,
    //             `RETURN type(r), r.name`
    //         ].join('\n')
    //         return session.run(query)
    //             .catch((err) => {
    //                 if (!(err.code === "Neo.ClientError.Schema.ConstraintValidationFailed")) {
    //                     throw err;
    //                 }
    //             }).then(() => {
    //                 console.log('finished relationship')
    //             })
    //     })
    //     const dislikePromises = player.dislikes.map((disliked) => {
    //         const query = [
    //             `MATCH (a:Player),(b:Player)`,
    //             `WHERE a.userId = '${player.id}' AND b.userId = '${disliked}'`,
    //             `CREATE (a)-[r:Dislikes { rel: a.name + '->' + b.name }]->(b)`,
    //             `RETURN type(r), r.name`
    //         ].join('\n')
    //         return session.run(query)
    //             .catch((err) => {
    //                 if (!(err.code === "Neo.ClientError.Schema.ConstraintValidationFailed")) {
    //                     throw err;
    //                 }
    //             }).then(() => {
    //                 console.log('finished relationship')
    //             })
    //     })
    //     return [...likePromises, ...dislikePromises]
    // }, {
    //     concurrency: 10
    // });
    // process.exit(0)
})();

workingFast.js

const _ = require('lodash');
const {generatePlayers} = require('./generatePlayers');
const {rateMatching} = require('./rateTeams');
const {k_combinations} = require('./combinations');
const combinatorics = require('js-combinatorics');
const neo4j = require('neo4j-driver').v1
const Promise = require('bluebird');

const driver = neo4j.driver("bolt://localhost:7687", neo4j.auth.basic("neo4j", "test"));
(async () => {
    const session = await driver.session();
    await session.run(`call apoc.periodic.iterate("MATCH (n:Player) return n", "DETACH DELETE n", {batchSize:1000}) yield batches, total return batches, total`);
    console.log('pruned data')
    await session.run(`CREATE CONSTRAINT ON (player:Player) ASSERT player.userId IS UNIQUE`);
    const players = generatePlayers(10000);
    console.log(`generated ${players.length} players`)

    async function playersToNeo4j() {
        const chunkedPlayers = _.chunk(players, 50);
        await Promise.map(chunkedPlayers, async (chunkOfPlayers) => {
            return session.run([
                `CREATE `,
                chunkOfPlayers
                    .map((player) => ` (:Player { userId: '${player.id}'})`)
                    .join(', ')
            ].join(''))
                .catch((err) => {
                    if (!(err.code === "Neo.ClientError.Schema.ConstraintValidationFailed")) {
                        throw err;
                    }
                }).then(() => {
                    console.log('finished insert')
                })
        }, {
            concurrency: 3
        })
    }

    await playersToNeo4j();
    const chunkSize = 10;
    const chunkedPlayers = _.chunk(players, chunkSize);

    function getBatch(chunkOfPlayers, type) {
        const batch = {batch: []};
        chunkOfPlayers.forEach((player) => {
            player[type].forEach((likedPlayer) => batch.batch.push({
                userId: player.id,
                [type]: likedPlayer
            }))
        });
        return batch;
    }

    function getConnectionQuery(type) {
        return [
            'UNWIND {batch} as row',
            'MATCH (from:Player {userId: row.userId})',
            `MATCH (to:Player {userId: row.${type}})`,
            `CREATE (from)-[rel:${type.charAt(0).toUpperCase() + type.substr(1)}]->(to)`,
        ].join('\n');
    }

    await Promise.each(chunkedPlayers, (chunkOfPlayers, index) => {
        const likeBatch = getBatch(chunkOfPlayers, 'likes');
        const dislikeBatch = getBatch(chunkOfPlayers, 'dislikes');
        const likeQuery = getConnectionQuery('likes');
        const dislikeQuery = getConnectionQuery('dislikes');

        return Promise.all([
            session.run(likeQuery, likeBatch),
            session.run(dislikeQuery, dislikeBatch)
        ]).then(() => {
            console.log(`finished relationships for Player # ${index * chunkSize} @ ${Math.floor(index * chunkSize / players.length * 100)}%`)
        });
    });
    process.exit(0)
})();