Skip to content

Instantly share code, notes, and snippets.

@clemp
Created March 25, 2022 20:57
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save clemp/64259c7654e88e4f86946874c4317a1c to your computer and use it in GitHub Desktop.
Save clemp/64259c7654e88e4f86946874c4317a1c to your computer and use it in GitHub Desktop.
Pattern to ingest data into Neo4j via s3 csvs
const mainTest = async () => {
/*
Configure Neo4j params
*/
const neo4j_params = {
uri: process.env.NEO4J_SANDBOX_URI,
username: process.env.NEO4J_SANDBOX_UN,
pw: process.env.NEO4J_SANDBOX_PW
};
const neo4j_driver = neo4j.driver(
neo4j_params.uri,
neo4j.auth.basic(
neo4j_params.username,
neo4j_params.pw
)
);
const session = neo4j_driver.session({database:"neo4j"});
var json = [
{
"ens_type":"name",
"twitterId":"3200004419",
"username":"000sourav",
"ens":"Sourav.eth"
}
, {
"ens_type":"name",
"twitterId":"2523228612",
"username":"01RafaelJimenez",
"ens":"rafa.eth"
}
];
const csvData = createCSV(json);
const s3 = new AWS.S3({
accessKeyId: process.env.AWS_ACCESS_KEY,
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
region: "us-east-2"
});
console.log(csvData);
const chainverse_aws_params = {
Bucket: 'chainverse/twitter-ens',
Key: 'twitter-ens-test.csv',
Body: csvData,
ACL: 'public-read'
};
s3.upload(chainverse_aws_params, function(err, data) {
if (err) {
console.log("ERROR: " + err);
} else {
console.log("upload successful");
}
});
const create_ens_query =
`
LOAD CSV WITH HEADERS FROM $s3csvUrl AS row
MERGE
(e:ENS:Test {name: row.ens})
ON CREATE SET
e:Token
ON CREATE SET
e:NFT
ON CREATE SET
e.tokenStandard = "ERC-721"
ON CREATE SET e.name = row.ens;
`;
// Create wallet nodes.
let result = await session.run(create_ens_query, {
s3csvUrl: 'https://chainverse.s3.us-east-2.amazonaws.com/twitter-ens/twitter-ens-test.csv'
});
session.close();
neo4j_driver.close();
console.log("ingest complete.")
}
// Run the program.
mainTest();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment