Skip to content

Instantly share code, notes, and snippets.

@joewagner
Created March 2, 2023 01:10
Show Gist options
  • Save joewagner/f240b91246081dbd1202b13a0714f802 to your computer and use it in GitHub Desktop.
Save joewagner/f240b91246081dbd1202b13a0714f802 to your computer and use it in GitHub Desktop.
An example of helpers for Tableland bulk inserts that will need to be split into multiple transactions.
// Dependencies: @tableland/sdk, @tableland/local
// Run `npx local-tableland` and then run this script in another window
import { Database, helpers } from "@tableland/sdk";
import { Wallet, getDefaultProvider, utils } from "ethers";
// NOTE: using 34000 here to give a 1000 byte padding for the `insert into table values` and
// in case the sql parser increases the size, which is unlikely but possible.
// If you wanted to limit for gas you'd have to kinda guess a value here.
const BYTES_LIMIT = 34000
async function main() {
const db = getDb();
const { meta: create } = await db
.prepare(
`CREATE TABLE drivers (permit_id integer primary key, race_team text, is_holo int, is_revealed int, driver_id int, permit_pfp_address text, permit_pfp_id int, is_discord_contributor int);`
)
.run();
const { name: tableName } = create.txn;
console.log(`Table name: ${tableName}`);
const formatValues = generateValues();
const results = await helpBatchBulkInsert({
tableName,
values: formatValues,
schema: "permit_id,race_team,is_holo,is_revealed,driver_id",
db,
bytesPerChunk: BYTES_LIMIT
})
// I randomized the choice of team, so these will be close to equal but probably not exactly.
console.log("team 1", (await db.prepare(`select count(*) from ${tableName} where race_team = 'team 1';`).all()).results[0]["count(*)"]);
console.log("team 2", (await db.prepare(`select count(*) from ${tableName} where race_team = 'team 2';`).all()).results[0]["count(*)"]);
console.log("team 3", (await db.prepare(`select count(*) from ${tableName} where race_team = 'team 3';`).all()).results[0]["count(*)"]);
console.log("team 4", (await db.prepare(`select count(*) from ${tableName} where race_team = 'team 4';`).all()).results[0]["count(*)"]);
}
const byteSize = str => new Blob([str]).size;
function helpGetInsertChunks(valuesStrings, bytesPerChunk) {
const chunks = []
while (valuesStrings.length > 0) {
let chunk = "";
while (valuesStrings.length > 0 && byteSize(chunk) < bytesPerChunk) {
// My intuition was fifo, but maybe filo makes more sense?
chunk += `${valuesStrings.shift()},`;
}
// if a single values
if (!chunk) throw new Error("could not chunk the values");
// slice the last comma off
chunks.push(chunk.slice(0, -1));
}
return chunks;
}
async function helpBatchBulkInsert(params) {
const chunks = helpGetInsertChunks(params.values, params.bytesPerChunk);
const responses = []
for (const chunk of chunks) {
// Since the db is using autoWait this won't return until the transaction is finished.
const res = await params.db.prepare(
`INSERT INTO ${params.tableName}(${params.schema})VALUES${chunk}`
).all();
responses.push(res)
if (!res.success) {
console.warn("insert failed");
// Now what?
// - Maybe you want to stop on the first failure?
// - Maybe you want log the failures then retry them separately?
}
// Note that `res.blockNumber` will be incrementing by 3 each time because we
// are waiting for the 3 blocks before the transaction is considered valid.
console.log(res);
}
return responses;
}
function getDb() {
// Hard code Hardhat private key
const privateKey =
"0x5de4111afa1a4b94908f83103eb1f1706367c2e68ca870fc3fb9a804cdab365a";
// Define the signer and connect to the provider.
const wallet = new Wallet(privateKey);
// A local Hardhat node from running `npx local-tableland`, but replace
// with any provider URL (e.g., Alchemy, Infura, Etherscan, etc.).
const provider = getDefaultProvider("http://127.0.0.1:8545");
const signer = wallet.connect(provider);
return new Database({
// If we wait until transactions are finished before returning
// it makes reasoning about control flow easier.
autoWait: true,
signer
});
}
const teams = ["team 1", "team 2", "team 3", "team 4"];
// This will probably come from a file or something a bit more intentional.
function generateValues() {
return Array(8000).fill().map(
function (_, i) {
const team = teams[Math.floor(Math.random() * teams.length)];
return `(${i},'${team}',0,1,${i})`;
}
);
}
await main();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment