duck_arrow_stream_repro.mjs

/* eslint-disable @typescript-eslint/ban-ts-comment */
// @ts-nocheck
import duckdb from "duckdb";
import assert from "node:assert";
import { tableFromIPC, RecordBatchReader, Table, Int32, makeData, Vector } from "apache-arrow";

const RANGE_SIZE = 1_000;
const SQL = `select * from range(0,${RANGE_SIZE}) tbl(i)`;

const expectedTable = new Table({
  i: new Vector([makeData({ type: new Int32, data: Array.from(new Array(RANGE_SIZE), (x, i) => i) })]),
});

async function loadArrowIpc(db) {
  const ipcStream = await db.arrowIPCStream(SQL);
  const reader = await RecordBatchReader.from(ipcStream);
  const table = await tableFromIPC(reader);
  const actualArray = table.toArray();

  console.log(`Columns: ${table.numCols}`);
  console.log(`Rows: ${table.numRows}`);

  console.log("Table from stream:", actualArray);

  assert.deepEqual(actualArray, expectedTable.toArray());
}

const db = new duckdb.Database(':memory:');

db.run("INSTALL arrow;", (err) => {
  assert.ifError(err);

  db.run("Load arrow;", (err) => {
    assert.ifError(err);

    loadArrowIpc(db)
      .then(() => {
        process.exit(0);
      })
      .catch(err => {
        console.error(err);
        process.exit(1);
      });
  });
});