Skip to content

Instantly share code, notes, and snippets.

@westonpace
Created April 14, 2021 22:51
Show Gist options
  • Save westonpace/1477fe0fcc2a859391f58815e7411688 to your computer and use it in GitHub Desktop.
Save westonpace/1477fe0fcc2a859391f58815e7411688 to your computer and use it in GitHub Desktop.
Example creating a feather file from a row-major double vector
#include <iostream>
#include <vector>
#include <arrow/api.h>
#include <arrow/filesystem/api.h>
#include <arrow/ipc/api.h>
arrow::Status RunMain(int argc, char **argv) {
std::vector<std::vector<double>> rows = {{1, 2, 3}, {4, 5, 6}, {7, 8, 9}};
// Convert your row-vector to a vector of columns (arrow::Array)
std::vector<arrow::DoubleBuilder> column_builders(rows.size());
for (const auto &row : rows) {
for (std::size_t col_idx = 0; col_idx < row.size(); col_idx++) {
column_builders[col_idx].Append(row[col_idx]);
}
}
std::vector<std::shared_ptr<arrow::Array>> columns;
for (std::size_t col_idx = 0; col_idx < column_builders.size(); col_idx++) {
ARROW_ASSIGN_OR_RAISE(auto col, column_builders[col_idx].Finish());
columns.push_back(col);
}
// Create schema (insert field names here)
std::vector<std::shared_ptr<arrow::Field>> fields;
for (std::size_t col_idx = 0; col_idx < columns.size(); col_idx++) {
fields.push_back(
arrow::field("Field " + std::to_string(col_idx), arrow::float64()));
}
auto schema = arrow::schema(fields);
// Create record batch
auto record_batch = arrow::RecordBatch::Make(
schema, static_cast<int64_t>(rows.size()), columns);
// Open output file
arrow::fs::LocalFileSystem fs;
ARROW_ASSIGN_OR_RAISE(auto output_stream,
fs.OpenOutputStream("/tmp/foo.feather"));
ARROW_ASSIGN_OR_RAISE(auto writer,
arrow::ipc::MakeFileWriter(output_stream, schema));
ARROW_RETURN_NOT_OK(writer->WriteRecordBatch(*record_batch));
return writer->Close();
}
int main(int argc, char **argv) {
arrow::Status st = RunMain(argc, argv);
if (!st.ok()) {
std::cerr << st << std::endl;
return 1;
}
return 0;
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment