This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#include <iostream> | |
#include <arrow/api.h> | |
#include <arrow/io/api.h> | |
#include <arrow/util/float16.h> | |
#include <parquet/arrow/writer.h> | |
arrow::Status WriteTableToParquetFile() { | |
std::shared_ptr<arrow::Array> array; | |
arrow::HalfFloatBuilder builder; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#include <iostream> | |
#include <random> | |
#include <vector> | |
#include <string> | |
#include <arrow/api.h> | |
#include <arrow/io/api.h> | |
#include <parquet/arrow/writer.h> | |
std::vector<std::string> GenerateUniqueStrings() { | |
// generates 26^4 = 456,976 unique 4-letter combinations |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import tempfile | |
import pathlib | |
import numpy as np | |
import pyarrow as pa | |
import pyarrow.compute as pc | |
import pyarrow.parquet as pq | |
import pyarrow.dataset as ds | |
# create a small dataset for example purposes |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#include <iostream> | |
#include <arrow/api.h> | |
#include <arrow/compute/api.h> | |
int main(int, char**) { | |
// lookup set | |
std::shared_ptr<arrow::Array> array; | |
arrow::Int32Builder builder; | |
if (!builder.Append(5).ok()) return 1; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#include <iostream> | |
#include <arrow/api.h> | |
#include <arrow/result.h> | |
#include <arrow/compute/api.h> | |
#include <arrow/compute/exec/exec_plan.h> | |
arrow::Status ExecutePlanAndCollectAsTable( | |
std::shared_ptr<arrow::compute::ExecPlan> plan, | |
std::shared_ptr<arrow::Schema> schema, | |
arrow::AsyncGenerator<std::optional<arrow::compute::ExecBatch>> sink_gen) { |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import google.auth | |
import ibis | |
from ibis import _ | |
credentials, billing_project = google.auth.default() | |
conn = ibis.bigquery.connect(billing_project, 'bigquery-public-data.samples') | |
t = conn.table('github_nested') | |
expr = ( |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# before running: | |
# 1. install Ibis and its Snowflake backend: https://ibis-project.org/backends/Snowflake/ | |
# 2. create and activate a Snowflake trial account | |
# 3. set environment variables SNOWSQL_USER, SNOWSQL_PWD, SNOWSQL_ACCOUNT | |
import os | |
import ibis | |
from ibis import _ | |
ibis.options.interactive = True |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# before running: | |
# 1. install Ibis and its Trino backend: https://ibis-project.org/backends/Trino/ | |
# 2. pull and run the Trino docker container: https://trino.io/docs/current/installation/containers.html | |
import ibis | |
from ibis import _ | |
# connect to Trino | |
conn = ibis.trino.connect(database='memory', schema='default') |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#include <iostream> | |
#include <arrow/api.h> | |
#include <arrow/result.h> | |
#include <arrow/compute/api.h> | |
#include <arrow/compute/exec/exec_plan.h> | |
arrow::Status ExecutePlanAndCollectAsTable( | |
std::shared_ptr<arrow::compute::ExecPlan> plan, | |
std::shared_ptr<arrow::Schema> schema, | |
arrow::AsyncGenerator<std::optional<arrow::compute::ExecBatch>> sink_gen) { |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# pip install 'ibis-framework[duckdb]' | |
import pandas as pd | |
import ibis | |
from ibis import _ | |
# create a pandas DataFrame and write it to a Parquet file | |
df = pd.DataFrame(data={'repo': ['pandas', 'duckdb', 'ibis'], | |
'stars': [36622, 8074, 2336]}) | |
df.to_parquet('repo_stars.parquet') |