westonpace

#include <iostream>
#include <string>
#include <charconv>
#include <cassert>

template <typename>
constexpr std::false_type can_to_chars_helper (long);

template <typename T>
constexpr auto can_to_chars_helper (int)

westonpace

import os
import tempfile

import pandas as pd
from fastparquet import write

with tempfile.TemporaryDirectory() as tempdir:

    df = pd.DataFrame({"x": [1, 1, 1, 2, 2, 2], "y": [1, 2, 3, 4, 5, 6]})

westonpace

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//

westonpace

import os
import psutil

import numpy as np
import pyarrow as pa
import pyarrow.ipc as ipc

process = psutil.Process(os.getpid())

Mi = 1 << 20

westonpace

initial mem usage 230264832 0
mem usage 324423680 80000000
mem usage 324423680 80000000
mem usage 328425472 81360000
mem usage 328536064 81360000
mem usage 328601600 81360000
mem usage 328486912 81360000
mem usage 328519680 81360000
mem usage 328663040 81360000
mem usage 328474624 81360000

westonpace

import pyarrow as pa
import pyarrow.compute as pc
import pyarrow.dataset as ds

table = pa.Table.from_pydict({'x': [1, 2, 3], 'y': ['a', 'a', 'b']})

expr = (ds.field('x') > 1) & (ds.field('y') == 'a')

print(ds.Scanner.from_batches(table.to_batches(), schema=table.schema, columns={'result': expr}).to_table().column(0))
# [ [ false, true, false ] ]

westonpace

import pandas as pd

df = pd.DataFrame({'x': [[1, 2], "hello"]})
# Will fail because x is not homogenous
# df.to_feather('/tmp/foo.arrow')
df.to_json('/tmp/foo.json')

# Will create:
# {
#   "x": {

westonpace

          value
0     first_val
1     first_val
2   first_other
3   first_other
4     first_val
5   first_other
6    second_new
7  second_other
          value

westonpace

cmake_minimum_required(VERSION 3.3)
set(CMAKE_CXX_STANDARD 11)
project(app LANGUAGES CXX)

#Set the location of where Windows can find the installed libraries of the SDK.
if(MSVC)
    string(REPLACE ";" "/aws-cpp-sdk-all;" SYSTEM_MODULE_PATH "${CMAKE_SYSTEM_PREFIX_PATH}/aws-cpp-sdk-all")
    list(APPEND CMAKE_PREFIX_PATH ${SYSTEM_MODULE_PATH})
endif()

westonpace