Skip to content

Instantly share code, notes, and snippets.

View westonpace's full-sized avatar

Weston Pace westonpace

  • LanceDB
  • Olympia, WA
  • 19:33 (UTC -07:00)
View GitHub Profile
@westonpace
westonpace / test.ts
Created May 24, 2017 15:47
Example showing the pitfalls of using an observable for init code
import { Observable, Observer } from 'rxjs';
function runTest(share: boolean, delay: boolean) {
let observable = Observable.create((observer: Observer<boolean>) => {
setTimeout(() => {
console.log('I did my expensive thingy');
observer.next(true);
}, 1000);
});
@westonpace
westonpace / gist:cf8419a3d8a6ae9144c24ccf556eb08c
Last active November 25, 2018 12:59
Quick test of markdown

Test

Hello World

@westonpace
westonpace / server-config-service-demo.js
Created December 19, 2018 14:09
Server configuration service backed by mongodb for use with discord
const MongoClient = require('mongodb').MongoClient;
const configService = require('./server-config-service');
function randint(max) {
return Math.round(Math.random() * max);
}
function randomstring() {
const letters = 'ABCDEFHIJKLMNOPQRSTUVWXYZ';
let result = '';
const { Subject } = require('rxjs');
const { switchMap, combineLatest, map, startWith } = require('rxjs/operators');
const data = new Subject();
const value = new Subject();
const clicks = new Subject();
const filtered = data.pipe(switchMap(data => {
return value.pipe(startWith(''), combineLatest(clicks.pipe(startWith(null))), map(([value, _]) => {
return data.filter(item => item.includes(value));
@westonpace
westonpace / fs_test.py
Created August 26, 2020 19:46
Test case demonstrating ResourceWarning from using OS files with a non-LocalFileSystem
import os
import pyarrow.filesystem as pafs
import pyarrow.parquet as pq
from pyarrow.util import implements
class RelativeFilesystem(pafs.FileSystem):
def __init__(self, basedir):
self.basedir = basedir
@westonpace
westonpace / gist:0c5ef01e21a40de5d16608b7f12de80d
Created August 26, 2020 22:50
Example attempting to write pyarrow with new filesystems API
import pyarrow.fs as pafs
import pyarrow.parquet as pq
filesystem = pafs.LocalFileSystem()
subtree_filesystem = pafs.SubTreeFileSystem('C:\\', filesystem)
in_path = 'Users\\westpace\\in.parquet'
out_path = 'Users\\westpace\\out.parquet'
table = pq.read_table(in_path, filesystem=subtree_filesystem)
@westonpace
westonpace / listener.py
Created March 3, 2021 19:35
Bare bones example of sending an arrow table from C++ to python via socket
#!/usr/bin/env python3
import socket
import pyarrow as pa
import pyarrow.ipc
listen = "127.0.0.1"
port = 56565
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
@westonpace
westonpace / to_feather_ex.cc
Created April 14, 2021 22:51
Example creating a feather file from a row-major double vector
#include <iostream>
#include <vector>
#include <arrow/api.h>
#include <arrow/filesystem/api.h>
#include <arrow/ipc/api.h>
arrow::Status RunMain(int argc, char **argv) {
std::vector<std::vector<double>> rows = {{1, 2, 3}, {4, 5, 6}, {7, 8, 9}};
using System;
using System.Globalization;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Design;
using Microsoft.Extensions.Configuration;
namespace Arena.Utilities
{
public abstract class BaseDbContextFactory<T> : IDesignTimeDbContextFactory<T> where T : DbContext
{
@westonpace
westonpace / time_str_to_time64.py
Created August 13, 2021 06:04
Converting a time string to a time64 using python (better methods exist in pyarrow >= 6.0.0, or use pandas)
import pyarrow
from datetime import datetime
start = datetime.strptime("00:00", "%H:%M")
time = datetime.strptime("12:15", "%H:%M")
delta = time - start
us = int(delta.total_seconds() * 1000000)
print(pyarrow.array([us], type=pyarrow.int64()).cast(pyarrow.time64('us')))
# <pyarrow.lib.Time64Array object at 0x7f0b7acad160>