Skip to content

Instantly share code, notes, and snippets.

View scravy's full-sized avatar

Julian Bertram scravy

View GitHub Profile
import pandas as pd
import pyarrow as pa
def main():
df1: pd.DataFrame = pd.DataFrame([
[1, 1.0, ],
[3, 2.0, ],
[None, 4.0, ],
[7, 8.0, ],
class Context {
parent = null;
vars = {};
constructor(parent = null) {
this.parent = parent;
}
get(key) {
if (key in this.vars) {
@scravy
scravy / gist:6715b319e4fa60ed396ba73d9c9175b9
Created July 3, 2021 07:06
Apply parquet tools in *.parquet files recursively
find . -name '*.parquet' -exec docker run -it -v `pwd`/'{}':/tmp/file.parquet nathanhowell/parquet-tools schema /tmp/file.parquet ';'
@scravy
scravy / frequencies.py
Last active June 12, 2021 19:07
Frequencies using Spark and a Pandas-UDF using Hamilton's method with exploding rows
import os
from typing import Dict
import numpy as np
import pandas as pd
import pyspark
import pyspark.sql.functions as F
from pyspark.sql import DataFrame, SparkSession
# noinspection PyUnresolvedReferences
from pyspark.sql.pandas.functions import pandas_udf
"""
Given a DataFrame like that:
date name mx my mz
0 2021-01-01 foo 46.676136 54.017562 1.882048
1 2021-01-01 bar 81.669122 51.525996 8.062095
2 2021-01-01 qux 54.584057 53.621155 9.475698
3 2021-01-02 foo 84.945618 51.289542 9.132064
4 2021-01-02 bar 87.438244 50.643675 1.881121
5 2021-01-02 qux 91.891133 50.347743 8.809469
<?php
class FlexBase {
private $spec = array();
private $bases = array();
private $checksum = 1;
private $num_digits;
private $pretty = FALSE;
private $base = 10;
@scravy
scravy / Makefile
Created February 2, 2013 12:42
AWT started from JNI (does not work, using Mac OS X)
JAVAHOME=/Library/Java/JavaVirtualMachines/1.7.0.jdk/Contents/Home
swing: swing.c StartAWT.class
gcc -Wall -pedantic --std=c99 swing.c -ljvm \
-L$(JAVAHOME)/jre/lib/server \
-I$(JAVAHOME)/include \
-I$(JAVAHOME)/include/darwin \
-o $@
install_name_tool -add_rpath $(JAVAHOME)/jre/lib/server $@
syn keyword xError Error
hi def link xError Error
syn keyword xConstant Constant
hi def link xConstant Constant
syn keyword xString String
hi def link xString String
syn keyword xCharacter Character
{-# LANGUAGE Haskell2010, LambdaCase #-}
import Data.List (nub)
data Symbol = M | I | U deriving (Eq, Show)
apply = \case
M : xs -> [ M : xs ++ xs ]
I : I : I : xs -> [ U : xs ]
U : U : xs -> [ xs ]
{-# LANGUAGE Haskell2010, LambdaCase, TupleSections #-}
import Data.List (nub)
data Symbol = M | I | U deriving (Eq, Show)
data Rule = Mx | III | UU | IU deriving (Eq, Show, Enum)
data Deduction = Axiom [Symbol] | Deduction [Symbol] Rule [Symbol]
instance Show Deduction where
show = \case