kmatt

set-option -g status-right '#{client_tty} (#(TZ=US/Mountain date +%%H:%%M)MT #(TZ=UTC date +%%H:%%M)Z) %Y-%m-%d %H:%M'

kmatt

# To configure for MinGW instead of nmake
#
# C:\TDM-GCC-64\mingwvars.bat
# cmake . -G "MinGw Makefiles" 
#
# Makefile: cmake ... -DCMAKE_TOOLCHAIN_FILE=TDM.cmake

set(CMAKE_SYSTEM_NAME Windows)

set(CMAKE_C_COMPILER C:/TDM-GCC-64/bin/gcc.exe)

kmatt

[cmdletbinding()]
Param(
    [Parameter(Mandatory=$true)]
    [string[]]$servers,
    
    [ValidateScript({
        if(-Not ($_ | Test-Path )) {
            throw "Folder does not exist"
        }
        return $true

kmatt

Here are some helpful commands for managing ZFS and ZPool on Ubuntu

VDEV

Useful for populating /etc/zfs/vdev_id.conf:

printDisks() {
    for i in /dev/sd[b-i]; do
        fdisk -l $i

kmatt

# Thrift is not finding Delta jars in Ivy2 cache, even when specified in spark-defaults.conf (spark.sql.catalog.spark_catalog) ?

# Probably a bad solution...

wget https://repo1.maven.org/maven2/io/delta/delta-core_2.12/2.4.0/delta-core_2.12-2.4.0.jar && \
mv delta-core_2.12-2.4.0.jar jars/

wget https://repo1.maven.org/maven2/io/delta/delta-storage/2.4.0/delta-storage-2.4.0.jar && \
mv delta-storage-2.4.0.jar jars/

kmatt

#!/bin/bash

# build/build-spark-pip.sh
# https://spark.apache.org/docs/3.4.1/building-spark.html

export MAVEN_OPTS="-Xss64m -Xmx2g -XX:ReservedCodeCacheSize=1g"

#./build/mvn -DskipTests clean package

pushd ..

kmatt

""" 
  Run DuckDB query on over SSH to avoid scanning full file set on a remote server,
  and make results availble to local DuckDB process
"""

import io, paramiko, duckdb

sql = "SELECT * FROM read_json_auto('/path/to/data.json')"
cmd = f'duckdb -csv -s "{sql}"'

kmatt

pip download --no-deps --dest vendor -r requirements.txt

kmatt

# https://spark.apache.org/docs/3.4.1/building-spark.html

export MAVEN_OPTS="-Xss64m -Xmx2g -XX:ReservedCodeCacheSize=1g"

#./dev/make-distribution.sh --name custom-spark --pip --r --tgz -Psparkr -Phive -Phive-thriftserver
./dev/make-distribution.sh --name custom-spark --pip --tgz

#python setup.py sdist

pip install python/dist/pyspark-3.4.1.tar.gz

kmatt

podman pull docker.io/library/postrges

podman volume create pg_data

podman run -dt --name postgres \
  -e POSTGRES_PASSWORD=*** \
  --mount type=volume,src=pg_data,target=/var/lib/postgresql/data \
  -p 5432:5432 \
  postgres