kmatt

#!/bin/bash

# build/build-spark-pip.sh
# https://spark.apache.org/docs/3.4.1/building-spark.html

export MAVEN_OPTS="-Xss64m -Xmx2g -XX:ReservedCodeCacheSize=1g"

#./build/mvn -DskipTests clean package

pushd ..

kmatt

SELECT R.session_id, DatabaseName = db_name(R.database_id), S.text, R.Status, R.Command,
    R.percent_complete,
    CAST(CAST(DATEADD(mi, R.total_elapsed_time/60000.0, 0) AS TIME) AS CHAR(5)) runTime,
    CAST(CAST(DATEADD(mi, R.estimated_completion_time/60000.0, 0) AS TIME) AS CHAR(5)) remain,
    R.reads, R.writes, R.cpu_time, R.status, R.wait_type
FROM sys.dm_exec_requests R
CROSS APPLY sys.dm_exec_sql_text(R.sql_handle) S
WHERE R.session_id <> @@SPID

kmatt

""" 
  Run DuckDB query on over SSH to avoid scanning full file set on a remote server,
  and make results availble to local DuckDB process
"""

import io, paramiko, duckdb

sql = "SELECT * FROM read_json_auto('/path/to/data.json')"
cmd = f'duckdb -csv -s "{sql}"'

kmatt

pip download --no-deps --dest vendor -r requirements.txt

kmatt

# On Linux or macOS

export CC="zig cc -target x86_64-windows-gnu"
export CXX="zig c++ -target x86_64-windows-gnu"

./configure --host x86_64-windows-gnu
make

kmatt

# https://spark.apache.org/docs/3.4.1/building-spark.html

export MAVEN_OPTS="-Xss64m -Xmx2g -XX:ReservedCodeCacheSize=1g"

#./dev/make-distribution.sh --name custom-spark --pip --r --tgz -Psparkr -Phive -Phive-thriftserver
./dev/make-distribution.sh --name custom-spark --pip --tgz

#python setup.py sdist

pip install python/dist/pyspark-3.4.1.tar.gz

kmatt

podman pull docker.io/library/postrges

podman volume create pg_data

podman run -dt --name postgres \
  -e POSTGRES_PASSWORD=*** \
  --mount type=volume,src=pg_data,target=/var/lib/postgresql/data \
  -p 5432:5432 \
  postgres

kmatt

-- original source: https://medium.com/adhawk-engineering/using-postgresql-to-generate-slugs-5ec9dd759e88

-- https://www.postgresql.org/docs/9.6/unaccent.html
CREATE EXTENSION IF NOT EXISTS unaccent;

CREATE OR REPLACE FUNCTION public.slugify(v TEXT) RETURNS TEXT
  LANGUAGE plpgsql
  STRICT IMMUTABLE AS
$function$
BEGIN

kmatt

An opinionated list of excellent Clojure learning materials

These resources (articles, books, and videos) are useful when you're starting to learn the language, or when you're learning a specific part of the language. This an opinionated list, no doubt. I've compiled this list from writing and teaching Clojure over the last 10 years.

• 🔴 Mandatory (for both beginners and intermediates)
• 🟩 For beginners
• 🟨 For intermediates

Table of contents

1. Getting into the language

kmatt

# Alternative to_sql() *method* for mssql+pyodbc or mssql+pymssql
#
# adapted from https://pandas.pydata.org/docs/user_guide/io.html#io-sql-method
import json

import pandas as pd
import sqlalchemy as sa


def mssql_insert_json(table, conn, keys, data_iter):