Skip to content

Instantly share code, notes, and snippets.

View timvw's full-sized avatar

Tim Van Wassenhove timvw

44 followers · 114 following
View GitHub Profile
@timvw
timvw / update.sh
Created April 18, 2024 06:10
Update k8s on ubuntu
curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.29/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.29/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list
sudo apt update
sudo apt-get upgrade
sudo snap refresh microk8s --channel=1.29/stable
@timvw
timvw / .env
Created February 27, 2024 09:30
Directory quickstart (using direnv)
PYENV_VERSION=3.9.15
VIRTUAL_ENV=.venv
AWS_PROFILE=xxx
@timvw
timvw / main.tf
Created February 1, 2024 20:37
Build infra for creating iceberg tables in snowflake with the terraform provider
resource "snowflake_unsafe_execute" "volume" {
execute = <<EOT
CREATE OR REPLACE EXTERNAL VOLUME ${var.external_volume_name}
STORAGE_LOCATIONS =
(
(
NAME = '${var.external_volume_bucket_name}'
STORAGE_PROVIDER = 'S3'
STORAGE_BASE_URL = 's3://${var.external_volume_bucket_name}'
STORAGE_AWS_ROLE_ARN = '${var.snowflake_iam_role_arn}'
@timvw
timvw / Dockerfile
Last active May 12, 2023 10:32
DBT on EMR/EKS with spark
FROM maven:3.6.3-jdk-8 as builder
RUN mvn dependency:copy -Dartifact=org.apache.hudi:hudi-spark3.3-bundle_2.12:0.13.0 -DoutputDirectory=/opt/jars
RUN mvn dependency:copy -Dartifact=io.delta:delta-core_2.12:2.2.0 -DoutputDirectory=/opt/jars
RUN mvn dependency:copy -Dartifact=io.delta:delta-storage:2.2.0 -DoutputDirectory=/opt/jars
FROM 107292555468.dkr.ecr.eu-central-1.amazonaws.com/spark/emr-6.9.0:latest
COPY --from=builder /opt/jars/hudi-spark3.3-bundle_2.12-0.13.0.jar /usr/lib/spark/jars
COPY --from=builder /opt/jars/delta-core_2.12-2.2.0.jar /usr/lib/spark/jars
COPY --from=builder /opt/jars/delta-storage-2.2.0.jar /usr/lib/spark/jars
@timvw
timvw / migrate.sh
Created January 17, 2023 15:21
Copy and update all libs from /usr/local to current directory
for file in $(ls .); do
for ll in $(otool -L $file | grep -oE "/usr/local/[^ ]*"); do
echo "copying $ll to ."
sudo cp $ll .
#echo ""
done
done
for file in $(ls .); do
for ll in $(otool -L $file | grep -oE "/usr/local/[^ ]*"); do
@timvw
timvw / cli.sh
Last active January 15, 2023 09:26
Ballista session
# launch cli and connect to scheduler
docker run \
--network host \
--rm -it \
-v /Users/timvw:/Users/timvw \
apache/arrow-ballista-cli:0.10.0 --host 127.0.0.1 --port 50050
create external table test stored as parquet location '/Users/timvw/Desktop/test.parquet';
select * from test limit 10;
@timvw
timvw / script.sh
Created November 23, 2022 10:04
From json to tsv with jq
```bash
cat LSQL-messages-2022-11-23-06-44-47.json | jq -r '.[].value | [.day, .count] | @tsv'
```
@timvw
timvw / port-user.sh
Created May 5, 2022 11:51
Describe which process is using a given TCP port (macos)
lsof -nP -i4TCP:$1
@timvw
timvw / Cargo.toml
Created May 3, 2022 20:58
datafusion-aws-glue-catalog (example)
[package]
name = "rust-playground"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
aws-config = "0.11.0"
aws-sdk-s3 = "0.11.0"
@timvw
timvw / deployment-controller-token
Last active January 21, 2022 10:20
Get token for k8s dashboard http://localhost:8001/api/v1/namespaces/kubernetes-dashboard/services/https:kubernetes-dashboard:/proxy/#/login
kubectl -n kube-system get secret -o json | \
jq -r '.items[] | select(.metadata.name | startswith("deployment-controller-token")) | .data.token' | \
base64 --decode | \
pbcopy