Skip to content

Instantly share code, notes, and snippets.

View mehd-io's full-sized avatar
🎥
Coding as much as recording

Mehdi OUAZZA mehd-io

🎥
Coding as much as recording
View GitHub Profile
@mehd-io
mehd-io / setup_duckdb.sh
Last active March 13, 2024 13:35
Script to install duckdb (Linux/Macos)
#!/usr/bin/env bash
set -e
CDN="https://github.com/duckdb/duckdb/releases/download"
INSTALL_DIR="$HOME/.local/bin"
# Function to initialize platform specifics
initPlatform() {
OS=$(uname -s | tr '[:upper:]' '[:lower:]')
ARCH=$(uname -m)
@mehd-io
mehd-io / convert_file.sh
Created March 11, 2024 10:53
Convert csv<->parquet
#!/bin/bash
# A simple script for converting files between CSV and Parquet formats using DuckDB. Requires DuckDB installation.
convert_file() {
local input_file="$1"
local output_extension="$2"
# Extracting the filename without extension
local base_name=$(basename -- "$input_file")
local name="${base_name%.*}"
@mehd-io
mehd-io / duckdb_finance_data_example.sql
Created April 5, 2023 12:04
Query Yahoo Finance Data w/ DuckDB
-- Install httpfs extension
INSTALL httpfs;
LOAD httpfs;
-- Install Scrooge extension https://github.com/pdet/Scrooge-McDuck
-- NOTE : You need to start DuckDB with `-unsigned` flag to authorized to install & load 3rd party extension
SET custom_extension_repository='scrooge-duckdb.s3.us-west-2.amazonaws.com/scrooge/s3_deploy';
INSTALL scrooge;
LOAD scrooge;
-- Example of query
FROM yahoo_finance("^GSPC", "2023-02-01", "2023-02-04", "1d");
@mehd-io
mehd-io / csv_to_parquet.sh
Created March 21, 2023 16:23
Convert CSV to Parquet using DuckDB CLI
#!/bin/bash
# You can put this in your .bashrc or .zshrc
function csv_to_parquet() {
file_path="$1"
duckdb -c "COPY (SELECT * FROM read_csv_auto('$file_path')) TO '${file_path%.*}.parquet' (FORMAT PARQUET);"
}
@mehd-io
mehd-io / conftest.py
Last active May 24, 2022 18:32
medium-tftest-api-conftest
from pathlib import Path
import pytest
import tftest
from google.auth.transport.requests import AuthorizedSession
from .cloud_run_client import (
get_auth_session,
get_service_account_file_path,
get_token_credentials_from_service_account,
@mehd-io
mehd-io / test_hello.py
Created May 24, 2022 18:25
medium-tftest-api-hello
import json
import pytest
from google.auth.transport.requests import AuthorizedSession
from .cloud_run_client import request_wrapper
@pytest.mark.integration
def test_health_check(auth_session: AuthorizedSession):
@mehd-io
mehd-io / test_cloud_run.py
Created May 24, 2022 18:24
medium-tftest-infra
import pytest
import tftest
from pathlib import Path
@pytest.fixture
def plan():
file_path = Path(__file__).resolve()
base_dir = file_path.parent.parent.parent.absolute()
tf = tftest.TerraformTest(tfdir="terraform", basedir=base_dir)
@mehd-io
mehd-io / build.yml
Created November 22, 2021 11:32
medium-python-api-boilerplate Github Actions snippet
- name: Prepare CI image
run: |
make get-img DOCKER_LAYER=dev PUSH_IMAGE=true
- name: Test
run: |
make test
- name: Build & Push app image
run: |
@mehd-io
mehd-io / Makefile
Created November 22, 2021 11:11
medium-python-api-boilerplate Make target get-img
get-img:
@echo 🐳 $@
@if [ "$$DOCKER_LAYER" != "base" ]; then\
echo 🔨 Pulling/Building base image first...; \
$(MAKE) pull-img DOCKER_LAYER=base || $(MAKE) _build-img DOCKER_LAYER=base;\
fi;\
$(MAKE) pull-img || $(MAKE) _build-img
@if [ "$$PUSH_IMAGE" = "true" ]; then\
echo ✨ Pushing the images to docker registry...; \
$(MAKE) push-img DOCKER_LAYER=base; \
@mehd-io
mehd-io / version.sh
Created November 22, 2021 11:01
medium-python-api-boilerplate managing version of dockerfile
#!/bin/bash
# Simple bash script to generate version based on hash of files or git command
ROOT=$(git rev-parse --show-toplevel)
OS=$(uname -s)
if [ "$OS" = "Linux" ]; then
md5_cmd="md5sum
elif [ "$OS" = "Darwin" ]; then
md5_cmd="md5 -r"