Skip to content

Instantly share code, notes, and snippets.

View ddelange's full-sized avatar
💥
["translatio", "imitatio", "aemulatio"]

ddelange ddelange

💥
["translatio", "imitatio", "aemulatio"]
View GitHub Profile
@ddelange
ddelange / ensure_avc_mp4.sh
Created April 25, 2024 09:34
Conditionally copy or encode video stream into avc (x264) mp4 with ffmpeg and ffprobe
#!/usr/bin/env bash
set -euxo pipefail
video_in="video_in.mkv"
video_out="video_out.mp4"
# full metadata payload: $(ffprobe -loglevel error -print_format json -show_format -show_streams "${video_in}")
is_avc=$(ffprobe -loglevel error -select_streams v:0 -show_entries stream=is_avc -of default=noprint_wrappers=1:nokey=1 "${video_in}")
@ddelange
ddelange / sharednumpyarray.py
Created March 25, 2024 11:38
Multiprocessing numpy using zero-copy SharedNumpyArray and ProcessPoolExecutor.imap
from collections import deque
from concurrent.futures import ProcessPoolExecutor as _ProcessPoolExecutor
from multiprocessing.shared_memory import SharedMemory
import numpy as np
class ProcessPoolExecutor(_ProcessPoolExecutor):
"""Subclass with a lazy consuming imap method."""
@ddelange
ddelange / build-git.sh
Last active February 15, 2024 01:51 — forked from ivan-c/build-git.sh
compile git with openssl instead of gnutls
#!/usr/bin/env bash
# original gist from pescobar/build-git.sh
# changes by LaggAt:
# * to be usable on Raspbian / tested RPi3 and
# * for automatic depency resolving
# changes by ivan-c:
# * add `apt-get update`
# changes by ddelange:
# * add `set -euxo pipefail`
# * remove `--allow-downgrades`
@ddelange
ddelange / stream_command.py
Last active August 13, 2023 13:38
Handling the live output stream of a command
import logging
from collections import deque
from concurrent.futures import ThreadPoolExecutor
from functools import partial
from subprocess import PIPE, CalledProcessError, CompletedProcess, Popen
def stream_command(
args,
*,
@ddelange
ddelange / threadpoolexecutor.py
Last active November 9, 2023 15:47
The missing ThreadPoolExecutor.imap
from collections import deque
from concurrent.futures import ThreadPoolExecutor as _ThreadPoolExecutor
class ThreadPoolExecutor(_ThreadPoolExecutor):
"""Subclass with a lazy consuming imap method."""
def imap(self, fn, *iterables, timeout=None, queued_tasks_per_worker=2):
"""Ordered imap that lazily consumes iterables ref https://gist.github.com/ddelange/c98b05437f80e4b16bf4fc20fde9c999."""
futures, maxlen = deque(), self._max_workers * (queued_tasks_per_worker + 1)
@ddelange
ddelange / URIDownloader.py
Last active May 15, 2023 08:27
Multithreaded S3 downloads
# pip install smart_open[s3]
from collections import deque
from concurrent.futures import ThreadPoolExecutor as _ThreadPoolExecutor
from functools import partial
from typing import Callable, Dict, Optional, Iterable, Iterator, Sequence
import boto3
import botocore
import smart_open
@ddelange
ddelange / release_assets_pip_find_links.py
Last active September 22, 2022 06:29
Create a PIP_FIND_LINKS page containing all release assets
@ddelange
ddelange / mp4_srt.py
Last active December 19, 2022 07:09
Mux multiple subtitle files into an mp4 file
# pip install sh pycountry
import re
from pathlib import Path
import pycountry
from sh import ffmpeg
def mux_mp4_subs(inp, outp, *subs, _cwd=None, **subs_map):
"""Mux multiple subtitle files into an mp4 file.
@ddelange
ddelange / playwright_github_stars_lists.py
Last active July 4, 2023 14:12
Scrape the GitHub Stars Lists for a user
#!/usr/bin/env python3
# $ pip install pandas playwright tabulate && python -m playwright install --with-deps webkit
# $ GITHUB_REPOSITORY_OWNER=octocat python playwright_github_stars_lists.py
import asyncio
import os
from contextlib import asynccontextmanager
import pandas as pd
from playwright.async_api import async_playwright
from playwright.async_api._generated import Browser, Locator
@ddelange
ddelange / kubetop.py
Last active March 15, 2023 10:08
kubetop - cluster resource monitoring
# View node and container resource requests, limits, and usage, grouped per node,namespace,pod,container
#
# Installation:
# pip install mapply pandas kubernetes sh
# brew install watch kubectl coreutils
#
# Usage:
# alias kubetop='watch -n4 python ~/Downloads/kubetop.py'
# kubetop