vadimkantorov

def load_dotenv(dotenv_path = '.env'):
    # https://snarky.ca/use-toml-for-env-files/
    # https://github.com/theskumar/python-dotenv
    '''
    # such simple key-value files are toml subset and can be read via tomllib without external packages or hacks
    a="b"
    c="d"
    '''
    import os, tomllib
    os.environ.update(tomllib.load(open(dotenv_path, 'rb')))

vadimkantorov

import sys, fsspec
with fsspec.open(sys.argv[1], 'rt') as f: # must pass 'rt' explicitly, as in fsspec the default mode is 'rb'
    print(f.read()) # msut use context manager as in fsspec the result of fsspec.open(...) does not have method read()

# echo world > hello.txt
# python catfsspec.py file://hello.txt

vadimkantorov

# reference: https://gist.github.com/0xjac/85097472043b697ab57ba1b1c7530274

git clone --bare git@github.com:volcengine/verl.git
cd verl.git

# create a bare repo vaidmkantorov/verl
git push --mirror git@github.com:vadimkantorov/verl.git
cd .. && rm -rf verl.git

git clone git@github.com:vadimkantorov/verl.git

vadimkantorov

# Save as tqdm.py in project dir, then `from tqdm import tqdm; from tqdm.auto import tqdm` should pick up this class, if fails use export PYTHONPATH=.
# Test run: python tqdm.py

import os, sys

# huggingface_hub/hf_api.py:
# from tqdm.auto import tqdm as base_tqdm
# from tqdm.contrib.concurrent import thread_map
# https://tqdm.github.io/docs/shortcuts/#tqdmauto
sys.modules['tqdm.auto'] = sys.modules[__name__]

vadimkantorov

# https://github.com/pypa/pip/issues/11440
# https://github.com/pypa/pip/issues/7822
# https://stackoverflow.com/a/79598932/445810
# tomllib is available starting from python --version >= 3.11

python -m pip install $(python -c 'import tomllib;print(*tomllib.load(open("pyproject.toml","rb"))["project"]["dependencies"])') # --user --break-system-packages

vadimkantorov

import http.client
http.client.HTTPConnection.debuglevel = 1

vadimkantorov

# https://superuser.com/questions/1687960/over-ssh-can-you-use-the-same-private-key-on-the-host-side-for-other-purposes
alias sshagentssh='ssh-agent ssh -A -o AddKeysToAgent=yes'

# generate ssh key for github
# https://docs.github.com/en/authentication/connecting-to-github-with-ssh/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent
ssh-keygen -t ed25519 -b 4096 -C "vadimkantorov@gmail.com" -f ./id_ed25519 -N="" # -q

# https://stackoverflow.com/questions/4565700/how-to-specify-the-private-ssh-key-to-use-when-executing-shell-command-on-git
# https://github.com/settings/ssh/new
export GIT_SSH_COMMAND="ssh -o IdentitiesOnly=yes -i $PWD/id_ed25519"

vadimkantorov

# Usage: python parquet2npyztsv.py test.npy data/train-*-of-*.parquet
# Usage: python parquet2npyztsv.py test.npz data/train-*-of-*.parquet
# Usage: python parquet2npyztsv.py test.tsv data/train-*-of-*.parquet

import sys
import numpy as np
import pyarrow.parquet as pq

output_path, *input_paths = sys.argv[1:]

vadimkantorov

# Usage: bash git_lfs_clone_dedup.sh https://huggingface.co/deepseek-ai/DeepSeek-V3-0324 ~/DeepSeek-V3-0324
# Usage: bash git_lfs_clone_dedup.sh git@hf.co:deepseek-ai/DeepSeek-V3-0324 ~/DeepSeek-V3-0324
# https://github.com/git-lfs/git-lfs/discussions/6029

GIT_LFS_SKIP_SMUDGE=1 git clone $1 $2
cd $2
git lfs fetch
git lfs ls-files -l | while read SHA DASH FILEPATH; do rm "$FILEPATH" && ln ".git/lfs/objects/${SHA:0:2}/${SHA:2:2}/$SHA" "$FILEPATH"; done

#git lfs ls-files -l | while read SHA DASH FILEPATH; do mv ".git/lfs/objects/${SHA:0:2}/${SHA:2:2}/$SHA" "$FILEPATH"; done

vadimkantorov

sudo apt-get install git-lfs
git lfs install

# git clone https://huggingface.co/deepseek-ai/DeepSeek-V3-0324
# du -sh DeepSeek-V3-0324
# # 1.3T    DeepSeek-V3-0324/
# du -sh DeepSeek-V3-0324/.git/lfs
# # 642G    DeepSeek-V3-0324/.git/lfs

# https://github.com/git-lfs/git-lfs/discussions/6029