Skip to content

Instantly share code, notes, and snippets.

View skliarpawlo's full-sized avatar

Pavlo Skliar skliarpawlo

  • Tubular
  • Ukraine, Kiev
View GitHub Profile
@skliarpawlo
skliarpawlo / huggingface_example.py
Last active May 6, 2023 17:31
Ray HuggingfaceTrainer problem
from datasets import load_dataset
import transformers
from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
import ray
from ray import tune, air
from ray.train.huggingface import HuggingFaceTrainer
from ray.air.config import ScalingConfig
import os
;;
;; Matcher should recognize and destruct URL by:
;; host: domain
;; path: parts, splitted with "/"
;; queryparam: name/value pairs of query
;; (see examples below)
;;
;; Each string that is started from "?" is a "bind"
;; (recognize matcher) should return nil or seq of binds
;;
This file has been truncated, but you can view the full file.
============================= test session starts ==============================
platform darwin -- Python 3.5.2, pytest-3.2.3, py-1.7.0, pluggy-0.4.0 -- /usr/local/bin/python3.5
cachedir: ../.cache
rootdir: /private/var/tmp/_bazel_pavloskliar/4ebdf5e551906a3d6bd81a9029c30c48/execroot/__main__/bazel-out/darwin-fastbuild/bin/tool_castor/castor_integration_tests.runfiles/__main__, inifile: tox.ini
collecting ... INFO:dd.datadogpy:No agent or invalid configuration file found
collected 63 items
tests/integration/test_activities_merger.py::TestActivitiesMergerWorkflow::test_abort_with_corrupted_delta INFO:root:Starting the new global session for <class 'castor.TestCastorSession'>
Warning: Ignoring non-spark config property: es.nodes.wan.only=true
https://nexus.tubularlabs.net/repository/libs-release-local/ added as a remote repository with the name: repo-1
@skliarpawlo
skliarpawlo / emacs-docker-compose.yml
Last active February 6, 2018 17:23
use emacs docker compose
...
emacs:
build:
dockerfile: /Users/psk/dev/dotfiles/ide/Dockerfile
context: /Users/psk/dev/dotfiles/ide/
args:
- SUBDIR=tool-castor
environment:
- DISPLAY=$DISPLAY
- SUBDIR=tool-castor
docker run --name mysql-test -e MYSQL_DATABASE='tube_test' -e MYSQL_USER='jenkins' -e MYSQL_PASSWORD='jenkins' -e MYSQL_ALLOW_EMPTY_PASSWORD='yes' -d mysql:5.5
docker run --name redis-test -d redis
docker run --name cassandra-test -m 500M --memory-reservation 100M --memory-swap 100M -e "MAX_HEAP_SIZE=300M" -e "HEAP_NEWSIZE=100M" -d cassandra
docker run --name rabbitmq-test -d rabbitmq:3.6.2-management
# docker rm $(docker ps -a -q)
# docker rmi $(docker images | grep "^<none>" | awk "{print $3}")
# docker volume rm $(docker volume ls -qf dangling=true)
@skliarpawlo
skliarpawlo / tmuxify.sh
Last active April 19, 2016 20:27
run muliple equal commands under single tmux session
#!/bin/bash
# $1 - session name
# $2 - num of processes
# $3 - cmd
tmux new-session -s $1 -d
for i in `seq $2`
do
tmux select-pane -t $1.0
#!/usr/bin/env python3
import argparse
import os
from pyspark import SparkContext, SparkConf
from pyspark.sql import HiveContext
args_parser = argparse.ArgumentParser('YT Activities join')
args_parser.add_argument(
'--drain',
@skliarpawlo
skliarpawlo / str_generator.py
Created November 25, 2015 18:21
str_generator
from itertools import product
def generate_from_pattern(pattern, open_sep='{{', close_sep='}}'):
""" Generates all possible variations that fits pattern.
For example:
>>> generate_from_pattern('number {{9-11}}')
['number 9', 'number 10', 'number 11']
>>> generate_from_pattern('number {{1-3,11}}')
@skliarpawlo
skliarpawlo / commit-msg
Created November 18, 2015 15:14
git hooks
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os
import re
import sys
EMO = os.environ.get('TUBULAR_COMMIT_EMOJI', '👮 ')
SHORT_DESCRIPTION_LINE_REGEX = re.compile('^((?:[A-Z]+-\d+)|FIX|HOTFIX|IMP) (.{1,50})$')
@skliarpawlo
skliarpawlo / parse_json_from_string.py
Created July 16, 2015 12:42
Parse json from piece of html/script
def _get_json_from_string(s):
"""Parses first found json object from string (piece of html).
>>> _get_json_from_string('{"a": "b"}')
{u'a': u'b'}
>>> _get_json_from_string('dskljasd{"a": "b"}asdkljasd')
{u'a': u'b'}
>>> _get_json_from_string('require(asd=[{"a": "b"}, {"c":"d"}]spamspamspam)')
[{u'a': u'b'}, {u'c': u'd'}]
>>> _get_json_from_string('{"a": [1,2,{"b":"d[1,2,3]sd"}]}')