Skip to content

Instantly share code, notes, and snippets.

@clarksun
clarksun / vimrc
Last active December 8, 2019 12:55
vim配置 #vim
set shell=/bin/bash
" config Vundle
filetype off
set rtp+=~/.vim/bundle/Vundle.vim
call vundle#begin()
Plugin 'VundleVim/Vundle.vim'
Plugin 'ctrlpvim/ctrlp.vim' " fuzzy find files
Plugin 'scrooloose/nerdtree' " file drawler, open with :NERDTreeToggle
@clarksun
clarksun / bash_profile
Created September 18, 2017 01:18
bash_profile配置 #shell
# config for python lib 'thefuck'
eval "$(thefuck --alias)"
eval "$(thefuck --alias fuck)"
function mkdircd () { mkdir -p "$@" && eval cd "\"\$$#\""; }
# linux core bash命令, mac是-G, linux是--color=auto
#alias ls='ls --color=auto'
#alias ls='ls -G'
alias vim="mvim -v"
@clarksun
clarksun / retry_and_rollback.py
Last active September 19, 2017 11:59
重试回滚注解 #decorator
# frontera/contrib/backends/sqlalchemy/revisiting.py
def retry_and_rollback(func):
def func_wrapper(self, *args, **kwargs):
tries = 5
while True:
try:
return func(self, *args, **kwargs)
except Exception as exc:
self.logger.exception(exc)
@clarksun
clarksun / base_model.py
Last active September 19, 2017 11:44
Model 基类 #sqlalchemy
# frontera/utils/graphs/models.py
Base = declarative_base()
class BaseModel(object):
__abstract__ = True
@classmethod
def get_pk_name(cls):
return cls.__mapper__.primary_key[0].name
@clarksun
clarksun / manager.py
Created September 19, 2017 11:57
使用 #sqlalchemy
# frontera/utils/graphs/manager.py
DEFAULT_ENGINE = 'sqlite:///:memory:'
class CrawlGraphManager(object):
def __init__(self, engine=DEFAULT_ENGINE, autocommit=False, autoFlush=False,
echo=False, drop_all_tables=False, clear_content=False):
self.engine = create_engine(engine, echo=echo)
if drop_all_tables:
Base.metadata.drop_all(self.engine)
@clarksun
clarksun / load_object.py
Created September 20, 2017 07:10
load_object by path
# frontera/utils/misc.py
from importlib import import_module
def load_object(path):
"""Load an object given its absolute object path, and return it.
object can be a class, function, variable o instance.
path ie: 'myproject.frontier.models.Page'
"""
@clarksun
clarksun / shell_prompt.sh
Created September 27, 2017 05:34
shell提示是否运行
# https://stackoverflow.com/questions/226703/how-do-i-prompt-for-yes-no-cancel-input-in-a-linux-shell-script
while true; do
read -p "Do you wish to install this program?" yn
case $yn in
[Yy]* ) make install; break;;
[Nn]* ) exit;;
* ) echo "Please answer yes or no.";;
esac
done
@clarksun
clarksun / priority_queue.py
Created October 3, 2017 02:26
优先级队列 #heapq
import heapq
class PriorityQueue:
def __init__(self):
self._queue = []
self._index = 0
def push(self, item, priority):
heapq.heappush(self._queue, (-priority, self._index, item))
self._index += 1
@clarksun
clarksun / kafka_topic_msg_count.sh
Created October 9, 2017 06:36
get kafka topic message count
kafka-run-class kafka.tools.GetOffsetShell --broker-list localhost:9092 --topic xxx --time -1 --offsets 1 | awk -F ':' '{sum += $3} END {print sum}'
@clarksun
clarksun / CaselessDict.py
Created November 23, 2017 08:31
忽略大小写key的dict from scrapy #python
class CaselessDict(dict):
__slots__ = ()
def __init__(self, seq=None):
super(CaselessDict, self).__init__()
if seq:
self.update(seq)
def __getitem__(self, key):
return dict.__getitem__(self, self.normkey(key))