Skip to content

Instantly share code, notes, and snippets.

@lmyyao
lmyyao / read_struct.py
Last active July 7, 2016 02:05
python read binary structure from file
from ctypes import *
libc = cdll.LoadLibrary("libc.so.6")
class POINT(Structure):
_fields_ = ("x", c_int), ("y", c_int)
f = open("test_structure", "w")
@lmyyao
lmyyao / tasks.py
Last active July 13, 2016 03:12
celery create periodic tasks in different ways
from celery.task import periodic_task
from datetime import timedelta
# first way
@periodic_task(run_every=timedelta(seconds=10), exchange="default", routing_key="default")
def every_10_seconds():
print("This is test")
return 1
@lmyyao
lmyyao / stop_thread.py
Created July 18, 2016 07:43
stop thread in python
import select
import threading
import time
class StoppableThread(threading.Thread):
"""Thread class with a stop() method. The thread itself has to check
regularly for the stopped() condition."""
def __init__(self):
@lmyyao
lmyyao / celery_scrapy_periodic.py
Last active July 19, 2016 06:45
Periodic Celery + Scrapy spider
from celery.task import PeriodicTask
from datetime import timedelta
class Lmy(PeriodicTask):
run_every = timedelta(seconds=60)
#celery queue router
options = {"exchange": "default", "routing_key": "default"}
name = "xxxxx"
@lmyyao
lmyyao / mulprocess_pipe.py
Last active July 27, 2016 02:50
pipe vs socketpair
import threading
class Consumer(threading.Thread):
def __init__(self, sock):
super(Consumer, self).__init__()
self.sock = sock
def run(self):
while True:
try:
@lmyyao
lmyyao / signal_timeout.py
Created July 27, 2016 07:03
python signal timeout
# encoding=utf8
import signal
import time
class TimeoutError(Exception):
pass
class Timeout(object):
@lmyyao
lmyyao / connection_pool.py
Created August 1, 2016 01:44
Connection Pool Demo
# from .celery import app
import heapq
import socket
from collections import UserDict
from functools import singledispatch, update_wrapper
from operator import itemgetter
from select import poll, POLLIN
from urllib.parse import urlparse
import httplib2
@lmyyao
lmyyao / sqlalchemy_pg_inherits.py
Created August 1, 2016 06:57
sqlalchemy PostgreSQLInheritance
# encoding=utf8
'''
https://bitbucket.org/zzzeek/sqlalchemy/wiki/UsageRecipes/PostgreSQLInheritance
'''
from sqlalchemy import create_engine
from sqlalchemy.ext.declarative import as_declarative, declared_attr
from sqlalchemy.orm import scoped_session, sessionmaker
engine = create_engine('postgresql:///test', convert_unicode=True, echo=False)
@lmyyao
lmyyao / udp_client.py
Last active August 2, 2016 02:50
python reading all the data from a UDP socket
import socket
import sys
'''
The correct maximum UDP message size is 65507,
as determined by the following formula: 0xffff - (sizeof(IP Header) + sizeof(UDP Header)) = 65535-(20+8) = 65507
http://stackoverflow.com/questions/1098897/what-is-the-largest-safe-udp-packet-size-on-the-internet
search gateway MTU:
ping -s 1500 -M do <gateway>
'''
messages = [b'x' * 65507] * 3
@lmyyao
lmyyao / limit_program_memory.markdown
Created August 5, 2016 09:41
create cgroup limit process

Limit Program Memory By Cgroup

  1. cgcreate -g memory:/myGroup
  2. echo $(( 500 * 1024 * 1024 )) > /sys/fs/cgroup/memory/myGroup/memory.limit_in_bytes(notes: centos cgroup mount on /sys/fs/cgroup)
  3. echo '@<username>:<command> memory /myGroup' >> /etc/cgrules.conf
  4. systemctl restart cgred
  5. cgget -g memory /myGroup (查看/myGroup的配置,以及内存使用情况)
  6. cat /proc//cgroup