tomotaka

#!/usr/bin/python
# -*- coding: utf-8 -*-
from pprint import pprint, pformat
import random


def randomize(items):
    randomized = []
    while 0 < len(items):
        idx = random.randint(0, len(items)-1)

tomotaka

#!/usr/bin/python
# -*- coding: utf-8 -*-
import os
import sys
import hashlib
import gevent
import gevent.queue
import gevent.fileobject as gfo

tomotaka

class Tuple {
	public Object obj;
	public int idx;
	Tuple(Object o, int i) { this.obj = o; this.idx = i; }
}
List<Tuple> tmp = new ArrayList<Tuple>();
int i = 0;
for (Object obj : objects) {
	tmp.add(new Tuple(obj, i++));
}

tomotaka

import gevent.queue as gq

class ConcurrentCrawler(object):
    def __init__(self, func=crawl_all, concurrency=10, q_max=100):
        self._func = func
        self._concurrency = concurrency
        self._q_max = q_max

        self._queue = gq.Queue(maxsize=self._q_max)
        self._workers = None

tomotaka

# -*- coding: utf-8 -*-
import time
import hashlib
import os
from contextlib import contextmanager
import shutil

import plyvel
from sqlitedict import SqliteDict

tomotaka

import numpy as np
import random
import time
from contextlib import contextmanager

data = []
for i in xrange(300000):
    data.append(random.randint(1, 100000000))

tomotaka

# -*- coding: utf-8 -*-
import time

# pip install click plyvel simplejson
import click
import plyvel
import simplejson as json


@click.command()

tomotaka

# -*- coding: utf-8 -*-
import logging
import sys

def main():
    logger = logging.getLogger(__name__)
    logger.setLevel(logging.DEBUG)

    handler = logging.StreamHandler(sys.stdout)
    formatter = logging.Formatter('[%(levelname)s] %(message)s (%(filename)s:%(lineno)s)')

tomotaka

# -*- coding: utf-8 -*-
import logging
import sys

def main():
    logger = logging.getLogger(__name__)
    logger.setLevel(logging.DEBUG)

    handler = logging.FileHandler('/tmp/aaa.log', mode='ab', encoding='utf-8')
    formatter = logging.Formatter('[%(levelname)s] %(message)s (%(filename)s:%(lineno)s)')

tomotaka

class ExResourceLock

  def initialize(map)
    @rmap = map
    @mtx = Mutex.new
    @cv = ConditionVariable.new
    @mtx.synchronize do
      @free_resources = @rmap.keys
      @locked_resources = []
    end