Skip to content

Instantly share code, notes, and snippets.

View duydo's full-sized avatar

Duy Do duydo

View GitHub Profile
from __future__ import with_statement
from kombu import BrokerConnection
from collections import defaultdict
import gevent
from gevent import monkey
monkey.patch_all()
class WorkerHub():
"""
WorkerHub controls the local mailboxes that the @worker decorator assigns.
@duydo
duydo / gist:455ff86fcc9c74349307
Created September 16, 2014 10:36 — forked from cuimuxi/gist:3719516
Gevent Crawler
import gevent
from gevent import monkey, queue
monkey.patch_all()
import urllib2
from time import sleep
import traceback
import logging
__author__ = 'duydo'
from base64 import encodestring
from json import loads as json_encode
from requests import get
class SearchAPI(object):
MAX_RESULTS = 10
name: ${HOSTNAME}
# The cluster should be named with the name of the AD domain
cluster:
name: bldrprod-0.14.0
routing:
allocation:
# As small as possible, very large performance hits during recovery
concurrent_recoveries: 1
@duydo
duydo / gist:9587121
Last active March 21, 2018 12:15
Elasticsearch mutiple language mapping
{
"analysis": {
"filter": {
"ar_stop_filter": {
"type": "stop",
"stopwords": ["_arabic_"]
},
"bg_stop_filter": {
"type": "stop",
"stopwords": ["_bulgarian_"]
import simplejson
from boto.s3.connection import S3Connection
from boto.s3.key import Key
class S3KeyStore(object):
def __init__(self, access_key, secret_key, bucket):
self.conn = S3Connection(access_key, secret_key)
self.bucket = self.conn.create_bucket(bucket)
def get(self, key):
curl -s -XPOST localhost:9200/_bulk?pretty=true --data-binary '
{ "index" : { "_index" : "parent_child", "_type" : "store", "_id" : "store1" } }
{ "name" : "auchan", "owner" : "chris" }
{ "index" : { "_index" : "parent_child", "_type" : "department", "_id" : "department1", "parent" : "store1" } }
{ "name" : "toys", "numberOfProducts" : 150 }
{ "index" : { "_index" : "parent_child", "_type" : "product", "_id" : "product1", "parent" : "department1", "routing" : "store1" } }
{ "name" : "gun", "trademark" : "tiger", "price" : 9, "store_id" : "store1" }
'
@duydo
duydo / app.rb
Created December 10, 2013 13:42 — forked from cpatni/app.rb
require 'sinatra'
require 'redis'
require 'json'
require 'date'
class String
def &(str)
result = ''
result.force_encoding("BINARY")
@duydo
duydo / gist:7701304
Created November 29, 2013 03:45 — forked from simonw/gist:104413
def extract_form_fields(self, soup):
"Turn a BeautifulSoup form in to a dict of fields and default values"
fields = {}
for input in soup.findAll('input'):
# ignore submit/image with no name attribute
if input['type'] in ('submit', 'image') and not input.has_key('name'):
continue
# single element nome/value fields
if input['type'] in ('text', 'hidden', 'password', 'submit', 'image'):
#!/bin/bash
# from here: http://www.codingsteps.com/install-redis-2-6-on-amazon-ec2-linux-ami-or-centos/
# and here: https://raw.github.com/gist/257849/9f1e627e0b7dbe68882fa2b7bdb1b2b263522004/redis-server
###############################################
# To use:
# wget https://raw.github.com/gist/2776679/04ca3bbb9f085b192f6aca945120fe12d59f15f9/install-redis.sh
# chmod 777 install-redis.sh
# ./install-redis.sh
###############################################
echo "*****************************************"