danfairs

"""
Utilities for working with bulk data and batches.
"""
import itertools


def batches(items, batch_size=500):
    """
    Given an iterable of items and a batch size, yield individual lists
    of items of maximum length `batch_size`.

danfairs

import base64
import hashlib
import hmac
import simplejson as json

def base64_url_decode(inp):
    padding_factor = (4 - len(inp) % 4) % 4
    inp += "="*padding_factor 
    return base64.b64decode(unicode(inp).translate(dict(zip(map(ord, u'-_'), u'+/'))))

danfairs

// Here's the simple version - first we have a conversion function, that takes a *ContentItem and converts to a *contentpb.ContentItem.
func ContentItemToPb(c *ContentItem) (*contentpb.ContentItem, error) {
	if c == nil {
		return nil, errors.New("ContentItemToPb: ContentItem was nil")
	}
	return &contentpb.ContentItem{
		Id: &contentpb.ContentId{
			SourceId: c.Id.SourceId,
			SourceName: c.Id.SourceName,
		},

danfairs

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  labels: {name: twist-server, role: app}
  name: twist-server
spec:
  replicas: 1
  selector:
    matchLabels:
      name: twist-server

danfairs

FROM ubuntu:16.04
MAINTAINER Dan Fairs <dan.fairs@gmail.com>
RUN apt-get update && apt-get install -y python3-pip
RUN mkdir /app
COPY main.py /app
COPY requirements.txt /app
RUN pip3 install --upgrade pip
RUN pip3 install -r /app/requirements.txt
EXPOSE 8888
CMD python3 /app/main.py

danfairs

$ gcloud compute disks list
NAME                                 ZONE            SIZE_GB  TYPE         STATUS
disk100-001                          europe-west1-d  100      pd-standard  READY
gke-unrest-micro-pool-199acc6c-3p31  europe-west1-d  100      pd-standard  READY
gke-unrest-micro-pool-199acc6c-4q55  europe-west1-d  100      pd-standard  READY

$ kubectl get pv
NAME             CAPACITY   ACCESSMODES   STATUS    CLAIM                           REASON    AGE
pv-disk100-001   100Gi      RWO           Bound     default/graphite-statsd-claim             2m

danfairs

Keybase proof

I hereby claim:

• I am danfairs on github.
• I am danfairs (https://keybase.io/danfairs) on keybase.
• I have a public key ASC0Yu-wF_zELLNx9mkEVloWIzjxa1_MxItY6qRbszjfzQo

To claim this, I am signing this object:

danfairs

Here's my email address:

dan.fairs@gmail.com

danfairs

Elasticsearch configuration for high sustainable bulk feed

Test on single node, MacBook Pro, 16 GB RAM, 1TB SSD, OS X Maverick

ES 1.1.0 with Java 8, G1 GC, 12 GB heap

/Library/Java/JavaVirtualMachines/jdk1.8.0.jdk/Contents/Home/bin/java -Xms12g -Xmx12g -Djava.awt.headless=true -XX:+UseG1GC -Delasticsearch -Des.foreground=yes -Des.path.home=/Users/es/elasticsearch-1.1.0 -cp :/Users/es/elasticsearch-1.1.0/lib/elasticsearch-1.1.0.jar:/Users/es/elasticsearch-1.1.0/lib/:/Users/es/elasticsearch-1.1.0/lib/sigar/ org.elasticsearch.bootstrap.Elasticsearch

Node

danfairs

(fabtools)skater:fabtools danfairs$ FABTOOLS_TEST_BOXES='precise64 opscode-ubuntu-13.04' tox -epy27
GLOB sdist-make: /Users/danfairs/Envs/fabtools/src/fabtools/setup.py
py27 inst-nodeps: /Users/danfairs/Envs/fabtools/src/fabtools/.tox/dist/fabtools-0.18.0-dev.zip
py27 runtests: PYTHONHASHSEED='559934745'
py27 runtests: commands[0] | /Users/danfairs/Envs/fabtools/src/fabtools/.tox/py27/bin/python -m unittest discover
................................[localhost] local: rm -f Vagrantfile
[localhost] local: vagrant init precise64
A `Vagrantfile` has been placed in this directory. You are now
ready to `vagrant up` your first virtual environment! Please read
the comments in the Vagrantfile as well as documentation on