danfairs

"""
Utilities for working with bulk data and batches.
"""
import itertools


def batches(items, batch_size=500):
    """
    Given an iterable of items and a batch size, yield individual lists
    of items of maximum length `batch_size`.

danfairs

import base64
import hashlib
import hmac
import simplejson as json

def base64_url_decode(inp):
    padding_factor = (4 - len(inp) % 4) % 4
    inp += "="*padding_factor 
    return base64.b64decode(unicode(inp).translate(dict(zip(map(ord, u'-_'), u'+/'))))

danfairs

// Here's the simple version - first we have a conversion function, that takes a *ContentItem and converts to a *contentpb.ContentItem.
func ContentItemToPb(c *ContentItem) (*contentpb.ContentItem, error) {
	if c == nil {
		return nil, errors.New("ContentItemToPb: ContentItem was nil")
	}
	return &contentpb.ContentItem{
		Id: &contentpb.ContentId{
			SourceId: c.Id.SourceId,
			SourceName: c.Id.SourceName,
		},

danfairs

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  labels: {name: twist-server, role: app}
  name: twist-server
spec:
  replicas: 1
  selector:
    matchLabels:
      name: twist-server

danfairs

FROM ubuntu:16.04
MAINTAINER Dan Fairs <dan.fairs@gmail.com>
RUN apt-get update && apt-get install -y python3-pip
RUN mkdir /app
COPY main.py /app
COPY requirements.txt /app
RUN pip3 install --upgrade pip
RUN pip3 install -r /app/requirements.txt
EXPOSE 8888
CMD python3 /app/main.py

danfairs

$ gcloud compute disks list
NAME                                 ZONE            SIZE_GB  TYPE         STATUS
disk100-001                          europe-west1-d  100      pd-standard  READY
gke-unrest-micro-pool-199acc6c-3p31  europe-west1-d  100      pd-standard  READY
gke-unrest-micro-pool-199acc6c-4q55  europe-west1-d  100      pd-standard  READY

$ kubectl get pv
NAME             CAPACITY   ACCESSMODES   STATUS    CLAIM                           REASON    AGE
pv-disk100-001   100Gi      RWO           Bound     default/graphite-statsd-claim             2m

danfairs

Keybase proof

I hereby claim:

• I am danfairs on github.
• I am danfairs (https://keybase.io/danfairs) on keybase.
• I have a public key ASC0Yu-wF_zELLNx9mkEVloWIzjxa1_MxItY6qRbszjfzQo

To claim this, I am signing this object:

danfairs

locust$ vagrant up
Bringing machine 'default' up with 'virtualbox' provider...
[default] Box 'precise64' was not found. Fetching box from specified URL for
the provider 'virtualbox'. Note that if the URL does not have
a box for this provider, you should interrupt Vagrant now and add
the box yourself. Otherwise Vagrant will attempt to download the
full box prior to discovering this error.
Downloading or copying the box...
Extracting box...te: 910k/s, Estimated time remaining: 0:00:02))
Successfully added box 'precise64' with provider 'virtualbox'!

danfairs

pgbouncer  3585   postgres   27u     IPv4   12329827       0t0        TCP 127.0.0.1:43238->127.0.0.1:postgresql (ESTABLISHED)
postgres  12363   postgres    9u     IPv4   12335304       0t0        TCP 127.0.0.1:postgresql->127.0.0.1:43238 (ESTABLISHED)

danfairs

Non-default config:

cluster.name: staging-es
http.port: 9600
transport.tcp.port: 9700
index.cache.field.type: soft
script.native:
    excludetags.type: com.secondsync.elasticsearch.script.ExcludeTagsScriptFactory