eddienko

"""
Custom Authenticator to use generic OAuth2 with JupyterHub
"""


import sys
import json
import os
import base64
import urllib

eddienko

#!/bin/sh
 
errorExit() {
    echo "*** $*" 1>&2
    exit 1
}
 
APISERVER_VIP=xxx.xxx.xx.xx
APISERVER_DEST_PORT=6443
 

eddienko

global_defs {
    router_id LVS_DEVEL
}
vrrp_script check_apiserver {
    script "/etc/keepalived/check_apiserver.sh"
    interval 3
    weight -2
    fall 10
    rise 2
}

eddienko

distributed:
  version: 2
  scheduler:
    bandwidth: 1000000000     # 100 MB/s estimated worker-worker bandwidth
  worker:
    memory:
      target: 0.90  # target fraction to stay below
      spill: False  # fraction at which we spill to disk
      pause: 0.80  # fraction at which we pause worker threads
      terminate: 0.95  # fraction at which we terminate the worker

eddienko

#!/usr/bin/env python3
"""
Update Dask configuration based on the configuration
of the running Pod.

To be run at startup.
"""

import os

eddienko

eddienko

# Makefile for converting the CSV files from http://cdn.gea.esac.esa.int/Gaia/gdr2/gaia_source/csv/
# to a single (vaex) hdf5 file
#  * https://docs.vaex.io
#  * https://github.com/maartenbreddels/vaex/
# It is multistage to work around opening 60 000 files at once.
# Strategy is
#  * stage1: convert all cvs.gz to csv to hdf5
#   * do this via xargs and calling make again, since gmake has trouble matching 60 000 rules
#  * stage2: Create part-<NUMBER>.txt files containing max FILES_PER_PART per file  
#  * stage3: convert the list of hdf5 files to single hdf5 files (part-<NUMBER>.hdf5)

eddienko

eddienko

eddienko

import pandas as pd

def parse(line):
    words = line.split()
    time = words[0]
    protocol = words[1]

    if protocol == 'IP':
        src_ip, src_port = words[2].rsplit('.', 1)
        dst_ip, dst_port = words[4].strip(':').rsplit('.', 1)