JohannesBuchner

# columns: id1,id2,id3,id4,id5,id6,v1,v2,v3

f=G1_1e7_1e2_0_0.csv
awk="time mawk"

# groupby simple
$awk -F, 'NR>1 { a[$1] += $7 } END {for (i in a) print i, a[i]}' $f >/dev/null
$awk -F, 'NR>1 { a[$1,$2] += $7 } END { for (comb in a) { split(comb,sep,SUBSEP); print sep[1], sep[2], a[sep[1],sep[2]]; }}' $f >/dev/null
$awk -F, 'NR>1 { a[$3] += $7; n[$3]++; b[$3] += $9; } END {for (i in a) print i, a[i], b[i]/n[i];}' $f >/dev/null
$awk -F, 'NR>1 { a[$4] += $7; n[$4]++; b[$4] += $8; } END {for (i in a) print i, a[i]/n[i], b[i]/n[i];}' $f >/dev/null

JohannesBuchner

import re
import pystan
import hashlib
import pickle
import os

def build_model(code):
    lines = code.split("\n")
    lines = [re.sub('//.*$', '', line).strip() for line in lines]
    lines = [line.replace('    ', ' ').replace('  ', ' ').replace('  ', ' ')

JohannesBuchner

#!/bin/bash

# how to identify the processes to STOP/CONT
# this is any part of process command line
PROCESS="myscript.py"

# run this in one terminal:
# it tries to resume the processes every so often
while sleep 10m; do
pgrep -f "${PROCESS}"|xargs -rt kill -CONT

JohannesBuchner

"""
If you ever made a plot of "fractions" or "rates" with symmetric error 
bars, like in the plot shown, this mini-tutorial is for you. Here is how
 to compute correct error bars, so that uncertainties in the fractions 
do not go below 0% or above 100%.

 If you have a histogram for 
instance, and you detected k objects in a given bin. What is the rate 
underlying at quantiles q=0.1, 0.5(median), 0.9?
"""

JohannesBuchner

import numpy as np

def mcmc(logfunction, x0, nsteps, sigma_p):
    samples = np.empty((nsteps, len(x0)))
    logL0 = logfunction(x0)
    naccepts = 0
    for i in range(nsteps):
        x1 = np.random.normal(x0, sigma_p)
        logL1 = logfunction(x1)
        if logL1 - logL0 > np.log(np.random.uniform()):

JohannesBuchner

import sys
import numpy
import h5py

filename = sys.argv[1]
outfilename = filename.replace('.npy', '').replace('.gz', '').replace('.csv', '') + '.h5'
if filename.endswith('.npy'):
    print('loading NPY...')
    data = numpy.load(filename)
else:

JohannesBuchner

"""
Download custom spectrum from http://www.swift.ac.uk/, programmatically

How to use:

$ python fetchswiftspectrum.py grbid timeslicespec
$ wget -nc --continue $(cat customspec.url)/a.tar.gz

"""

JohannesBuchner

wget -nc http://cdn.gea.esac.esa.int/Gaia/gdr2/gaia_source/csv/

grep -Eo GaiaSource_.*.csv.gz index.html | while read i; do

if [ -e $i ]; then
	true
else
	wget "http://cdn.gea.esac.esa.int/Gaia/gdr2/gaia_source/csv/$i" -O - | zcat | LC_ALL=C awk -F, '{print $2,$6,$8,sqrt($7**2+$9**2),$48}' | tail -n +2 | gzip > $i.tmp && mv $i.tmp $i
fi

JohannesBuchner

#!/bin/bash
# Author: Johannes Buchner (C) 2013
# tool for publishing sphinx documentation on github
#      and releasing packages on PyPI

case "$1" in
doc)
	# see https://help.github.com/articles/creating-project-pages-manually
	make -C doc/ html && 
	git checkout gh-pages && 

JohannesBuchner

# To submit just at the right time (deadline 14:00 EST), 
# 
# How to use:
#  - Make sure your clock is synced to https://arxiv.org/localtime
#  - Prepare arxiv submission up to last page, move cursor over submission button
#  - execute below function which will produce a left-button mouse click at 14:00
#  - if too early, unsubmit, fix clock, try again the next day
# 
# Hopefully one day arxiv will randomize their submission ordering.