Skip to content

Instantly share code, notes, and snippets.

Avatar

Cheewai cynici

View GitHub Profile
@cynici
cynici / shp-to-pg.py
Last active Jul 12, 2016
Example using Python coroutine to insert large dataset into PostgreSQL in batches
View shp-to-pg.py
#
# CAVEAT: untested code intended to demonstrate idea only
#
from osgeo import ogr, osr
import psycopg2
# Read gory details http://www.dabeaz.com/coroutines/Coroutines.pdf
def coroutine(func):
def start(*args,**kwargs):
cr = func(*args,**kwargs)
@cynici
cynici / pginsert-customize.py
Created May 31, 2016
Customise raster2pgsql, shp2pgsql INSERT statements
View pginsert-customize.py
#! /usr/bin/env python
import sys
import os
import argparse
import re
import logging
help_text = """Get input from stdin and modify INSERT statements. Useful for customizing output from PostGIS shp2pgsql, raster2pgsql."""
def modify_statement(st, args, clist, vlist, do_delete=False):
@cynici
cynici / haproxy.cfg
Created Apr 30, 2016 — forked from GABeech/haproxy.cfg
Stack Exchange HAProxy
View haproxy.cfg
# This is an example of the Stack Exchange Tier 1 HAProxy config
# The only things that have been changed from what we are running are:
# 1. User names have been removed
# 2. All Passwords have been remove
# 3. IPs have been changed to use the example/documentation ranges
# 4. Rate limit numbers have been changed to randome numbers, don't read into them
userlist stats-auth
group admin users $admin_user
user $admin_user insecure-password $some_password
@cynici
cynici / gist:7300d065fb38759e13a84213384f9a9e
Created Apr 20, 2016 — forked from willejs/gist:043ffaeec61c4501760e
haproxy 1.5 logstash grok filter - working!
View gist:7300d065fb38759e13a84213384f9a9e
HAPROXYTIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])
HAPROXYDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{HAPROXYTIME}.%{INT}
HAPROXYHTTP <%{BASE10NUM}>%{SYSLOGTIMESTAMP} %{SYSLOGPROG}: %{IP:client}:%{INT:port} \[%{HAPROXYDATE:accept_date}\] %{NOTSPACE:frontend_name} %{NOTSPACE:backend_name}/%{NOTSPACE:server_name} %{INT:time_request}/%{INT:time_queue}/%{INT:time_backend_connect}/%{INT:time_backend_response}/%{NOTSPACE:time_duration} %{INT:http_status_code} %{NOTSPACE:bytes_read} %{DATA:captured_request_cookie} %{DATA:captured_response_cookie} %{NOTSPACE:termination_state} %{INT:actconn}/%{INT:feconn}/%{INT:beconn}/%{INT:srvconn}/%{NOTSPACE:retries} %{INT:srv_queue}/%{INT:backend_queue} \"(<BADREQ>|(%{WORD:http_verb} (%{URIPROTO:http_proto}://)?(?:%{USER:http_user}(?::[^@]*)?@)?(?:%{URIHOST:http_host})?(?:%{URIPATHPARAM:http_request})?( HTTP/%{NUMBER:http_version})?))?\"
@cynici
cynici / ubuntu.seed
Last active Dec 26, 2015
64-bit Ubuntu 12.04.03 Desktop preseed configuration file
View ubuntu.seed
####
#### Contents of the preconfiguration file (for Ubuntu 12.04 Precise)
#### Recipe: http://cynici.wordpress.com/2013/10/30/custom-install-of-ubuntu-12-04-desktop-via-usb-flash/
####
### Localization
# Preseeding only locale sets language, country and locale.
d-i debian-installer/locale string en_US
# The values can also be preseeded individually for greater flexibility.
#d-i debian-installer/language string en
@cynici
cynici / afisparser.py
Last active Feb 7, 2018
Using pyparsing to reduce expressions into a single boolean value. An expression may contain simple arithmetic and keywords (substituted with values from user-given parameter dictionary).
View afisparser.py
import sys
import traceback
import logging
from pyparsing import *
ParserElement.enablePackrat()
def EvalSignOp(s, l, t):
"Evaluate expressions with a leading + or - sign"
sign, value = t[0]
mult = {'+':1, '-':-1}[sign]
@cynici
cynici / modisraw.py
Created Feb 26, 2013
dirmon.py plugin module to deal with incoming MODIS raw PDS files from Orbital Systems
View modisraw.py
import os, re
terra_re = re.compile(r'P0420064A{14}(?P<timestamp>\d{11})(?P<one>\d{3})\.PDS$')
terra_fn = 'P0420064AAAAAAAAAAAAAA%(timestamp)s001.PDS'
data_re = re.compile(r'^P1540064A{14}(?P<timestamp>\d{11})(?P<one>\d{3})\.PDS$')
data_fn = 'P1540064AAAAAAAAAAAAAA%(timestamp)s001.PDS'
gbad_re = re.compile(r'^P1540957A{14}(?P<timestamp>\d{11})(?P<one>\d{3})\.PDS$')
gbad_fn = 'P1540957AAAAAAAAAAAAAA%(timestamp)s001.PDS'
@cynici
cynici / avafo_h5_to_csv.py
Last active Dec 13, 2015
Convert CIMSS CSPP S-NPP VIIRS-AF output from HDFv5 to CSV
View avafo_h5_to_csv.py
#!/usr/bin/env python
#
# Author: cheewai.lai@gmail.com
# Source URL: https://gist.github.com/cynici/4984007
#
import os, sys, re
from optparse import OptionParser
from datetime import datetime
import logging
@cynici
cynici / gist:4671537
Last active Dec 11, 2015
Steps to reinstall NPP DRL IPOPP on Ubuntu 12.04 64-bit
View gist:4671537
#
# Reset corrupted IPOPP database
#
$HOME/drl/tools/services.sh stop
$HOME/drl/dsm/bin/erase_database.sh
rm -rf /raid/pub/ancillary/*
$HOME/drl/tools/services.sh start
@cynici
cynici / firedanger_chart.html
Created Jan 24, 2013
firedanger plot using jquery flot library written as python django template
View firedanger_chart.html
{% extends "base.html" %}
{% load i18n %}
{% block css %}
<style type="text/css">
.calendar {
font-family: 'Trebuchet MS', Tahoma, Verdana, Arial, sans-serif;
font-size: 0.9em;
background-color: #EEE;
color: #333;