sudo aptitude install git php5-cli nodejs
npm install -g npm
npm install -g maxogden/dat
dat init
dat listen
- Create a Twitter application and generate OAuth keys for your user - add the app and user credentials to
stream.php
. composer init && composer require fennb/phirehose:dev-master && composer install
php stream.php | dat import -json
- Install and start dat-editor then open dat-editor to browse the table.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var red = '#c00' | |
var white = '#fff' | |
console.log('<table border=0 cellspacing=0 cellpadding=0 style="line-height:10px">\n' + ([ | |
'111111111111111111', | |
'1 1 1 1', | |
'1 1 1 1 1 1 1 1', | |
'1 1 1 1 1 1 1 1', | |
'1 1 1 1 1 1 1', | |
'111111 1111111111', |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"name": "fcc-ecfs", | |
"version": "1.0.0", | |
"gasket": { | |
"main": [ | |
"gasket run import -- http://www.fcc.gov/files/ecfs/14-28/14-28-RAW-Solr-1.xml", | |
null, | |
"gasket run import -- http://www.fcc.gov/files/ecfs/14-28/14-28-RAW-Solr-2.xml", | |
null, | |
"gasket run import -- http://www.fcc.gov/files/ecfs/14-28/14-28-RAW-Solr-3a.xml", |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var once = require('once') | |
var concurrent = 10 | |
var pending = 0 | |
through(function(data, enc, cb) { | |
cb = once(cb) | |
pending++ | |
if (pending < concurrent) cb() | |
rr().write(data, function(err) { | |
if (err) return cb(err) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# A quick function to save a PBM (http://en.wikipedia.org/wiki/Netpbm_format) | |
# visualize *a lot* of missing data pretty quickly (outside of R). | |
writeMissingPBM <- function(x, file) { | |
dims <- dim(x) | |
x[] <- as.integer(is.na(x)) | |
con <- file(file, open="wt") | |
writeLines(sprintf("P1\n%d %d", ncol(x), nrow(x)), con) | |
write.table(x, file=con, sep=" ", col.names=FALSE, row.names=FALSE, quote=FALSE) | |
close(con) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
https://www.youtube.com/watch?x-yt-cl=84359240&x-yt-ts=1421782837&feature=player_embedded&v=ZACVcJt0oJA#t=7303 | |
Kai Blin: If we containerize all these things won’t it just encourage | |
worse software development practices; right now developers still need | |
to consider someone other than themselves installing the software. | |
Michael Barton: | |
“It’s a good point. Ultimately, though, if I can get a container, and | |
it works, and I know it will work, do you care how well it was |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env node | |
var fs = require('fs'); | |
var fasta = require('bionode-fasta'); | |
var crypto = require('crypto'); | |
fasta.obj('proteins.fasta').on('data', function(x) { | |
var seq = x.seq | |
.replace(/\*/g, '') // remove asterisks (stop codons) | |
.toUpperCase(); // enforce case |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var atomicQueue = require(...) | |
var queue = atomicQueue(db, function (data, cb) { | |
console.log('got some work', data) | |
// do stuff ... | |
cb(null, result) // if no error this work will be removed from the leveldb | |
}) | |
// write will add this 'work' to the leveldb | |
queue.write({some: 'world'}) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
library(magrittr) | |
write.table(iris, file="iris.txt", sep="\t", row.names=FALSE, quote=FALSE) | |
plines <- function(cmd) readLines(pipe(cmd)) | |
ptabs <- function(cmd) read.delim(pipe(cmd)) | |
# just as a trivial example, process some shuffled lines using GNU shuf, sort, | |
# etc. (we could do this in R, but this is to simulate command line operations) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
define run_fastqc | |
$$(addsuffix .ok,$(1)) : $(1) | |
mkdir -p $$(dir $$@) && \ | |
unzip -p $$< $$(addsuffix .tmp_fastqc/fastqc_data.txt,$$(notdir $$<)) |\ | |
grep ">>" | grep -v ">>END_MODULE" | cut -c 3- |\ | |
awk -F ' ' 'BEGIN{N=0;} {if(($$$$1=="Basic Statistics" || $$$$1=="Per base sequence quality" || $$$$1=="Per base N content") && $$$$2=="pass" ) N++;} END { if(N>2) {print "__PASS__"}}' |\ | |
grep __PASS__ && echo "FASTQC ok for $$<" > $$@ | |
OlderNewer