philsch

*/5 *   * * *   www    vnstati -s -i eth0 -o /var/www/traffic/1.png
*/5 *   * * *   www    vnstati -h -i eth0 -o /var/www/traffic/2.png
*/5 *   * * *   www    vnstati -m -i eth0 -o /var/www/traffic/3.png

philsch

"scripts": {
  "test_isolated": "/bin/bash -c 'if [ -z $TEAMCITY_VERSION ]; then reporter=\"spec\"; else reporter=\"mocha-teamcity-reporter\"; fi; find ./test -type f -iname \"*.spec.js\" -exec ./node_modules/.bin/mocha --compilers js:babel-core/register {} \\;'"
}

philsch

----# LOAD / UNLOAD DATA #----

-- UNLOAD using IAM session
UNLOAD ('SELECT * FROM your_table where tstamp between ''2017-02-01 12:00'' and ''2017-02-02 13:00'' ') 
TO 's3://your_bucket/dump/' 
CREDENTIALS 'aws_access_key_id=XXXXXXX;aws_secret_access_key=XXXXXX;token=XXXXXXX'
MANIFEST 
ESCAPE;

----# MAINTENANCE #----

philsch

const {ApolloError} = require('apollo-server-express');

class DatabaseError extends ApolloError {
  constructor(...args) {
    super(...args);
  }
}

module.exports = DatabaseError;

philsch

// ...
const simulateDbDown = async () => {
  if (Math.random() > 0.7) {
    throw new DatabaseError('[TEST] Oh no the database is down!');
  }
};

const resolvers = {
  Query: {
    books: async (parent, args) => {

philsch

// ...
const server = new ApolloServer({
  typeDefs,
  resolvers,
  introspection: true,
  playground: true,
  formatError: (error) => {

    /*
     * Database connection error should appear in logs

philsch

{
  "name": "avro_transform",
  "description": "Pipeline to transform Big Table exported avro files",
  "parameters": [
    {
      "name": "input",
      "label": "Input Cloud Storage File(s)",
      "help_text": "Path of the file pattern glob to read from. For example gs://bucket/test-*.avro",
      "regexes": [
        "^gs:\/\/[^\n\r]+$"

philsch

class CellTransformDoFn(beam.DoFn):
    def __init__(self):
        super(CellTransformDoFn, self).__init__()
        self.total_counter = Metrics.counter(self.__class__, 'total')

    def map_hash_to_new_prefix(self, old_hash):
        """
        Our new rowkey should have a prefix 0 - 3, as 4 is our 
        expected number of nodes in the BigTable cluster

philsch

def run():
    pipeline_options = PipelineOptions()
    pipeline = beam.Pipeline(options=pipeline_options)
    options = pipeline_options.view_as(AvroTransformOptions)

    steps = (
            pipeline
            | 'ReadData' >> beam.io.ReadFromAvro(options.input, use_fastavro=True)
            | 'Transaform rowkey' >> beam.ParDo(CellTransformDoFn())
            | 'WriteData' >> beam.io.WriteToAvro(options.output, BIG_TABLE_SCHEMA, use_fastavro=True))

philsch

{
 receiveTimestamp:  "2019-02-05T11:53:31.329098822Z"  
 resource: {…}  
 textPayload: "{"message":"Syntax Error: Expected Name, found )",
  "locations":[{"line":2,"column":22}],
  "extensions":{"code":"GRAPHQL_PARSE_FAILED"}}"  
 timestamp: "2019-02-05T11:53:20Z"
}