iaingray

import boto3
import datetime

# Amazon Cognito User Pool Configs
LIMIT = 60
REGION = 'us-east-1'
USER_POOL_ID = 'us-east-1_aaaaaaaaa'

# Create boto3 CognitoIdentityProvider client
client = boto3.client('cognito-idp', REGION)

iaingray

-- Collection of functions to safely cast data from one type to another


-- Cast a VARCHAR to an INT and return NULL if it doesn't match accepted regexp
-- Prevents ETL dying due to data quality errors

-- Purpose: This UDF takes a VARCHAR argument,
-- and returns an INT, or NULL if regexp doesn't match accepted format for INT
-- Internal dependencies: None
-- External dependencies: None

iaingray

Byobu Commands

Level 0 Commands (Quick Start)

    <F2>                               Create a new window
    <F3>                               Go to the previous window
    <F4>                               Go to the next window
    exit                               Close the current window       

Level 1 Commands

iaingray

#tunnel pg through ssh

#pg test
require 'pg'
require 'net/ssh/gateway'

gate = Net::SSH::Gateway.new(
  'BASTION_HOST_ADDR',
  'BASTION_USER',
  {port: 22, key_data: ['PRIVATE_KEY( as string)'], keys_only: true}

iaingray

/**
 * hashpii.js
 *
 * @description Function to standardise and hash PII strings
 * @author      Various (see below)
 * @license     MIT / GPL v2
 *
 * Modifications:
 * - Various (see below)
 *

iaingray

/**
         * Extends an object with addtional namespace nodes e.g. animals.birds.ducks.mallard
         * @param base {Object} base object to be extended
         * @param nodes {array} nodes to add
         */
        extendObject : function(base, nodes){
            if(typeof base !== 'object' || !(nodes instanceof Array)){
                throw new Error ('Incorrect parameters passed to extendObject');
            }

iaingray

( LEFT(ip_address, STRPOS(ip_address, '.')-1) * 16777216) + (LEFT(SUBSTRING(ip_address, LEN(LEFT(ip_address, STRPOS(ip_address, '.')+1)), LEN(ip_address) - LEN(LEFT(ip_address, STRPOS(ip_address, '.')-1)) - LEN(LEFT(REVERSE(ip_address), STRPOS(REVERSE(ip_address), '.')-1)) - 2), STRPOS( SUBSTRING(ip_address, LEN(LEFT(ip_address, STRPOS(ip_address, '.')+1)), LEN(ip_address) - LEN(LEFT(ip_address, STRPOS(ip_address, '.')-1)) - LEN(LEFT(REVERSE(ip_address), STRPOS(REVERSE(ip_address), '.')-1)) - 2), '.')-1) * 65536) + (RIGHT( SUBSTRING(ip_address, LEN(LEFT(ip_address, STRPOS(ip_address, '.')+1)), LEN(ip_address) - LEN(LEFT(ip_address, STRPOS(ip_address, '.')-1)) - LEN(LEFT(REVERSE(ip_address), STRPOS(REVERSE(ip_address), '.')-1)) - 2), LEN(SUBSTRING(ip_address, LEN(LEFT(ip_address, STRPOS(ip_address, '.')+1)), LEN(ip_address) - LEN(LEFT(ip_address, STRPOS(ip_address, '.')-1)) - LEN(LEFT(REVERSE(ip_address), STRPOS(REVERSE(ip_address), '.')-1)) - 2)) - STRPOS(SUBSTRING(ip_address, LEN(LEFT(ip_address, STRPOS(ip_a

iaingray

#!/bin/ruby

require 'fog'
require 'inifile'

INI_FILE = ENV['HOME']+'/.aws/config'

#details from command line
location = /^s3:\/\/(?<bucket>[a-zA-Z0-9-]+)\/(?<folder>[\w-]*)\/?$/.match(ARGV[0])

iaingray

#db settings contains params for @conn
require_relative 'db_settings.rb'
require 'pg'
require 'open-uri'

conn = PG::Connection.new(:dbname => @database, :host => @host , :port => @port, :user => @user, :password => @password )
decoded_cids = {}


sql = {

iaingray

  # Converting ASCII-8BIT to UTF-8 based domain-specific guesses
  if new_value.is_a? String
    begin
      # Try it as UTF-8 directly
      cleaned = new_value.dup.force_encoding('UTF-8')
      unless cleaned.valid_encoding?
        # Some of it might be old Windows code page
        cleaned = new_value.encode( 'UTF-8', 'Windows-1252' )
      end
      new_value = cleaned