| import boto3 | |
| import gzip | |
| import json | |
| import time | |
| import re | |
| def export(workgroup, region, hours=1): | |
| athena = boto3.client('athena', region_name=region) | |
| current_hour_hist = [] | |
| next_token = None |
| require 'csv' | |
| # This will create join of 2 CSVs based on key-column without deduplication | |
| # The output CSV by order of csv1 | |
| # more memory-efficient is to put csv1 longest table and csv2 shortest with less key column duplication | |
| def join_csv(csv1_path, key1, csv2_path, key2, csv3_path, key_convertor: nil, ignore_cols: [], filter_csv1: nil, filter_csv2: nil) | |
| start_ts = Time.now.to_i | |
| puts "Reading #{csv2_path}, size: #{File.size(csv2_path)}..." | |
| csv2 = CSV.read(csv2_path, headers: true) |
| #!/bin/bash -e | |
| usage(){ | |
| cat <<'HELP' | |
| Usage: aws-assume-role <base-profile> <mfa-ARN> <role-ARN> <session-name> [<command>...] | |
| This script is designed to be used from alias like this (of course you can also call it from other scripts): | |
| alias assume-admin-prod='/path/to/aws-assume-role bob arn:aws:iam::1234567:mfa/bob arn:aws:iam::321321321:role/admin bob' | |
| Arguments are positional and not key-word options on purpose. You just copy-paste it into .bash_profile to create alias and replace the parameters with right values. | |
| <base-profile> is needed in case your default profile already includes role switch, in this case 'sts assume-role' won't work. |
| require 'csv' | |
| require 'aws-sdk-athena' | |
| def export_athena_usage(limit:, region:, filter_scanned: 10**9) | |
| athena = Aws::Athena::Client.new region: region | |
| next_token = nil | |
| fields = %w[query_execution_id query work_group] | |
| stats_fields = %w[data_scanned_in_bytes total_execution_time_in_millis] | |
| scanned = 0 | |
| found = 0 |
| #!/usr/bin/env ruby | |
| # frozen_string_literal: true | |
| require 'fileutils' | |
| require 'time' | |
| class AirflowLogCleaner | |
| attr_reader :list, :to_delete | |
| def list_folders |
| # frozen_string_literal: true | |
| # SimpleThreadPool gives a simple implementation for producer/consumers pattern | |
| # Example: | |
| # | |
| # pool = SimpleThreadPool.new(10, max_queue: 5) do |user_params| | |
| # Net::HTTP.post(UPDATE_API, user_params) | |
| # end | |
| # users.each do |user| | |
| # pool << user.update_api_params |
| # frozen_string_literal: true | |
| module ActionRateLimit | |
| LIMIT_BY_OPTIONS = { | |
| user: proc { request.cookies['user_id'] }, | |
| ip: proc { request.remote_ip }, | |
| }.freeze | |
| def self.included(klass) | |
| klass.extend ClassMethods |
This Python (3.6+) script is for migrating Amazon AWS System Manager (SSM) Parameter Store keys from one path to another.
To install the script do the following:
chmod +x /path/to/copy-ssm-ps-path.py)pip install boto3 (if you don't have it installed already)
| class CaseSensitiveHeadersPost < Net::HTTP::Post | |
| def initialize_http_header(headers) | |
| @header = {} | |
| headers.each do |k,v| | |
| @header[k] = [v] | |
| end | |
| end | |
| def capitalize(name) | |
| name |