| #!/usr/bin/ruby | |
| require 'rubygems' | |
| require 'open-uri' | |
| require 'nokogiri' | |
| require 'net/http' | |
| require 'uri' | |
| require 'amazon/aws/search' | |
| include Amazon::AWS |
| require "rubygems" | |
| require "twitter" | |
| # get these from apps.twitter.com | |
| CONSUMER_KEY = "foo" | |
| CONSUMER_SECRET = "bar" | |
| OAUTH_TOKEN = "blee" | |
| OAUTH_TOKEN_SECRET = "baz" | |
| TWITTER_USER = "your_username" # needs to be the one associated with keys above |
| -- FS_USAGE provides logging of all file system events however the output format is awkward to use. | |
| -- Instead we can use `duckdb` to quickly and simply summarize the log. https://duckdb.org/docs/installation/ | |
| -- For a description of `fs_usage` output see https://ss64.com/osx/fs_usage.html and the man page | |
| -- See also: https://www.macobserver.com/tmo/article/os_x_using_fs_usage_as_a_troubleshooting_tool | |
| -- CREATE SINGLE COLUMN TABLE | |
| DROP TABLE IF EXISTS fs_usage_raw; | |
| CREATE TABLE fs_usage_raw AS | |
| SELECT * |
| # FS_USAGE provides logging of all file system events however the output format is awkward to use. | |
| # This script attempts to use `sed` to convert the output into valid JSON data. | |
| # Results may vary. Recommend using `jq -r` to validate the JSON output. | |
| # For a description of `fs_usage` output see https://ss64.com/osx/fs_usage.html and the man page | |
| gsed -E 's/([0-9\:]{8}\.[0-9]{6}) +([a-zA-Z0-9]+\[*[a-zA-Z0-9_]*\]*)/{"timestamp":"\1", "call":"\2"/g' \ | |
| | gsed -E 's/ +([[:alpha:]])\=([0-9a-fA-FxX]+)/, "\1":"\2"/g;' \ | |
| | gsed -E 's/\[ *([[:digit:]]+)\]/, "errno":"\1"/g;' \ | |
| | gsed -E 's/ +(0\.[0-9]{6}) (W)* +([[:alpha:]].*)\.([[:digit:]]+)$/, "elapsed":\1, "wait":"\2", "process":"\3", "pid":"\4"}/g;' \ | |
| | gsed -E 's/ +(\/*[a-zA-Z0-9]+\/.*) +/, "pathname":"\1"/g;' \ |
| # Create empty file with header | |
| echo "status, qry_num, run_num, start_time, user_name, rows, total_ms, compile_ms, exec_ms, cached, query_text" > ~/Documents/dbr_results.csv | |
| # Get 10k last query results and convert to CSV | |
| curl --netrc \ | |
| -X GET \ | |
| -H 'Content-Type: application/json' \ | |
| -d '{"filter_by":{"endpoint_id":"0a7f2417e20e742c","status":"FINISHED", | |
| "query_start_time_range": {"start_time_ms": 1656126000000,"end_time_ms": 1656130400000}}, | |
| "include_metrics": true, "max_results": 10000}' \ | |
| https://workspace-name.cloud.databricks.com/api/2.0/sql/history/queries \ |
| *.log | |
| .DS_Store | |
| .databricks/cache | |
| .dbfseventsd | |
| .DocumentRevisions-V100 | |
| .dropbox | |
| .dropbox.cache | |
| .fseventsd | |
| .gradle | |
| .hotfiles.btree |
| sudo yum -y install tmux; | |
| echo -e "set -g mode-mouse on\nset -g mouse-resize-pane on\nset -g mouse-select-pane on\nset -g mouse-select-window on" > .tmux.conf | |
| tmux |
| /* ----------------------------------------- | |
| SNOWFLAKE TPC-DS PARQUET LOAD | |
| Load parquet using SF's insane syntax | |
| ----------------------------------------- */ | |
| -- SHOW DATABASES; | |
| CREATE DATABASE tpcds_1g; | |
| USE tpcds_1g; | |
| -- SHOW SCHEMAS; |
| /* ------------------------------------------------------ | |
| Delta Lake `DESCRIBE HISTORY` outputs JSON objects | |
| This query summarizes the JSON in consistent columns | |
| ------------------------------------------------------ */ | |
| SELECT | |
| `version`, | |
| `timestamp`, | |
| operation, | |
| operationMetrics.executionTimeMs AS exec_ms, |
The number of nodes in each Snowflake warehouse is obfuscated but not really a secret
https://stackoverflow.com/questions/58973007/what-are-the-specifications-of-a-snowflake-server
one part states the server EC2 instance type as "warehouseServerType" : "c5d.2xlarge",
https://select.dev/posts/snowflake-warehouse-sizing
it’s fairly well known that on AWS each node (except for 5XL and 6XL warehouses) is a c5d.2xlarge EC2 instance, with 16GB of RAM and a 200GB SSD.