JavaScript snippets

bash_snippets.md

Bash useful and neat snippets

Ensure a process exist

# incomplete
ps -o pid -C ssh-agent|sed -n '2p'

Append to a file

echo 'zend_extension=xdebug.so' | tee -a /etc/php7/conf.d/xdebug.ini

or take commented content from a file, and paste it uncommented

cat /etc/php7/conf.d/xdebug.ini | sed 's/^;zend_extension/zend_extension/g' >> /etc/php7/conf.d/Z99_dev.ini

Ensure a command exist

command -v foobar >/dev/null 2>&1 || { echo >&2 "Required binary foobar not found. This script will not work. Aborting."; exit 1; }

Extract value from JSON API output

ETCD_VERSION=$(curl -s -L http://127.0.0.1:2379/version | python -c "import sys, json; print(json.load(sys.stdin)['etcdserver'])")

Ensure a script runs only as root

if test "$(id -g)" -ne "0"; then
  (>&2 echo "You must run this as root."; exit 1)
fi

Send to Standard Error Output

function stderr { printf "$@\n" >&2; }

Validate if number

function is_number(){
  is_number='^[0-9]+$'
  if [[ ! $1 =~ $is_number ]]; then
    return 1
  else
    return 0
  fi
}

Do the same complex command with incremental numbers

Imagine you want to get the IPv4 address of a number of machines. You know the names will be in sequence (e.g. 0..n) and you want to create a zone format.

seq 0 4 | awk '{printf "+short @192.168.0.3 node%d.local\n", $1}' | xargs -L1 dig 

Loop commands from array

declare -a IMAGES=(\
                  ubuntu:14.04 \
                  ubuntu:16.04 \
                   )

for image in "${IMAGES[@]}"; do
    docker pull $image
done

Create a file

With contents that contains variables

(cat <<- _EOF_
iface eth0:1 inet static
  address ${IPV4_INTERNAL}
_EOF_
) >> /etc/network/interfaces.d/eth0

As an iteration

use-case; you want to format an hosts file (or DNS zone file) based on a list of nodes where we assign a name and internal IP incrementally.

Desired output would be like this.

##########
192.168.0.10	node0	# cluster_member
127.0.1.1	node1	# cluster_member  self
192.168.0.12	node2	# cluster_member
192.168.0.13	node3	# cluster_member
192.168.0.14	node4	# cluster_member
192.168.0.15	node5	# cluster_member
192.168.0.16	node6	# cluster_member
192.168.0.17	node7	# cluster_member
192.168.0.18	node8	# cluster_member
192.168.0.19	node9	# cluster_member
192.168.0.20	node10	# cluster_member
##########

Here it is

# e.g. this hosts file would be for node1
NODE_NUMBER=1
# e.g. our cluster would have node names starting by this
NODE_NAME_PREFIX=node
# e.g. our cluster would have a maximum of 10 nodes... node0 to node9
NODE_COUNT_MAX=10
# e.g. this hosts file would list IP addresses in sequences starting by this
IPV4_INTERNAL_PREFIX="192.168.0."

LIST=""
for i in `seq 0 $NODE_COUNT_MAX`; do
  if [[ ! "${NODE_NUMBER}" = "${i}" ]]; then
    NODE_POS=$(printf %d $((${i} + 10)))
    IP="${IPV4_INTERNAL_PREFIX}${NODE_POS}"
    APPEND_CLUSTER_MEMBER=""
  else
    IP="127.0.1.1"
    APPEND_CLUSTER_MEMBER=" self"
  fi
  LIST+="${IP}\t${NODE_NAME_PREFIX}${i}\t# cluster_member ${APPEND_CLUSTER_MEMBER}\n"
done

# Append conditionnally to /etc/hosts only if last node is not found in the file
grep -q -e "${NODE_NAME_PREFIX}${i}" /etc/hosts || printf $"\n##########\n${LIST}##########\n" >> /etc/hosts

Rename many files

Add increment number, replace part of name

Input is a list of files with pattern similar to this

02.09.09_Something photo.jpg
02.09.10_Something else - a note.jpg
02.09.11_Another one.jpg

... (a few hundreds like this)

And we want out

001 Something photo.jpg
002 Something else - a note.jpg
003 Another one.jpg
...

Could be done like this

ls | cat -n | while read n f; do i=`printf '%03d' "$n"`; fn=`echo $f|sed -E 's/^[0-9\.]+_//g'`; mv "$f" "$i $fn"; done

Awk

Filter file with path names, truncate last parts

1. Have file with similar contents

File: example.txt

/foo/bar/bazz/buzz/bizz.jpg
/foo/bar1/bazz1/buzz1/bizz1.jpg
/foo/bar2/bazz2/buzz2/bizz2.jpg
/foo/bar3/bazz3/buzz3/bizz3.jpg

2. What we want out

/foo/bar/bazz
/foo/bar1/bazz1
/foo/bar2/bazz2
/foo/bar3/bazz3

3. Create path.awk with following contents

NOTICE the truncate=2, so we want to take the two last path parts off.

File: path.awk

BEGIN { truncate=2; print "Path Muncher\n" }
function join(array, start, end, result, i) {
  result = array[start]
  for (i = start + 1; i <= end; i++)
      result = result"/"array[i]
  return result
}
{
    split($0, arr, "/");
    end = length(arr) - truncate;
    print join(arr, 0, end, "/")
}
END { print "\n" }

4. Use

cat example.txt | awk -f path.awk
Path Muncher

/foo/bar/bazz
/foo/bar1/bazz1
/foo/bar2/bazz2
/foo/bar3/bazz3

More

• Getting data about files

get_artifacts.sh

#!/usr/bin/env bash

#set -xue
set -ue


################################################################################
# MUST HAVE Arguments and their name. Everything stops if one here is missing.

argEndpointBaseUrl="${1:?Missing first argument}"


################################################################################
# Functions. That could be distributed and re-used.

must_command_exist()
{
    command -v "${1}" >/dev/null 2>&1 || { echo >&2 "Command ${1} must exist."; exit 1; }
}

# https://stackoverflow.com/questions/11362250/in-bash-how-do-i-test-if-a-variable-is-defined-in-u-mode#answer-19874099
is_var_defined()
{
    if [ $# -ne 1 ]
    then
        echo "Expected exactly one argument: variable name as string, e.g., 'my_var'"
        exit 1
    fi
    # Tricky.  Since Bash option 'set -u' may be enabled, we cannot directly test if a variable
    # is defined with this construct: [ ! -z "$var" ].  Instead, we must use default value
    # substitution with this construct: [ ! -z "${var:-}" ].  Normally, a default value follows the
    # operator ':-', but here we leave it blank for empty (null) string.  Finally, we need to
    # substitute the text from $1 as 'var'.  This is not allowed directly in Bash with this
    # construct: [ ! -z "${$1:-}" ].  We need to use indirection with eval operator.
    # Example: $1="var"
    # Expansion for eval operator: "[ ! -z \${$1:-} ]" -> "[ ! -z \${var:-} ]"
    # Code  execute: [ ! -z ${var:-} ]
    eval "[ ! -z \${$1:-} ]"
    return $?  # Pedantic.
}

must_var_defined()
{
    is_var_defined "$1" || { echo >&2 "Variable ${1} is not defined and must exist."; exit 1; }
}


################################################################################
# Sanity checks before executing anything else.

must_command_exist jq
must_command_exist curl

must_var_defined PRIVATE_TOKEN


################################################################################
# Parametrized HTTP API URLs.

TOKEN=$PRIVATE_TOKEN


################################################################################
# https://gitlab.msu.edu/help/user/project/pipelines/job_artifacts.md
# https://docs.gitlab.com/ee/api/jobs.html

## e.g. curl -sSL -v 'https://gitlab.example.org:5443/Foo/bar/-/jobs/artifacts/branchName/raw/dist/dist.zip?job=Release' -H "PRIVATE-TOKEN: $PRIVATE_TOKEN"
#BRANCHNAME=branchName
#DIRNAME=dist
#FILE=dist.zip
#JOBNAME=Release
#endpoint="/-/jobs/artifacts/${BRANCHNAME}/raw/${DIRNAME}/${FILE}?job=${JOBNAME}" 

## IN PROGRESS

# https://docs.gitlab.com/ee/api/jobs.html#list-project-jobs
## e.g. curl -sSL -v --H "PRIVATE-TOKEN: 111" 'https://gitlab.example.com/api/v4/projects/1/jobs?scope[]=pending&scope[]=running'
endpoint="/api/v4/projects/373/jobs?scope[]=success"

# https://gitlab.example.org:5443/Foo/bar/-/jobs/artifacts/branchName/browse?job=Release
# https://gitlab.example.org:5443/Foo/bar/-/jobs/11377/artifacts/browse

## e.g. /builds/artifacts/master/browse?job=coverage
#endpoint="/builds/artifacts/${BRANCHNAME}/browse?job=${JOBNAME}" 




URL=${argEndpointBaseUrl}
URL+=${endpoint}
echo $URL

curl -gsSL -v "${URL}" -H "PRIVATE-TOKEN: $TOKEN"

http.sh

#!/usr/bin/env bash


set -e


function error { printf "Error: %s \n " "$@" >&2; exit 1; }


command -v curl >/dev/null 2>&1 || { error "cURL is not installed on this system and this script requires it."; }


display_usage() {
    echo "HTTP helper to maintain data on WebAPI JSON/MongoDB storage."
    echo -e "\nUsage:\n  ${0} <VERB> <URL> [<path/to/file.json>]\n"
    echo -e "<VERB>, an HTTP verb:"
    echo -e "- get"
    echo -e "- post"
    echo -e "- delete"
    echo -e "\n"
    echo -e "<URL>, an HTTP server address (e.g. \"http://example.org/foo/bar\")"
    echo -e "\n"
    echo -e "Examples:"
    echo -e "  ${0} get http://example.org/foo/bar"
    echo -e "  ${0} get 'http://example.org/foo/bar?buzz'"
    echo -e "  ${0} delete http://example.org/foo/bar"
    echo -e "  ${0} post http://example.org/foo/bar path/to/file.json"
    echo -e "\n"
}




if [[ "${1}" == "" ]]; then
    display_usage
fi



possibleactions=(delete get post)
if [[ "${1}" == "" ]]; then
    display_usage
    error "There MUST be an action argument, one of [${possibleactions[*]}]."
fi
if [[ ! "${possibleactions[@]}" =~ ${1} ]]; then
    display_usage
    error "Invalid action \"${1}\". It must be one of [${possibleactions[*]}]."
fi
ACTION="${1^^}" # Make the action name be UPPERCase, like HTTP Verbs typically are.


endpointUrlRegEx='https?://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]'
if [[ ! ${2} =~ ${endpointUrlRegEx} ]]; then
    display_usage
    error "Invalid URL \"${2}\"."
fi
ENDPOINT="${2}"


if [[ $ACTION == "POST" ]]; then
    if [[ "${3}" == "" ]]; then
        display_usage
        error "No File argument supplied."
    fi
    if [ ! -f "${3}" ]; then
        echo "File \"${3}\" not found!"
    fi
    SOURCE_FILE="${3}"
    curl -v -sSL -X"${ACTION}" -H "Content-Type: application/json" -H "Authorization: ${Authorization:-SET_AUTHORIZATION_SHELL_ENVIRONMENT}" "${ENDPOINT}" --data "@${SOURCE_FILE}"
else
    curl -v -sSL -X"${ACTION}" -H "Content-Type: application/json" -H "Authorization: ${Authorization:-SET_AUTHORIZATION_SHELL_ENVIRONMENT}" "${ENDPOINT}"
fi

mixing_bash_python.sh

#!/bin/bash

## UNFINISHED shell script
## Illustrate how to mix Python and Bash
## Hat tip to http://bhfsteve.blogspot.ca/2014/07/embedding-python-in-bash-scripts.html

ENDPOINT=https://api.enlightns.com
NAME=example.enlightns.info

set -e

command -v http >/dev/null 2>&1 || { echo >&2 "HTTPie Python CLI is required for this script to work. Aborting."; exit 1; }
command -v python >/dev/null 2>&1 || { echo >&2 "Python is required for this script to work. Aborting."; exit 1; }

if [[ -z ${EMAIL} ]]; then
  (>&2 echo "Missing EMAIL shell environment variable"; exit 1)
fi



function extract_data_from_json() {
    PYTHON_WHAT=$1 PYTHON_INPUT=$2 python - <<END
import sys, json, os;
selector = os.environ['PYTHON_WHAT']
json_data = json.loads(os.environ['PYTHON_INPUT'])
print(json_data[selector])
END
}



if [[ -z ${ENLIGHTNS_AUTHORIZATION_TOKEN} ]]; then

  if [[ -z ${ENLIGHTNS_PASSWORD} ]]; then
    read -p "EnlightNS account password for ${EMAIL}? " ENLIGHTNS_PASSWORD
  fi

  AUTH_REQUEST_CREDS="{\"email\": \"${EMAIL}\", \"password\": \"${ENLIGHTNS_PASSWORD}\"}"

## See enlightns docs
## http://enlightns-docs.readthedocs.io/en/latest/
##
## Get Authentication token
##
## echo '{"email": "self@example.org", "password": "foobar"}' |  http --verbose POST https://api.enlightns.com/api-token-auth/ Content-Type:application/json
##
## Expected outcome
##
## {"token": "JWT 0000.REDACTED.REDACTED-AGAIN"}
##

AUTH_REQUEST_TOKEN=$(echo $AUTH_REQUEST_CREDS|http -b POST ${ENDPOINT}/api-token-auth/ Content-Type:application/json)
TOKEN=$(extract_data_from_json "token" "${AUTH_REQUEST_TOKEN}")

(cat <<- STATUS
---------------------------------------------------------------
Authentication to ${ENDPOINT} successful
subsequent requests will be made with the following HTTP header

    Authorization: ${TOKEN}

To prefent re-authenticating, use ENLIGHTNS_AUTHORIZATION_TOKEN
shell environment variable.
---------------------------------------------------------------
STATUS
)
ENLIGHTNS_AUTHORIZATION_TOKEN=$TOKEN
fi

## UNFINISHED here.
http --verbose GET ${ENDPOINT}/user/record/ Content-Type:application/json "Authorization: ${ENLIGHTNS_AUTHORIZATION_TOKEN}"

##
## Output would look like
##
#HTTP/1.1 200 OK
#Allow: GET, HEAD, OPTIONS
#Connection: keep-alive
#Content-Language: en
#Content-Type: application/json
#Date: Thu, 27 Apr 2017 17:36:21 GMT
#Server: nginx
#Transfer-Encoding: chunked
#Vary: Accept, Accept-Language, Cookie
#X-Frame-Options: DENY
#
#[
#    {
#        "active": true,
#        "auth": null,
#        "content": "0.0.0.0",
#        "id": 11,
#        "is_free": false,
#        "name": "example.enlightns.info",
#        "owner": "self@example.org",
#        "ttl": 300,
#        "type": "A"
#    }
#]

snippets.js

// Get All Style tags in one string
var stylesheetsElementsString = ''; [...document.head.childNodes].filter(a => a.rel == 'stylesheet').forEach(a => stylesheetsElementsString += a.outerHTML);

// Remove all Stylesheet tags from head
[...document.head.childNodes].forEach(e => {if(e.rel === "stylesheet"){e.remove();}});

// Get All Script src in one string
var scriptTags = [...document.head.childNodes].filter(a => a.localName === 'script' && !!a.src).map(a => a.src).join("\n");


/**
 * toCamelCase('foo-bar') === 'fooBar'
 */
function toCamelCase(str) {
    var regex = /-([a-z])/g;
    if (str.match(regex).length < 1) {
        throw new Error('Invalid input "'+str+'", is not in kebab-case-format, as per '+regex.toString());
    }
    return str.replace(regex, function (g) { return g[1].toUpperCase(); });
}


/**
 * toKebabCase('fooBar') === 'foo-bar'
 */
function toKebabCase(str) {
    var regex = /[A-Z\\d]/g;
    if (str.match(regex).length < 1) {
        throw new Error('Invalid input "'+str+'", is not in camelCaseSeparator, as per '+regex.toString());
    }
    return str.replace(regex, function (g) { return '-'+g[0].toLowerCase(); });
}

var tests = {cases:[], subjects: {toCamelCase: toCamelCase, toKebabCase: toKebabCase}};
tests.cases.push(['toCamelCase', 'foo-bar-baz', 'fooBarBaz']);
tests.cases.push(['toCamelCase', 'bar-baz', 'barBaz']);
tests.cases.push(['toKebabCase', 'fooBarBaz', 'foo-bar-baz']);
tests.cases.push(['toKebabCase', 'bazBarBuzzBizzz', 'baz-bar-buzz-bizzz']);
tests.cases.forEach(function poorManTests(cur) {
    var methodName = cur[0],
        testClosure = tests.subjects[methodName] || null,
        input = cur[1],
        expected = cur[2];
    if (testClosure !== null) {
        var testRun = testClosure.call(null, input);
        if (testRun !== expected) {
            console.log('FAIL: Function call '+methodName+'("'+input+'"), we expected "'+expected+'", we got "'+testRun+'".');
        }
    }
});


/**
 * We want to make sure someInput object has at least the following keys
 */
var SomeConstructorClass = function SomeConstructorClassConstructor(someInput){
    var requiredArguments = ['foo', 'bar', 'baz'];
    for (var property in someInput) {
        if (someInput.hasOwnProperty(property)) {
            // Do other things
            requiredArguments.splice(requiredArguments.indexOf(property), 1);
        }
    }
    if (requiredArguments.length > 0) {
        throw new Error('SomeConstructorClass error, we have missing arguments: ' + requiredArguments.join(', '));
    }
};

Vim

Delete all lines that does not contain us-ascii At the end of a line

:v/us-ascii$/d

Remove everything, on each line, that's after the first :

%s/:.*$//g

• https://github.com/progrium/bashstyle
• https://carlosbecker.com/posts/lint-shell-scripts/

Awesome bash libraries:

• https://github.com/mietek/bashmenot/tree/master/src
• https://github.com/martinburger/bash-common-helpers/blob/master/bash-common-helpers.sh

See also https://stackoverflow.com/questions/11369522/bash-utility-script-library and https://dberkholz.com/2011/04/07/bash-shell-scripting-libraries/

Increment version number using awk. Thanks https://stackoverflow.com/questions/8653126/how-to-increment-version-number-in-a-shell-script

#!/usr/bin/env awk
# Incomplete
function inc(s,    a, len1, len2, len3, head, tail)
{
split(s, a, ".")

len1 = length(a)
if(len1==0)
    return -1
else if(len1==1)
    return s+1

len2 = length(a[len1])
len3 = length(a[len1]+1)

head = join(a, 1, len1-1)
tail = sprintf("%0*d", len2, (a[len1]+1)%(10^len2))

if(len2==len3)
    return head "." tail
else
    return inc(head) "." tail
}
function join(a, x, y,    s)
{
for(i=x; i<y; i++)
    s = s a[i] "."
return s a[y]
}

Expected outcome

1.2.3.99 => 1.2.4.00

git -c http.extraheader='PERSONAL-TOKEN:111' clone https://gitlab.example.org:50043/Foo/bar.git
git -c http.extraheader='PERSONAL-TOKEN:111' clone https://username@gitlab.example.org:50043/Foo/bar.git
export GIT_TRACE=1
git -c credential.useHttpPath=1 -c http.extraheader='PERSONAL-TOKEN:111' clone https://username@gitlab.example.org:50043/Foo/bar.git
git help -a | grep credential
git help credential-store

20160101 bashrc

Yet another .bashrc

alias dc="docker-compose"
alias di="docker images"
alias dp="docker ps"
alias drmi="docker images -q |xargs docker rmi"
alias drmv="docker volume ls | grep -v data | awk '{print \"docker volume rm \"$2}'|sh"

alias ll="ls -al"
alias projects="cd /data/projects"

alias reup='docker-compose stop && docker-compose rm -f && docker-compose up -d && docker-compose ps'

lazy_memusage () {
  ps -ylC "$@" --sort:rss | awk '!/RSS/ { s+=$8 } END { printf "%s\n", "Total memory used:"; printf "%dM\n", s/1024 }'
}

# ref: http://www.commandlinefu.com/commands/view/3543/show-apps-that-use-internet-connection-at-the-moment.
lazy_lsof_network_services() {
  lsof -P -i -n | cut -f 1 -d " "| uniq | tail -n +2
}

lazy_openssl_read() {
  openssl x509 -text -in $1 | more
}

lazy_lsof_socks(){
  lsof -P -i -n | less
}

lazy_lsof() {
  /usr/bin/lsof -w -l | less
}

lazy_trailing_whitespace() {
  if [ -f "$1" ]; then
    sed -i 's/[ \t]*$//' "$1"
  fi
}

#List all commiters
#git shortlog -sn

# ref: http://stackoverflow.com/questions/26370185/how-do-criss-cross-merges-arise-in-git
lazy_git_log() {
  git log --graph --oneline --decorate --all
}

lazy_git_search_file() {
  git log --all --name-only --pretty=format: | sort -u | grep "$1"
}

# ref: http://hardenubuntu.com/disable-services
alias lazy_processes="initctl list | grep running"

alias lazy_connections="netstat -ntu | awk '{print $5}' | cut -d: -f1 | sort | uniq -c | sort -n"

lazy_html_compact () {
  if [ -f "$1" ]; then
    cat $1 | tr '\t' ' ' | tr '\n' ' ' | sed 's/  //g' | sed 's/> </></g' > $1
  fi
}

lazy_serve_http () {
  python -m SimpleHTTPServer 8080
}

lazy_convert_yml_json () {
  if [ -f "$1" ]; then
    python -c 'import sys, yaml, json; json.dump(yaml.load(sys.stdin), sys.stdout, indent=2)' < $1 > $1.json
  fi
}

https://canihazip.com/s