loop_vs_parallel.txt

loop_vs_parallel.txt

# via https://www.reddit.com/r/debian/comments/9ha2dj/ive_written_a_useful_system_utility_how_do_i_get/e6abuht/

$ loop 'ls' --every 10s
$ yes | parallel -N0 -j1 --delay 10s ls 

$ loop 'touch $COUNT.txt' --count-by 5
$ yes | parallel -N0 -j1 echo touch '{= $_=seq()*5 =}'.txt

$ loop './get_response_code' --until-contains 200
# Only show the lines containing 200
$ yes | parallel -N0 -j1 --halt now,success=1 './get_response_code | grep 200'

$ loop './poke_server' --for-duration 8h
$ parallel -u --timeout 8h -N0 'yes | parallel -N0 -uj1 ./poke_server' ::: 1

$ loop './poke_server' --until-success
$ yes | parallel --halt now,success=1 -N0 -j1 ./poke_server

$ cat files_to_create.txt | loop 'touch $ITEM'
$ cat files_to_create.txt | parallel touch {}

$ loop 'ls' --for-duration 10min --summary
$ parallel -u --timeout 10m -N0 'yes | parallel -N0 -uj1 --joblog - ./poke_server' ::: 1

$ loop 'echo hello'
$ yes | parallel -N0 -j1 echo hello

$ loop 'echo $COUNT'
# GNU Parallel counts from 1
$ yes | parallel echo {#}
# Counting from 0 can be forced
$ yes | parallel echo '{= $_=seq()-1 =}'

$ loop 'echo $COUNT' --count-by 2
$ yes | parallel echo '{= $_=2*(seq()-1) =}'

$ loop 'echo $COUNT' --count-by 2 --offset 10
$ yes | parallel echo '{= $_=10+2*(seq()-1) =}'

$ loop 'echo $COUNT' --count-by 1.1
# GNU Parallel rounds 3.3000000000000003 to 3.3
$ yes | parallel echo '{= $_=1.1*(seq()-1) =}'

$ loop 'echo $COUNT $ACTUALCOUNT' --count-by 2
$ yes | parallel echo '{= $_=2*(seq()-1) =} {#}'

$ loop 'echo $COUNT' --num 3 --summary
# GNU Parallel's summary is a bit more verbose
$ seq 3 | parallel --joblog my.log echo; cat my.log

$ loop 'ls -foobarbatz' --num 3 --summary
$ seq 3 | parallel --joblog my.log -N0 ls -foobarbatz; cat my.log

$ loop 'echo $COUNT' --count-by 2 --num 50 --only-last
# Can be emulated by running 2 jobs
$ seq 49 | parallel echo '{= $_=2*(seq()-1) =}' >/dev/null 
$ echo 50| parallel echo '{= $_=2*(seq()-1) =}'

$ loop 'date' --every 5s
$ yes | parallel -N0 --delay 5s date

$ loop 'date' --for-duration 8s --every 2s
$ parallel -u -N0 --timeout 8s 'yes | parallel -N0 --delay 2s date' ::: 1

$ loop 'date -u' --until-time '2018-05-25 20:50:00' --every 5s
$ parallel -N0 -u --timeout $((`date -d 2019-05-25T20:50:00 +%s` - `date  +%s`))s 'yes | parallel -N0 --delay 5s date -u' ::: 1     

$ loop 'echo $RANDOM' --until-contains "666"
# Only show the lines containing 666
$ yes | parallel -N0 -j1 --halt now,success=1 'echo $RANDOM | grep 666'

$ loop 'if (( RANDOM % 2 )); then (echo "TRUE"; true); else (echo "FALSE"; false); fi' --until-success
$ yes | parallel --halt now,success=1 -j1  -N0 'if (( $RANDOM % 2 )); then (echo "TRUE"; true); else (echo "FALSE"; false); fi' 

$ loop 'if (( RANDOM % 2 )); then (echo "TRUE"; true); else (echo "FALSE"; false); fi' --until-error
$ yes | parallel --halt now,fail=1 -j1  -N0 'if (( $RANDOM % 2 )); then (echo "TRUE"; true); else (echo "FALSE"; false); fi' 

$ loop 'date' --until-match "(\d{4})"
# Only show the lines matching \d{4} 
$ yes | parallel -N0 -j1 --halt now,success=1 'date | grep [0-9][0-9][0-9][0-9]'

$ loop 'echo $ITEM' --for red,green,blue
$ parallel echo ::: red green blue

$ cat /tmp/my-list-of-files-to-create.txt | loop 'touch $ITEM'
$ cat /tmp/my-list-of-files-to-create.txt | parallel touch

$ ls | loop 'cp $ITEM $ITEM.bak'; ls
$ ls | parallel cp {} {}.bak; ls

$ loop 'echo $ITEM | tr a-z A-Z' -i
$ parallel 'echo {} | tr a-z A-Z'
# Or more efficiently:
$ parallel --pipe tr a-z A-Z

$ loop 'echo $ITEM' --for "`ls`"
$ parallel echo {} ::: "`ls`"

$ ls | loop './my_program $ITEM' --until-success;
$ ls | parallel --halt now,success=1 ./my_program {}

$ ls | loop './my_program $ITEM' --until-fail;
$ ls | parallel --halt now,fail=1 ./my_program {}

$ ./deploy.sh; loop 'curl -sw "%{http_code}" http://coolwebsite.biz' --every 5s --until-contains 200; ./announce_to_slack.sh
# Only prints the line containing 200
$ ./deploy.sh; yes | parallel -N0 -j1 --delay 5s --halt now,success=1 'curl -sw "%{http_code}" http://coolwebsite.biz | grep 200'; ./announce_to_slack.sh

$ loop "ping -c 1 mysite.com" --until-success; ./do_next_thing
$ yes | parallel -N0 -j1 --halt now,success=1 ping -c 1 mysite.com; ./do_next_thing

$ ./create_big_file -o my_big_file.bin; loop 'ls' --until-contains 'my_big_file.bin'; ./upload_big_file my_big_file.bin
# inotify is a better tool - it even makes sure you are not uploading a file that is not complete
$ inotifywait -qmre MOVED_TO -e CLOSE_WRITE --format %w%f . | grep my_big_file.bin

$ ls | loop 'cp $ITEM $ITEM.bak'
$ ls | parallel cp {} {}.bak

$ loop './do_thing.sh' --every 15s --until-success --num 5 
$ parallel --retries 5 --delay 15s ::: ./do_thing.sh

# A couple of functions will make the code easier to read
$ loopy() {
    yes | parallel -uN0 -j1 "$@"
  }
$ export -f loopy
$ time_out() {
    parallel -q -uN0 --timeout "$@" ::: 1
  }

$ loop 'ls' --every 10s
$ loopy --delay 10s ls

$ loop 'touch $COUNT.txt' --count-by 5
$ loopy touch '{= $_=seq()*5 =}'.txt

$ loop './get_response_code' --until-contains 200
# Only shows the lines containing 200
$ loopy --halt now,success=1 './get_response_code | grep 200'

$ loop './poke_server' --for-duration 8h
$ time_out 8h loopy ./poke_server

$ loop './poke_server' --until-success
$ loopy --halt now,success=1 ./poke_server

$ loop 'ls' --for-duration 10min --summary
$ time_out 10m  loopy --joblog - ./poke_server

$ loop 'echo hello'
$ loopy echo hello

$ loop 'echo $COUNT'
# GNU Parallel counts from 1
$ loopy echo {#}
# Counting from 0 can be forced
$ loopy echo '{= $_=seq()-1 =}'

$ loop 'echo $COUNT' --count-by 2
$ loopy echo '{= $_=2*(seq()-1) =}'

$ loop 'echo $COUNT' --count-by 2 --offset 10
$ loopy echo '{= $_=10+2*(seq()-1) =}'

$ loop 'echo $COUNT' --count-by 1.1
# GNU Parallel rounds 3.3000000000000003 to 3.3
$ loopy echo '{= $_=1.1*(seq()-1) =}'

$ loop 'echo $COUNT $ACTUALCOUNT' --count-by 2
$ loopy echo '{= $_=2*(seq()-1) =} {#}'

$ loop 'date' --for-duration 8s --every 2s
$ time_out 8s loopy --delay 2s date

$ loop 'date -u' --until-time '2018-05-25 20:50:00' --every 5s
$ seconds=$((`date -d 2019-05-25T20:50:00 +%s` - `date  +%s`))s
$ time_out $seconds loopy --delay 5s date -u

$ loop 'echo $RANDOM' --until-contains "666"
# Only show the lines containing 666
$ loopy --halt now,success=1 'echo $RANDOM | grep 666'

$ loop 'if (( RANDOM % 2 )); then (echo "TRUE"; true); else (echo "FALSE"; false); fi' --until-success
$ loopy --halt now,success=1 'if (( $RANDOM % 2 )); then (echo "TRUE"; true); else (echo "FALSE"; false); fi' 

$ loop 'if (( RANDOM % 2 )); then (echo "TRUE"; true); else (echo "FALSE"; false); fi' --until-error
$ loopy --halt now,fail=1 'if (( $RANDOM % 2 )); then (echo "TRUE"; true); else (echo "FALSE"; false); fi' 

$ loop 'date' --until-match "(\d{4})"
# Only shows the lines matching \d{4} 
$ loopy --halt now,success=1 'date | grep [0-9][0-9][0-9][0-9]'

$ ./deploy.sh; loop 'curl -sw "%{http_code}" http://coolwebsite.biz' --every 5s --until-contains 200; ./announce_to_slack.sh
# Only prints the line containing 200
$ ./deploy.sh; loopy --delay 5s --halt now,success=1 'curl -sw "%{http_code}" http://coolwebsite.biz | grep 200'; ./announce_to_slack.sh

$ loop "ping -c 1 mysite.com" --until-success; ./do_next_thing
$ loopy --halt now,success=1 ping -c 1 mysite.com; ./do_next_thing