| "A distributed framework for managing and processing massive volumes | |
| of diverse and complex datasets possibly generated at different rates | |
| from different sources to solve complex problems an discover new knowledge | |
| in a scalable, efficient, and reliable manner" - Jemal Abawajy, BDCloud2014, 05/12/14 |
| #!/bin/bash | |
| ANALYSIS_LENGTH=$(shuf -i 10-1000 -n 1) | |
| echo -e "`date -u`: Analysis will take $ANALYSIS_LENGTH seconds" | |
| sleep $ANALYSIS_LENGTH | |
| echo "`date -u`: timeout completed" | |
| ANALYSIS_SUCCESS=$[$RANDOM % 2] | |
| echo -e "`date -u`: Analysis exit code: $ANALYSIS_SUCCESS" | |
| exit $ANALYSIS_SUCCESS |
| # This is a thought exercise. | |
| # Write down the output for the following two invocations | |
| # - `> me false` | |
| # - `> me true` | |
| def me(ensure_fail = false) | |
| begin | |
| puts "inside" | |
| fail "inside error" | |
| rescue => error | |
| puts "rescue" |
| ls . -Directory | % { $count = (ls $_ -Recurse).Count; $_ | Select *,@{Name="count"; Expression={$count} } } | Group-Object Count |
| $pattern = "<html>`n" | |
| ls *.wav | %{ | |
| echo ("Fixing" + $_.FullName) | |
| if ([System.Text.Encoding]::ASCII.GetString((gc $_.FullName -Encoding Byte -TotalCount ($pattern.Length * 2))).StartsWith($pattern)) { | |
| echo ("Reading" + $_.FullName) | |
| $stream = $_.OpenRead() | |
| $stream.Seek($pattern.Length, [System.IO.SeekOrigin]::Begin) | |
| $destStream = [System.IO.File]::OpenWrite($_.FullName + ".trimmed.wav") | |
| echo ("Writing" + $_.FullName) | |
| $stream.CopyTo($destStream) |
| function Set-EnvironmentVariable($key, $value) { | |
| Set-Content -Value $value -Path ("env:" + $key) | |
| New-Variable -Scope Global -Name $key -Value $value -Force | |
| } | |
| function export($envVar) { | |
| $split = $envVar.Trim().Split("="); | |
| Set-EnvironmentVariable $split[0] $split[1] | |
| } |
| /// MIT License | |
| /// Copyright (c) 2017 Anthony Truskinger | |
| /// | |
| /// Permission is hereby granted, free of charge, to any person obtaining a copy | |
| /// of this software and associated documentation files (the "Software"), to deal | |
| /// in the Software without restriction, including without limitation the rights | |
| /// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | |
| /// copies of the Software, and to permit persons to whom the Software is | |
| /// furnished to do so, subject to the following conditions: | |
| /// |
I'm testing file transfer methods (particularly to a SFTP remote).
In particular FileZilla does not detect corrupted files.
Use the above script to corrupt a remote file as a test.
If you are transferring files with rclone (https://rclone.org/)
using the --checksums argument it will detect the fault and re-transfer
the corrupt file.
| empty |
| /* | |
| * This is a template for creating very complex PostgreSQL queries. | |
| * Large queries quickly become very unreadable because variables and | |
| * other snippets of code cannot be refactored out. This template demos | |
| * using variables within an anonymous PL/pgSQL block and returning the | |
| * table result, WITHOUT requiring write permission to the database! | |
| * The use of variables are trivial within this example. | |
| */ | |
| DO $$ |