Bash urlencode and urldecode
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
urlencode() { | |
# urlencode <string> | |
old_lc_collate=$LC_COLLATE | |
LC_COLLATE=C | |
local length="${#1}" | |
for (( i = 0; i < length; i++ )); do | |
local c="${1:$i:1}" | |
case $c in | |
[a-zA-Z0-9.~_-]) printf '%s' "$c" ;; | |
*) printf '%%%02X' "'$c" ;; | |
esac | |
done | |
LC_COLLATE=$old_lc_collate | |
} | |
urldecode() { | |
# urldecode <string> | |
local url_encoded="${1//+/ }" | |
printf '%b' "${url_encoded//%/\\x}" | |
} |
i had to add a check for systems where collate is not set
if [ -n "$old_lc_collate" ] ; then LC_COLLATE=$old_lc_collate ; fi
LC_ALL=C
is needed to support unicode = loop bytes, not characters.
LC_COLLATE=C
or LANG=C
do not work.
this also must be set before ${#1}
to get the length of $1
in bytes
#!/usr/bin/env bash
# MIT License
# encode special characters per RFC 3986
urlencode() {
local LC_ALL=C # support unicode = loop bytes, not characters
local c i n=${#1}
for (( i=0; i<n; i++ )); do
c="${1:i:1}"
case "$c" in
[-_.~A-Za-z0-9]) # also encode ;,/?:@&=+$!*'()# == encodeURIComponent in javascript
#[-_.~A-Za-z0-9\;,/?:@\&=+\$!*\'\(\)#]) # dont encode ;,/?:@&=+$!*'()# == encodeURI in javascript
printf '%s' "$c" ;;
*) printf '%%%02X' "'$c" ;;
esac
done
echo
}
_test_urlencode() {
local fname=urlencode
local auml=$'\xC3\xA4' # ä = %C3%A4
local euro=$'\xE2\x82\xAC' # € = %E2%82%AC
local tick=$'\x60' # ` = %60
local backtick=$'\xC2\xB4' # ´ = %C2%B4
local input="a:/b c?d=e&f#g-+-;-,-@-\$-!-*-'-(-)-#-$tick-$backtick-$auml-$euro"
# note: we expect uppercase hex codes from %02X format string
local expected="a%3A%2Fb%20c%3Fd%3De%26f%23g-%2B-%3B-%2C-%40-%24-%21-%2A-%27-%28-%29-%23-%60-%C2%B4-%C3%A4-%E2%82%AC" # also encode ;,/?:@&=+$!*'()#
#local expected="a:/b%20c?d=e&f#g-+-;-,-@-\$-!-*-'-(-)-#-%60-%C2%B4-%C3%A4-%E2%82%AC" # dont encode ;,/?:@&=+$!*'()#
local actual="$($fname "$input")"
if [[ "$actual" != "$expected" ]]; then
echo "error in $fname"
# debug
echo "input: $input"
echo "input hex:"; echo -n "$input" | hexdump -v -e '/1 "%02X"' | sed 's/\(..\)/\\x\1/g'; echo
echo "input hexdump:"; echo -n "$input" | hexdump -C
printf "actual: "; echo "$actual"
printf "expected: "; echo "$expected"
exit 1
fi
}
_test_urlencode
This works for me.
https://stackoverflow.com/questions/296536/how-to-urlencode-data-for-curl-command
rawurlencode() { local string="${1}" local strlen=${#string} local encoded="" local pos c o for (( pos=0 ; pos<strlen ; pos++ )); do c=${string:$pos:1} case "$c" in [-_.~a-zA-Z0-9] ) o="${c}" ;; * ) printf -v o '%%%02x' "'$c" esac encoded+="${o}" done echo "${encoded}" # You can either set a return variable (FASTER) REPLY="${encoded}" #+or echo the result (EASIER)... or both... :p }
@ThePredators this breaks on unicode
input: a:/b c?d=e&f#g-+-`-´-ä-€
input hex:
\x61\x3A\x2F\x62\x20\x63\x3F\x64\x3D\x65\x26\x66\x23\x67\x2D\x2B\x2D\x60\x2D\xC2\xB4\x2D\xC3\xA4\x2D\xE2\x82\xAC
input hexdump:
00000000 61 3a 2f 62 20 63 3f 64 3d 65 26 66 23 67 2d 2b |a:/b c?d=e&f#g-+|
00000010 2d 60 2d c2 b4 2d c3 a4 2d e2 82 ac |-`-..-..-...|
0000001c
actual: a%3A%2Fb%20c%3Fd%3De%26f%23g-%2B-%60-%B4-%E4-%20AC
expected: a%3A%2Fb%20c%3Fd%3De%26f%23g-%2B-%60-%C2%B4-%C3%A4-%E2%82%AC
@ThePredators works like a charm
Hi,
Characters used in France are not taken into account: (é è à ù ê â û ...) if you work in fr_FR locale.
You need to convert your data source from Windows-1252 to UTF-8 before entering in the function ::
data_utf8=$(echo "$data_ISO" | iconv -f iso8859-1 -t utf-8)
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Thanks for the script, but i don't know why when calling urlencode i got in the encoded data a : % at the end !