Last active
May 25, 2020 10:01
-
-
Save felixlohmeier/b2a4382e2c4e05e8eb133008c2bf67a5 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
# demo script for using json in bash script | |
# download links | |
jq_url="https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64" | |
client_url="https://github.com/opencultureconsulting/openrefine-client/releases/download/v0.3.8/openrefine-client_0-3-8_linux" | |
openrefine_url="https://github.com/opencultureconsulting/OpenRefine/releases/download/2017-10-28-with-pr1294/openrefine-linux-2017-10-28-with-pr1294.tar.gz" | |
# install jq | |
if [ ! -f "jq" ]; then | |
echo "Download jq..." | |
wget -q -O jq $jq_url | |
chmod +x jq | |
echo "" | |
fi | |
jq=$(readlink -f jq) | |
# install openrefine-client | |
if [ ! -f "openrefine-client" ]; then | |
echo "Download openrefine-client..." | |
wget -q -O openrefine-client $client_url | |
chmod +x openrefine-client | |
echo "" | |
fi | |
client=$(readlink -f openrefine-client) | |
# install OpenRefine | |
if [ ! -d "openrefine" ]; then | |
echo "Download OpenRefine..." | |
mkdir -p openrefine | |
wget -q $openrefine_url | |
echo "Install OpenRefine in openrefine..." | |
tar -xzf "$(basename $openrefine_url)" -C openrefine --strip 1 --totals | |
rm -f "$(basename $openrefine_url)" | |
sed -i '$ a JAVA_OPTIONS=-Drefine.headless=true' openrefine/refine.ini | |
sed -i 's/#REFINE_AUTOSAVE_PERIOD=60/REFINE_AUTOSAVE_PERIOD=1500/' openrefine/refine.ini | |
sed -i 's/-Xms$REFINE_MIN_MEMORY/-Xms$REFINE_MEMORY/' openrefine/refine | |
echo "" | |
fi | |
openrefine=$(readlink -f openrefine/refine) | |
# OpenRefine starten | |
$openrefine -v warn & | |
pid_server=$! | |
until wget -q -O - http://localhost:3333 | cat | grep -q -o "OpenRefine" ; do sleep 1; done | |
echo "" | |
# create example data | |
echo "Beispieldaten laden..." | |
$client --format=csv --projectName=example --create /dev/stdin << "EOF" | |
a,b,c | |
1,11,111 | |
2,22,222 | |
3,33,333 | |
EOF | |
echo "" | |
# export to stdout | |
echo "Export Beispieldaten..." | |
$client --export "example" | |
echo "" | |
# add column test | |
echo "Spalte test ergänzen..." | |
json1=$(cat << "EOF" | |
[ { | |
"op": "core/column-addition", | |
"engineConfig": { | |
"mode": "row-based" | |
}, | |
"newColumnName": "test", | |
"columnInsertIndex": 2, | |
"baseColumnName": "b", | |
"expression": "grel:value.replace('22','FOO')", | |
"onError": "set-to-blank" | |
} ] | |
EOF | |
) | |
$jq -n "$json1" | $client --apply /dev/stdin "example" | |
echo "" | |
# export to stdout | |
echo "Erneuter Export Beispieldaten..." | |
$client --export "example" | |
echo "" | |
# add column test2 by variable | |
echo "Spalte test2 ergänzen..." | |
column="test2" | |
json2=$(cat << EOF | |
[ { | |
"op": "core/column-addition", | |
"engineConfig": { | |
"mode": "row-based" | |
}, | |
"newColumnName": "${column}", | |
"columnInsertIndex": 3, | |
"baseColumnName": "b", | |
"expression": "grel:value.replace('22','BAR')", | |
"onError": "set-to-blank" | |
} ] | |
EOF | |
) | |
$jq -n "$json2" | $client --apply /dev/stdin "example" | |
echo "" | |
# export to stdout | |
echo "Erneuter Export Beispieldaten..." | |
$client --export "example" | |
echo "" | |
# remove columns test and test2 | |
echo "Spalten löschen..." | |
columns=("test" "test2") | |
json=() | |
for column in "${columns[@]}"; do | |
json+=("$(cat <<- EOF | |
[ { | |
"op": "core/column-removal", | |
"columnName": "${column}" | |
} ] | |
EOF | |
)") | |
done | |
echo "${json[@]}" | jq -s add | $client --apply /dev/stdin "example" | |
echo "" | |
# export to stdout | |
echo "Erneuter Export Beispieldaten..." | |
$client --export "example" | |
echo "" | |
# add both columns again | |
echo "Nochmal beide Spalten ergänzen..." | |
echo "$json1" "$json2" | jq -s add | $client --apply /dev/stdin "example" | |
echo "" | |
# export to stdout | |
echo "Erneuter Export Beispieldaten..." | |
$client --export "example" | |
echo "" | |
# OpenRefine beenden | |
echo "OpenRefine beenden..." | |
kill -9 $pid_server # SIGKILL verhindert Speichern der OpenRefine-Projekte |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment