openrefine-client-json-in-bash.sh

#!/bin/bash
# demo script for using json in bash script

# download links
jq_url="https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64"
client_url="https://github.com/opencultureconsulting/openrefine-client/releases/download/v0.3.8/openrefine-client_0-3-8_linux"
openrefine_url="https://github.com/opencultureconsulting/OpenRefine/releases/download/2017-10-28-with-pr1294/openrefine-linux-2017-10-28-with-pr1294.tar.gz"

# install jq
if [ ! -f "jq" ]; then
    echo "Download jq..."
    wget -q -O jq $jq_url
    chmod +x jq
    echo ""
fi
jq=$(readlink -f jq)

# install openrefine-client
if [ ! -f "openrefine-client" ]; then
    echo "Download openrefine-client..."
    wget -q -O openrefine-client $client_url
    chmod +x openrefine-client
    echo ""
fi
client=$(readlink -f openrefine-client)

# install OpenRefine
if [ ! -d "openrefine" ]; then
    echo "Download OpenRefine..."
    mkdir -p openrefine
    wget -q $openrefine_url
    echo "Install OpenRefine in openrefine..."
    tar -xzf "$(basename $openrefine_url)" -C openrefine --strip 1 --totals
    rm -f "$(basename $openrefine_url)"
    sed -i '$ a JAVA_OPTIONS=-Drefine.headless=true' openrefine/refine.ini
    sed -i 's/#REFINE_AUTOSAVE_PERIOD=60/REFINE_AUTOSAVE_PERIOD=1500/' openrefine/refine.ini
    sed -i 's/-Xms$REFINE_MIN_MEMORY/-Xms$REFINE_MEMORY/' openrefine/refine
    echo ""
fi
openrefine=$(readlink -f openrefine/refine)

# OpenRefine starten
$openrefine -v warn &
pid_server=$!
until wget -q -O - http://localhost:3333 | cat | grep -q -o "OpenRefine" ; do sleep 1; done
echo ""

# create example data
echo "Beispieldaten laden..."
$client --format=csv --projectName=example --create /dev/stdin << "EOF"
a,b,c
1,11,111
2,22,222
3,33,333
EOF
echo ""

# export to stdout
echo "Export Beispieldaten..."
$client --export "example"
echo ""

# add column test
echo "Spalte test ergänzen..."
json1=$(cat << "EOF"
[ {
  "op": "core/column-addition",
  "engineConfig": {
    "mode": "row-based"
  },
  "newColumnName": "test",
  "columnInsertIndex": 2,
  "baseColumnName": "b",
  "expression": "grel:value.replace('22','FOO')",
  "onError": "set-to-blank"
} ]
EOF
)
$jq -n "$json1" | $client --apply /dev/stdin "example"
echo ""

# export to stdout
echo "Erneuter Export Beispieldaten..."
$client --export "example"
echo ""

# add column test2 by variable
echo "Spalte test2 ergänzen..."
column="test2"
json2=$(cat << EOF
[ {
  "op": "core/column-addition",
  "engineConfig": {
    "mode": "row-based"
  },
  "newColumnName": "${column}",
  "columnInsertIndex": 3,
  "baseColumnName": "b",
  "expression": "grel:value.replace('22','BAR')",
  "onError": "set-to-blank"
} ]
EOF
)
$jq -n "$json2" | $client --apply /dev/stdin "example"
echo ""

# export to stdout
echo "Erneuter Export Beispieldaten..."
$client --export "example"
echo ""

# remove columns test and test2
echo "Spalten löschen..."
columns=("test" "test2")
json=()
for column in "${columns[@]}"; do
    json+=("$(cat <<- EOF
    [ {
        "op": "core/column-removal",
        "columnName": "${column}"
    } ]
EOF
    )")
done
echo "${json[@]}" | jq -s add | $client --apply /dev/stdin "example"
echo ""

# export to stdout
echo "Erneuter Export Beispieldaten..."
$client --export "example"
echo ""

# add both columns again
echo "Nochmal beide Spalten ergänzen..."
echo "$json1" "$json2" | jq -s add | $client --apply /dev/stdin "example"
echo ""

# export to stdout
echo "Erneuter Export Beispieldaten..."
$client --export "example"
echo ""

# OpenRefine beenden
echo "OpenRefine beenden..."
kill -9 $pid_server # SIGKILL verhindert Speichern der OpenRefine-Projekte