Consistency-check script for per-language exports

check_exports.sh

#!/bin/bash

if [ ! -d "$DL_DIR" ]; then
  echo "Please run this script using the runner, like this:"
  echo "./docs/cron/runner.sh $0"
  exit 1
fi

DL_DIR=${DL_DIR%/}

check_per_language_consistency() {
  local kind="$1"

  diff -u \
      <(tar xOf "$DL_DIR"/$kind.tar.bz2 $kind.csv | sort) \
      <(bzcat "$DL_DIR"/per_language/*/*_$kind.tsv.bz2 | sort)
}

for kind in sentences_detailed links sentences tags \
            sentences_in_lists sentences_with_audio \
            user_languages sentences_CC0            \
            transcriptions sentences_base; do
  echo -n "Checking consistency of $kind... "
  check_per_language_consistency $kind
  if [ $? -ne 0 ]; then
    echo "FAILED (see above)"
  else
    echo "OK"
  fi
done