Script to remove packages from CKAN API

bulk-remove.sh

#!/bin/bash
# bulk-remove.sh < ids-to-delete.txt > package-backup.txt

set -o errexit
set -o pipefail
set -o nounset

function fetch_and_remove () {
  local identifier="$1"

  # tempfile for package data
  package_file=$(mktemp)

  # get the package
  curl --silent --fail --get --data-urlencode "fq=identifier:$identifier" 'https://admin-catalog.data.gov/api/action/package_search' > "$package_file"

  rows=$(jq --raw-output '.result.count' "$package_file")

  if [[ "$rows" -ne 1 ]]; then
    echo "$identifier found $rows > 1" >&2
    return 1
  fi

  id=$(jq --raw-output '.result.results[0].id' "$package_file")

  if [[ -z "$id" ]]; then
    echo "Invalid Id for $identifier" >&2
    cat "$package_file" >&2
    return 1
  fi

  # dump the package to stdout
  jq --compact-output '.result.results[0]' "$package_file"

  # delete the package
  if ! curl --silent --fail -X POST -H "Authorization: $CKAN_API_KEY" 'https://admin-catalog.data.gov/api/action/package_delete' --data "{\"id\":\"$id\"}" > /dev/null; then
    # make sure to record the identifier if there was an error
    echo "$identifier failed to delete." >&2

    # back-off to give the server some breathing room
    sleep 60
  fi

  # cleanup the tempfile
  rm "$package_file"
}

while read identifier; do
  fetch_and_remove "$identifier" || true
done