run_airbyte_images.sh

#!/usr/bin/env bash

set -euo pipefail
set -x

ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "${ROOT_DIR}"

# SOURCE_CONFIG_URL="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-integrations/connectors/source-file/integration_tests/config.json"
SOURCE_IMAGE_NAME="airbyte/source-file"
DESTINATION_IMAGE_NAME="airbyte/destination-local-json"

docker pull "${SOURCE_IMAGE_NAME}"
docker pull "${DESTINATION_IMAGE_NAME}"

# Prepare source
docker run \
  --rm \
  -v "${PWD}:${PWD}" \
  "${SOURCE_IMAGE_NAME}" \
    spec \
      | jq . > "${PWD}/source-spec.json"

# curl -s -o source-config.json "${SOURCE_CONFIG_URL}"
echo '{
  "dataset_name": "integrationTestFile",
  "format": "csv",
  "reader_options": "{\"sep\": \",\", \"nrows\": 20}",
  "url": "https://storage.googleapis.com/covid19-open-data/v2/latest/epidemiology.csv",
  "provider": {
    "storage": "HTTPS",
    "reader_impl": "gcsfs"
  }
}
' > "${PWD}/source-config.json"

docker run \
  --rm \
  -v "${PWD}:${PWD}" \
  "${SOURCE_IMAGE_NAME}" \
    check \
      --config "${PWD}/source-config.json"

# Prepare catalog
docker run \
  --rm \
  -v "${PWD}:${PWD}" \
  "${SOURCE_IMAGE_NAME}" \
    discover \
      --config "${PWD}/source-config.json" \
        > "${PWD}/catalog-raw.json"

cat "${PWD}/catalog-raw.json" | jq -s '
{
    streams: (
        (.[1].catalog.streams | map({
            stream: ., 
            sync_mode: "full_refresh", 
            destination_sync_mode: "overwrite" 
        }))     
    )
}' > "${PWD}/catalog.json"

# Prepare destination
docker run \
  --rm \
  -v "${PWD}:${PWD}" \
  "${DESTINATION_IMAGE_NAME}" \
    spec \
      | jq -R "fromjson?" \
      > "${PWD}/destination-spec.json"

echo '{"destination_path": "/files/"}' > "${PWD}/destination-config.json" 
docker run \
  --rm \
  -v "${PWD}:${PWD}" \
  "${DESTINATION_IMAGE_NAME}" \
    check \
      --config "${PWD}/destination-config.json" 

# Transfer data
docker run \
  --rm \
  -v "${PWD}:${PWD}" \
  "${SOURCE_IMAGE_NAME}" \
    read \
    --config "${PWD}/source-config.json" \
    --catalog "${PWD}/catalog.json" \
      > data.json

(
  sleep 1; 
  cat data.json
) | docker run \
  -t \
  --rm \
  -v "${PWD}:${PWD}" \
  -v "${PWD}/local/:/local/" \
  "${DESTINATION_IMAGE_NAME}" \
    write \
      --config "${PWD}/destination-config.json" \
      --catalog "${PWD}/catalog.json"