Google Maps Platform - Datasets API CLI

gmp_datasets.sh

#!/bin/bash 

## Licensed under the Apache License, Version 2.0 (the "License");
## you may not use this file except in compliance with the License.
## You may obtain a copy of the License at
## 
##    https://www.apache.org/licenses/LICENSE-2.0
##
## Unless required by applicable law or agreed to in writing, software
## distributed under the License is distributed on an "AS IS" BASIS,
## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
## See the License for the specific language governing permissions and
## limitations under the License.


## note : 
# This Bash script enables CLI access to Google Maps Platform Datasets API.

# OAuth is used instead of API Keys so that you do not have to worry about key management.
# This requires the user of the script to have: 
# 1. Cloud Project id/number of a project enabled with Datasets API 
# 2. The right IAM role (roles/mapsplatformdatasets.admin) on the project.
#   ref : https://cloud.google.com/iam/docs/understanding-roles#mapsplatformdatasets.admin

# Datasets can be created from a file uploaded to the API or a one already stored on GCS.
# This CLI covers both. 
# "name" and "displayName" can be confusing. Make sure to understand the difference. 

# functions and some variables are exported. 
# this is so that the functions can be called in child processes with commands such as xarg

## prerequisite tools/comands for this script
# * gcloud cli : https://cloud.google.com/sdk/docs/install
# * curl : https://everything.curl.dev/get
# * jq : https://stedolan.github.io/jq/download/  

## API Reference
# this sample cli was written based on the API spec here. 
# please check official docs for detailed definitions.
# https://developers.google.com/maps/documentation/datasets

## self link : 
# * https://gist.github.com/n0531m/ae7bb4bcf5f6bb804dbcf89da1efd4f1 (gist page)
# * https://gist.githubusercontent.com/n0531m/ae7bb4bcf5f6bb804dbcf89da1efd4f1/raw (raw)


## Log
# some of the request/response is logged for ease of debugging

DIR_DATASETSAPI_LOGS=~/logs/gmp_datasets
if [ ! -d $DIR_DATASETSAPI_LOGS ]; then
    mkdir -p $DIR_DATASETSAPI_LOGS
    echo "Log directory $DIR_DATASETSAPI_LOGS created"
fi
export DIR_DATASETSAPI_LOGS


# ref : https://developers.google.com/maps/documentation/datasets/reference/rest/v1alpha/projects.datasets/list
function datasets_list {
    if [ ! "$#" -eq 1 ]; then
        echo "usage : ${FUNCNAME[0]} <PROJECT(id or number)>"
        return 1
    else
        
        #Project ID or Project number
        local PROJECT=$1
        #GET https://mapsplatformdatasets.googleapis.com/v1alpha/{parent=projects/*}/datasets
        
        local PAGESIZE=100

        local FILE_RESPONSE="$DIR_DATASETSAPI_LOGS/${FUNCNAME[0]}_response.json"
        
        local parent=projects/$PROJECT
        
        URL=https://mapsplatformdatasets.googleapis.com/v1alpha/$parent/datasets
        curl -s -G $URL \
        -H "Authorization: Bearer $(gcloud auth application-default print-access-token)" \
        -H "X-Goog-User-Project: $PROJECT" \
        -H "accept: application/json" \
        --data "pageSize=$PAGESIZE" > $FILE_RESPONSE
        
        local TOKEN=$(cat $FILE_RESPONSE | jq -r 'select(.nextPageToken != null) | .nextPageToken')
        #echo $TOKEN
        cat $FILE_RESPONSE | jq -c '.datasets[]'
        
        ## if there is a page token returned, do the same recursively
        while [[ $TOKEN != "" ]] && [[ $TOKEN != "null" ]] ; do
            FILE_RESPONSE="$DIR_DATASETSAPI_LOGS/${FUNCNAME[0]}_response_$TOKEN.json"
            
            URL=https://mapsplatformdatasets.googleapis.com/v1alpha/$parent/datasets
            curl -s -G $URL \
            -H "Authorization: Bearer $(gcloud auth application-default print-access-token)" \
            -H "X-Goog-User-Project: $PROJECT" \
            -H "accept: application/json" \
            --data "pageToken=$TOKEN" \
            --data "pageSize=$PAGESIZE" > $FILE_RESPONSE
            TOKEN=$(cat $FILE_RESPONSE | jq -r 'select(.nextPageToken != null) | .nextPageToken' )
            #echo $TOKEN
            cat $FILE_RESPONSE | jq -c '.datasets[]'
        done
    fi
}
export -f datasets_list



function datasets_check_if_exists_by_displayname {
    if [ ! "$#" -eq 2 ]; then
        echo "usage : ${FUNCNAME[0]} <PROJECT(id or number)> <DISPLAYNAME>"
        return 1
    else
        local PROJECT=$1
        local DISPLAYNAME=$2
        
        #datasets_list $PROJECT | jq --arg n "$DISPLAYNAME" -c 'select(.displayName==$n)'
        if [ "$(datasets_list $PROJECT | jq --arg n "$DISPLAYNAME" -c 'select(.displayName==$n)' | wc -l)" -eq 1 ]; then
            echo true
        else
            echo false
        fi
    fi
}
export -f datasets_check_if_exists_by_displayname

function datasets_get_by_displayName {
    if [ ! "$#" -eq 2 ]; then
        echo "usage : ${FUNCNAME[0]} <PROJECT(id or number)> <DISPLAYNAME>"
        return 1
    else
        local PROJECT=$1
        local DISPLAYNAME=$2
        
        # there is now way to directly fetch by displayName so filtering the whole list
        local NAME=$(datasets_list $PROJECT | jq -r --arg n "$DISPLAYNAME" -c 'select(.displayName==$n) | .name')
        datasets_get_by_name $NAME
    fi
}
export -f datasets_get_by_displayName


# ref : https://developers.google.com/maps/documentation/datasets/reference/rest/v1alpha/projects.datasets/get
function datasets_get {
    if [ ! "$#" -eq 2 ]; then
        echo "usage : ${FUNCNAME[0]} <PROJECT(id or number)> <DATASET(uuid)>"
        return 1
    else
        local PROJECT=$1
        local DATASET=$2
        
        #GET https://mapsplatformdatasets.googleapis.com/v1alpha/{name=projects/*/datasets/*}
        local name="projects/$PROJECT/datasets/$DATASET"
        
        datasets_get_by_name $name
    fi
}
export -f datasets_get

function datasets_get_by_name {
    if [ ! "$#" -eq 1 ]; then
        echo "usage : ${FUNCNAME[0]} <NAME=projects/*/datasets/*>"
        return 1
    else
        local NAME=$1
        
        URL=https://mapsplatformdatasets.googleapis.com/v1alpha/$NAME
        
        curl -s -G $URL \
        -H "Authorization: Bearer $(gcloud auth application-default print-access-token)" \
        -H "X-Goog-User-Project: $PROJECT" \
        -H "accept: application/json"
    fi
}
export -f datasets_get_by_name


function datasets_delete {
    datasets_delete_by_uuid $@
}
export datasets_delete

function datasets_delete_by_uuid {
    if [ ! "$#" -eq 2 ]; then
        echo "usage : ${FUNCNAME[0]} <PROJECT(id or number)> <DATASET(uuid)>"
        return 1
    else
        local PROJECT=$1
        local DATASET=$2
        
        local NAME="projects/$PROJECT/datasets/$DATASET"
        datasets_delete_by_name $PROJECT $NAME
    fi
}
export -f datasets_delete_by_uuid

function datasets_delete_by_displayName {
    if [ ! "$#" -eq 2 ]; then
        echo "usage : ${FUNCNAME[0]} <PROJECT(id or number)> <DISPLAYNAME>"
        return 1
    else
        local PROJECT=$1
        local DISPLAYNAME=$2
        local NAME=$(datasets_list $PROJECT | jq -r --arg n "$DISPLAYNAME" -c 'select(.displayName==$n) | .name')
        
        if [[ $NAME != "" ]] && [[ $NAME != "null" ]] ; then
            echo $NAME
            datasets_delete_by_name $PROJECT $NAME
        fi
    fi
}
export -f datasets_delete_by_displayName

# ref : https://developers.google.com/maps/documentation/datasets/reference/rest/v1alpha/projects.datasets/delete
function datasets_delete_by_name {
    
    if [ ! "$#" -eq 2 ]; then
        echo "usage : ${FUNCNAME[0]} <PROJECT(id or number)> <NAME=projects/*/datasets/*>"
        return 1
    else
        local PROJECT=$1
        local NAME=$2
        
        #DELETE https://mapsplatformdatasets.googleapis.com/v1alpha/{name=projects/*/datasets/*}
        
        local URL="https://mapsplatformdatasets.googleapis.com/v1alpha/$NAME"
        
        local FILE_RESPONSE="$DIR_DATASETSAPI_LOGS/${FUNCNAME[0]}_response.json"
        
        curl -s -X DELETE $URL \
        -H "Authorization: Bearer $(gcloud auth application-default print-access-token)" \
        -H "X-Goog-User-Project: $PROJECT" \
        -H "Content-Type: application/json" \
        > ${FILE_RESPONSE}
        cat ${FILE_RESPONSE}

    fi
    
}
export -f datasets_delete_by_name

# ref : https://developers.google.com/maps/documentation/datasets/reference/rest/v1alpha/projects.datasets/create
function datasets_create_gcs {

    #POST https://mapsplatformdatasets.googleapis.com/v1alpha/{parent=projects/*}/datasets

    if [ ! "$#" -eq 5 ]; then
        echo "usage : ${FUNCNAME[0]} <PROJECT(id or number)> <DISPLAY_NAME> <DESCRIPTION> <GCS_URL> <FILE_FORMAT>"
        return 1
    else
        local PROJECT=$1
        local DISPLAY_NAME=$2
        local DESCRIPTION=$3
        local GCS_URL=$4
        local FILE_FORMAT=$5
        
        local PARENT="projects/$PROJECT"
        local URL=https://mapsplatformdatasets.googleapis.com/v1alpha/$PARENT/datasets
        
        local FILE_REQUEST="$DIR_DATASETSAPI_LOGS/${FUNCNAME[0]}_${DISPLAY_NAME}_request_body.json"
        local FILE_RESPONSE="$DIR_DATASETSAPI_LOGS/${FUNCNAME[0]}_${DISPLAY_NAME}_response.json"

        #TIME=$(date -u "+%Y-%m-%dT%H:%M:%SZ")
        cat << EOM > $FILE_REQUEST
{
    "displayName": "$DISPLAY_NAME",
    "description": "$DESCRIPTION",
    "usage":[
        "USAGE_DATA_DRIVEN_STYLING"
    ]
}
EOM
        curl -s POST $URL \
        -H "Authorization: Bearer $(gcloud --project $PROJECT auth application-default print-access-token)" \
        -H "X-Goog-User-Project: $PROJECT" \
        -H "Content-Type: application/json" \
        -H "Accept: application/json" \
        --data @${FILE_REQUEST}  \
        -o /dev/null \
        > $FILE_RESPONSE
        cat $FILE_RESPONSE

        local DATASET=$(jq -r '.name | split("/")[3]' < $FILE_RESPONSE)
        datasets_import_gcs "$PROJECT" "$DATASET" "$GCS_URL" "$FILE_FORMAT" ""
    fi
}
export -f datasets_create_gcs

# ref : https://developers.google.com/maps/documentation/datasets/reference/rest/v1alpha/projects.datasets/create
function datasets_create_local {

    #POST https://mapsplatformdatasets.googleapis.com/v1alpha/{parent=projects/*}/datasets

    if [ ! "$#" -eq 5 ]; then
        echo "usage : ${FUNCNAME[0]} <PROJECT(id or number)> <DISPLAY_NAME> <DESCRIPTION> <FILE_NAME> <FILE_FORMAT>"
        return 1
    else
        local PROJECT=$1
        local DISPLAY_NAME=$2
        local DESCRIPTION=$3
        local FILE_NAME=$4
        local FILE_FORMAT=$5
        
        local PARENT="projects/$PROJECT"
        local URL=https://mapsplatformdatasets.googleapis.com/v1alpha/$PARENT/datasets

        local FILE_REQUEST="$DIR_DATASETSAPI_LOGS/${FUNCNAME[0]}_${DISPLAY_NAME}_request_body.json"
        local FILE_RESPONSE="$DIR_DATASETSAPI_LOGS/${FUNCNAME[0]}_${DISPLAY_NAME}_response.json"

        #TIME=$(date -u "+%Y-%m-%dT%H:%M:%SZ")
        cat << EOM > $FILE_REQUEST
{
    "displayName": "$DISPLAY_NAME",
    "description": "$DESCRIPTION",
    "usage":[
        "USAGE_DATA_DRIVEN_STYLING"
    ]
}
EOM
        curl -s POST $URL \
        -H "Authorization: Bearer $(gcloud --project $PROJECT auth application-default print-access-token)" \
        -H "X-Goog-User-Project: $PROJECT" \
        -H "Content-Type: application/json" \
        -H "Accept: application/json" \
        --data @${FILE_REQUEST}  \
        -o /dev/null \
        > $FILE_RESPONSE
        cat $FILE_RESPONSE

        local DATASET=$(jq -r '.name | split("/")[3]' < $FILE_RESPONSE)
        datasets_import_local "$PROJECT" "$DATASET" "$FILE_NAME" "$FILE_FORMAT" ""
    fi
}
export -f datasets_create_local


# ref : https://developers.google.com/maps/documentation/datasets/reference/rest/v1alpha/media/upload
function datasets_import_gcs {
    if [ ! "$#" -eq 5 ]; then
        echo "usage : ${FUNCNAME[0]} <PROJECT(id or number)> <DATASET(uuid)> <GCS_URL> <FILE_FORMAT> <DESCRIPTION>"
        return 1
    else
    local PROJECT=$1
    local DATASET=$2
    local GCS_URL=$3
    local FILE_FORMAT=$4
    local DESCRIPTION=$5

    # POST https://mapsplatformdatasets.googleapis.com/v1alpha/{name=projects/*/datasets/*}:import
    local NAME=projects/$PROJECT/datasets/$DATASET
    local URL=https://mapsplatformdatasets.googleapis.com/v1alpha/$NAME:import

    local FILE_REQUEST="$DIR_DATASETSAPI_LOGS/${FUNCNAME[0]}_${DATASET}_request_body.json"
    local FILE_RESPONSE="$DIR_DATASETSAPI_LOGS/${FUNCNAME[0]}_${DATASET}_response.json"

    cat << EOM > $FILE_REQUEST
{
    "versionDescription": "$DESCRIPTION",
    "gcsSource":
    {
        "inputUri": "$GCS_URL",
        "fileFormat": "$FILE_FORMAT"
    }
}
EOM
    curl -s POST $URL \
        -H "Authorization: Bearer $(gcloud auth application-default print-access-token)" \
        -H "X-Goog-User-Project: $PROJECT" \
        -H "Content-Type: application/json" \
        -H "Accept: application/json" \
        --data @${FILE_REQUEST}  \
        -o /dev/null \
        > $FILE_RESPONSE
        cat $FILE_RESPONSE
    fi

}
export -f datasets_import_gcs


# ref : https://developers.google.com/maps/documentation/datasets/reference/rest/v1alpha/media/upload
# https://developers.devsite.corp.google.com/maps/documentation/datasets/create#upload_data_from_a_file
function datasets_import_local {
    if [ ! "$#" -eq 5 ]; then
        echo "usage : ${FUNCNAME[0]} <PROJECT(id or number)> <DATASET(uuid)> <FILE_NAME> <FILE_FORMAT> <DESCRIPTION>"
        return 1
    else
        local PROJECT=$1
        local DATASET=$2
        local FILE_NAME=$3
        local FILE_FORMAT=$4
        local DESCRIPTION=$5

        local NAME=projects/$PROJECT/datasets/$DATASET
        local URL=https://mapsplatformdatasets.googleapis.com/upload/v1alpha/$NAME:import

        local FILE_REQUEST="$DIR_DATASETSAPI_LOGS/${FUNCNAME[0]}_${DATASET}_request_body.json"
        local FILE_RESPONSE="$DIR_DATASETSAPI_LOGS/${FUNCNAME[0]}_${DATASET}_response.json"

        cat << EOM > $FILE_REQUEST
{
    "versionDescription": "$DESCRIPTION",
    "local_file_source":
    {
        "file_format": "$FILE_FORMAT"
    }
}
EOM
        cat $FILE_REQUEST
        curl -s -X POST $URL \
            -H "Authorization: Bearer $(gcloud auth application-default print-access-token)" \
            -H "X-Goog-User-Project: $PROJECT" \
            -H "X-Goog-Upload-Protocol: multipart" \
            -F "metadata=@$FILE_REQUEST" \
            -F "rawdata=@$FILE_NAME" \
            > $FILE_RESPONSE
            cat $FILE_RESPONSE
    fi

}
export -f datasets_import_local


function tokenInfo {
    local URL=https://www.googleapis.com/oauth2/v1/tokeninfo
    
    curl $URL \
    -H "Content-Type: application/x-www-form-urlencoded" \
    -d "access_token=$(gcloud auth application-default print-access-token)"
}
export -f tokenInfo


function usage {
cat << EOF
usage : ${0##*/} <FUNCTION NAME> <FUNCTION PARAMS>*
Function list (check code for latest): 
* datasets_create_gcs
* datasets_create_local
* datasets_delete
* datasets_delete_by_displayName
* datasets_delete_by_name
* datasets_delete_by_uuid
* datasets_get
* datasets_get_by_displayName
* datasets_get_by_name
* datasets_import_gcs
* datasets_import_local
* datasets_list
* datasets_check_if_exists_by_displayname
EOF
    # source ${0##*/}
    # source <(curl -s https://gist.githubusercontent.com/n0531m/ae7bb4bcf5f6bb804dbcf89da1efd4f1/raw)
}

"$@"