Hadoop Job configurations via direct REST API call

hadoop_config_via_rest_api.ps1

# An Example of using passing hadoop configurations for a job in HDInsight, via direct REST API
$MyHDInsightUserName = "YourClusterUserName"
$MyHDInsightPwd = "YourPwd"
$clusterName = "YourClusterName"
$storageAcctname = "YourStorageAcctname"
$containerName = "YourDefaultContainerName"

$HdInsightPwd = ConvertTo-SecureString $MyHDInsightPwd -AsPlainText -Force
$HdInsightCreds = New-Object System.Management.Automation.PSCredential ($MyHDInsightUserName, $HdInsightPwd)

# I am submitting a Hive job/Query and would like to use the following Hive configurations to control number of mappers
# hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat
# mapred.map.tasks=10
# This is just an example of how to pass configurations, focus is not on specific configurations or why we are using these configurations.

# Note that I am passing the cofigurations as part of the URI
$uriHiveJobsubmit = "https://$clusterName.azurehdinsight.net/templeton/v1/hive?user.name=$MyHDInsightUserName&define=hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat&define=mapred.map.tasks=10"

# I am using a HQL file for hive Query. Alternatively, you can also use execute=$queryString
#$queryString ="SELECT querytime, market, deviceplatform, devicemake, devicemodel, state, country FROM hivesampletable WHERE ClientId < 100 LIMIT 10;"
#$bodyHiveJobsubmit = @{execute=$queryString;enablelog="false"}

$bodyHiveJobsubmit = @{file="wasb://$containerName@$storageAcctname.blob.core.windows.net/user/azim/TestQuery.hql";statusdir="MyHiveJobStatus";enablelog="false"}

# Invoke-RestMethod just submits the job and gets is a job ID in return
$result = Invoke-RestMethod -Method Post -Uri $uriHiveJobsubmit -Credential $HdInsightCreds -Body $bodyHiveJobsubmit

$jobId = $result.id
write-host $jobId

# Check the job status
$uriJobStatus = "https://$clusterName.azurehdinsight.net/templeton/v1/jobs/$jobid`?user.name=$MyHDInsightUserName"
$jobStatusInfo = Invoke-RestMethod -Method Get -Uri $uriJobStatus -Credential $HdInsightCreds