Churn through the Kiln Queue with restarts

churnQueue.ps1

function get-queueLength(){
  try {
    $s = (New-Object net.webclient).DownloadString('http://localhost:56785/stats.json')
  }
  catch {
    return "queue length unavailable"
  }
  $queueLength = $s -split (',') | foreach{if ($_ | select-string "queueLength" -quiet){ ($_ -split ":")[1]}}
  return $queueLength
}
function churn-queue(){
  pushd
  cd "C:\Program Files\Kiln\queue"
  # We want to stop the Kiln Queuing Service because we're going to be running it
  # manually.
  write "---------------- Stopping Queuing Service ----------------"
  sc.exe stop "Kiln Queuing Service"
  $continue = $true
  $restartKss = $false
  # We're going to loop until we're out of work for the queue service.
  while ($continue){
    if ($restartKss) {
      # With the QueueService hammering Redis, this can cause Redis to fail at the
      # network level. This causes the Queuing Service to fail. Restarting the 
      # Kiln Storage Service restarts Redis so that the Queuing Service can continue.
      $restartKss = $false
      write "---------------- Restart Kiln Storage Service ----------------"
      sc.exe stop "KilnStorageService"
      sc.exe start "KilnStorageService"
    }
    $getQueueLength = $true
    # The following code is going to process the Queue Service manually, printing the
    # output. Powershell is then piping the output as a stream, which is tested for 
    # various conditions to determine if the process is working.
    write "---------------- Churn Through Kiln Queue ----------------"
    . .\QueueService.exe  /verbose /noservice | out-string -stream | foreach{
      if ($getQueueLength) {
        # We'll use get-queueLength (defined above) to query the queue service 
        # stats to see how many items we have left in the queue. This number
        # should go down over time. We'll query it once each time we start the
        # Queue Service.
        $getQueueLength = $false
        $queueLength = get-queueLength
        write "---------------- Get Kiln Queue Length ----------------"
        write "KILN QUEUE LENGTH: $queueLength"
      }
      write $_
      if($_ | select-string "no work" -quiet){
        # If there are no items left for the Queue, we'll see a message that says "no work",
        # at which point we'll want to exit the entire loop.
        write "---------------- No more Work in the Queue ----------------"
        $continue = $false
	continue
      }
      if($_ | select-string "Exception" -quiet){
        # If we see the word "Exception", this almost certainly means that Redis has crashed.
        # We're going set a flag to restart the Kiln Storage Service, then continue to 
        # restart the current loop. The Storage Service will get restarted at the top of 
        # the loop.
        write "---------------- Exception While Processing the Queue ----------------"
        $restartKss = $true
        continue
      }
    }
  }
  popd
}