Put-Elasticsearch. powershell用のelasticsearch投入スクリプト

gistfile1.txt

function Put-Elasticsearch {
<#
.SYNOPSIS
elasticsearch投入用の汎用メソッド。
現状、

_ID　DATEが渡された場合は「$date+$e.objKey」。
_ID　DATEが渡されない場合は「$e.objKey」。

@timestamp　Dateが渡された場合は、@timestampをdateより生成
@timestamp　Dateが渡されない場合は、@timestampが、「Get-Date -Date $_ -format s」の書式で入っているとする。

.DESCRIPTION

.PARAMETER date
日付の配列。渡されたオブジェクトを渡された日付名義で登録する

.PARAMETER putDatas
投入用データ。現状MAPはNG

#>
[CmdletBinding()]
Param(  
   $date,
   [Parameter(Mandatory=$True)]
   $putDatas,
   $bulk,
   $index,
   $type,
   $elaHost,
   $port,
   [switch]$dateIndex

   
)
  begin { 
  		$dateList= @()
		$error.Clear()
	}	
  process  {  	
  	if($date -ne $null){
		$dateList +=$date
	}
  }
  end  {
	try{
		$url    = 'http://'+$elaHost+':'+$port+'/_bulk'
		$urlOne = 'http://'+$elaHost+':'+$port+'/'+$index+'/'+$type  
		$objType=$type
  		
		Write-Host "データ投入先"
		Write-Host $urlOne  "bulk:" $bulk
		
		
		$postParams=""
		
		if($dateList.length -eq 0){
			$rowDateMode=$true;
			$dateList += Get-Date
		}
		
		$dateList | %{
			$datetemp = Get-Date -Date $_ 
			$datetemp　= $datetemp.AddHours(-9)
			$dateStr = Get-Date -Date $datetemp -format s
			
			if($dateIndex){
				$indexSuffix ="-"+ (Get-Date -Date $datetemp -Format d |% {$_ -replace '/' ,'.' })
			}else{
				$indexSuffix =""
			}
			
			$putDatas | % {
						$e=$_
						if($rowDateMode){
							##キーはそのまま使う
							##@timestampは投入済み
							
							$k=$e.objKey -replace "[/:%]", "_"
							if($dateIndex){
								$indexSuffix ="-"+ (Get-Date -Date $e."@timestamp" -Format d |% {$_ -replace '/' ,'.' })
							}
							
							
						}else{
							
							$e.PSObject.Properties.Remove('@timestamp')						
							$e | Add-Member -MemberType NoteProperty -Name "@timestamp" -Value $dateStr 
							$k=$dateStr+$e.objKey
						}
						
						## 型が頻繁に変わる場合に
						## objTypeに型を入れておく
						if($type -eq $null){
							$objType = $e.objType
						}
						
						$str = $e  | ConvertTo-Json -Depth 2 -Compress						
						
						if($bulk -eq $false	){
							$postData = [System.Text.Encoding]::UTF8.GetBytes($str)
							
							$urlOne = 'http://'+$elaHost+':'+$port+'/'+$index+$indexSuffix+'/'+$type  
							$urlPut =$urlOne +"/"+$k
							
							$r=Invoke-WebRequest -Uri $urlPut -Method POST -Body $postData
							Write-Host $r
						}
						if( $bulk ){						
							$r =Measure-Command   {	
								$postParams  +='{ "create" : { "_index" : "'+$index+$indexSuffix+'", "_id" : "' + $k  +'", "_type" : "'+$objType+ '" }'
								$postParams  +="`n"
								$postParams  +=$str
								$postParams  +="`n"
							}
							$rStr += $r.TotalSeconds 								
						}
			}
			Write-Verbose ("post  {0} " -f $postParams )
			if($bulk -and $postParams.length -gt 1000*1000*3 ){
				$r =Measure-Command   {	
					$postData = [System.Text.Encoding]::UTF8.GetBytes($postParams)
					$result =Invoke-WebRequest -Uri $url -Method POST -Body $postData			
					$postParams=""
				}				
				#Write-Host $result 
				Write-Host "LOG:Invoke-WebRequest:bulk 		" $r.TotalSeconds      
				
			}
		}  
		
		if($bulk ){
				$r =Measure-Command   {	
					$postData = [System.Text.Encoding]::UTF8.GetBytes($postParams)
					$result =Invoke-WebRequest -Uri $url -Method POST -Body $postData			
					$postParams=""
				}				
				#Write-Host $result 
				Write-Host "LOG:Invoke-WebRequest:bulk 		" $r.TotalSeconds
		}  
	} catch [Exception] {
		        Write-Host "Put-Kibana error--------------------------------"
				Write-Host $error
				Write-Host $error.InvocationInfo
				Write-Host $error.TargetObject

				Write-Host $_.toString()
				Write-Host $_.Exception.StackTrace.toString()
						
				$error.Clear()				
		        
		        Write-Host  "e.--------------------------------"
		    	Write-Host $e
		        
			
	}  
  }
}