Recorded Future IOCs to InsightIDR

recorded_future_indicators_v1.ps1

############################################################################
# Copyright (c) Rapid7, LLC 2020  https://www.rapid7.com/
# All rights reserved. This material contains unpublished, copyrighted
# work including confidential and proprietary information of Rapid7.
############################################################################
#
# recorded_future_indicators.ps1
#
# Script version: 2.1.0
# PowerShell Version: 4.0.1.1
# Source: consultant-public
#
# THIS CODE AND INFORMATION ARE PROVIDED "AS IS" WITHOUT WARRANTY OF ANY
# KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A
# PARTICULAR PURPOSE.
#
# Tags: INSIGHTIDR
#
# Description:
# This script will download indicators from the location specified, place
# them into a CSV file, and then upload them to the private threat feed
# specified.  This script is intended to be used with the InsightIDR
# Threat Community threats and uses the InsightiDR REST API v1.

#Enable protocols for a secure communication
$AllProtocols = [System.Net.SecurityProtocolType]'Ssl3,Tls,Tls11,Tls12'
[System.Net.ServicePointManager]::SecurityProtocol = $AllProtocols
#[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12

#Set a working directory, this is where the temp files will live
Set-Location -Path C:\Users\Public\Documents

#***** VARIABLES TO BE UPDATED *****
$rfheader = @{}
#Recorded Future Key and URLs as per their API doc https://api.recordedfuture.com/v2/
$rfheader["X-RFToken"] = "rf_key_here"
#Set your urls for the RF API
$IOCURL1 = "https://api.recordedfuture.com/v2/ip/risklist?format=csv%2Fsplunk&gzip=false"
$IOCURL2 = "https://api.recordedfuture.com/v2/domain/risklist?format=csv%2Fsplunk&gzip=false"
$IOCURL3 = "https://api.recordedfuture.com/v2/hash/risklist?format=csv%2Fsplunk&gzip=false"
$IOCURL4 = "https://api.recordedfuture.com/v2/url/risklist?format=csv%2Fsplunk&gzip=false"

#Change this value to the Threat Key for the threat that is being modified.
#Get the threat key by opening your community threat and selecting Threat Key.
$ThreatKey = "threat_key"

$headers = @{}
#Enter in your platform API key.  This can be generated from the Rapid7 Platform home.
#Log into https://insight.rapid7.com and use the API Management section to generate a key.
$headers["X-Api-Key"] = "api_key"

#***** END OF VARIABLES TO BE UPDATED *****

#These files are used when downloading the indicators and converting them to CSV format.
#They are left insitu on purpose so that you can verify that the script works.  If this bothers you,
#use the sections below to delete these temp files after the indicators are uploaded.
#The first file contains a list of indicators scraped from the $IOCURL website.  It is not cleaned up.
$IOCOutputFileName = "indicators.txt"
#The CSV file is clean and ready to be uploaded.
$CSVOutputFileName = "indicators.csv"

# Get the location of the script for the output files.  Output files
# will be located where script is being run.
$path = Get-Location
$IOCFilePath = "$path\" + "$IOCOutputFileName"
$CSVFilePath = "$path\" + "$CSVOutputFileName"

#This location is where the threats will be uploaded.
$Url = "https://us.api.insight.rapid7.com/idr/v1/customthreats/key/" + $ThreatKey + "/indicators/replace?format=csv"

#Configure the download to use TLS 1.2
#[System.Net.ServicePointManager]::ServerCertificateValidationCallback = {$true}


#delete text download file if it exists already
if (Test-Path tempindicators.txt) {
    Write-Host "Deleting existing indicator file: tempindicators.txt"
    Remove-Item tempindicators.txt
    }

#delete text download file if it exists already
if (Test-Path $IOCFilePath) {
    Write-Host "Deleting existing indicator file: $IOCFilePath"
    Remove-Item $IOCFilePath
    }

#delete csv file of downloaded indicators if it exists already
if (Test-Path $CSVFilePath) {
    Write-Host "Deleting existing CSV file: $CSVFilePath"
    Remove-Item $CSVFilePath
    }
#Download the indicators from the specified URLs.
New-Item "tempindicators.txt" | Out-Null
Write-Host "Downloading indicators from Recorded Future."
Invoke-WebRequest -Uri $IOCURL1 -Headers $rfheader -UseBasicParsing -Method GET -ContentType "application/json" -OutFile "rf_ioc_ip.txt"
Invoke-WebRequest -Uri $IOCURL2 -Headers $rfheader -UseBasicParsing -Method GET -ContentType "application/json" -OutFile "rf_ioc_domain.txt"
Invoke-WebRequest -Uri $IOCURL3 -Headers $rfheader -UseBasicParsing -Method GET -ContentType "application/json" -OutFile "rf_ioc_hash.txt"
Invoke-WebRequest -Uri $IOCURL4 -Headers $rfheader -UseBasicParsing -Method GET -ContentType "application/json" -OutFile "rf_ioc_url.txt"

#We join all files into one for a single upload
Get-Content rf_ioc_*.txt | Set-Content tempindicators.txt

#Clean up the temp file of downloaded indicators.
#This script pulls out an indicator from the first field in the list of output.  You may need to select a different field.
#Change the Select Field1 line to match whatever field has the indicators in it.
#The rest of this block cleans up the download and adds commas to end of each line (so it will be a CSV file).

Write-Host "Reformat the downloaded list of indicators into a comma-delimited text file"
$IOCblocklist = Import-CSV tempindicators.txt -Header "Field1", "Field2", "Field3", "Field4", "Field5", "Field6" `
	| Select Field1 `
	| ConvertTo-CSV -NoTypeInformation `
	| % {$_ -replace  ` '\G(?<start>^|,)(("(?<output>[^,"]*?)"(?=,|$))|(?<output>".*?(?<!")("")*?"(?=,|$)))' ` ,'${start}${output}'} `
	| %{$_ -replace '$',','}`
	| Out-File $IOCFilePath -fo -en ascii ;


#You can uncomment the following line to delete blank lines from the output, if there are any.
#(Get-Content $IOCFilePath) | ? {$_.trim() -ne "" } | set-content $CSVFilePath
Write-Host "Clean up the file by removing the header"
#Skip reading the first two lines of the file, which are headers describing the fields in the file.
#Delete all of the lines that start with a #, which are also part of the header.
Get-Content $IOCFilePath | Select-Object -Skip 2 | Where { $_ -notmatch '^\#' } | Set-Content $CSVFilePath

#checks for indicator csv file.  If it does not exist, end script.
if (-not (Test-Path $CSVFilePath)) {
	Write-Host "Empty Indicators List, Ending Script without uploading any content"
	Break
	}

#Command to emulate curl with powershell.
Write-Host "Starting command to connect to API and upload indicators"
$ContentType = 'text/csv'
Invoke-WebRequest -Uri $Url -Headers $headers -InFile $CSVFilePath -Method Post -ContentType $ContentType -UseBasicParsing

Write-Host "Script has finished running.  Check your results."