Skip to content

Instantly share code, notes, and snippets.

@glennsarti
Created July 10, 2015 04:04
Show Gist options
  • Save glennsarti/ea09e9b346c842ff5d45 to your computer and use it in GitHub Desktop.
Save glennsarti/ea09e9b346c842ff5d45 to your computer and use it in GitHub Desktop.
These two scripts can be used to convert the data from the Star Wars API (http://swapi.co) into an import file for Neo4j. These scripts download the JSON data from the API and converts it into Cypher statements. It's crude and probably not the best way to do it, but, it works!. Firstly download the JSON data using GetSWAPI.ps1, and then run Conv…
$ErrorActionPreference = 'Stop'
$VerbosePreference = 'SilentlyContinue'
Function Safe-CypherString($value) {
$value = $value.Replace('\','\\').Replace("`n",'\n').Replace("`r",'\r').Replace("'","\'")
Write-Output $value
}
Function Convert-JSONToCypher($dataObject) {
Write-Verbose "Processing $($dataObject.url)"
if ($matches -ne $null) { $matches.Clear() }
if (($dataobject.Url) -match '\/([a-zA-Z]+)\/\d+\/$') {
switch ($matches[1].ToLower()){
'films' { $nodeLabel = 'Film' }
'people' { $nodeLabel = 'Person' }
'planets' { $nodeLabel = 'Planet' }
'species' { $nodeLabel = 'Species' }
'starships' { $nodeLabel = 'Starship' }
'vehicles' { $nodeLabel = 'Vehicle' }
default { Throw "Unknown URL Type $($matches[1])" }
}
}
$cypher = ''
$cypherSet = ''
$relationships = @()
Get-Member -InputObject $dataObject -MemberType NoteProperty | % {
$thisProperty = $_
Write-Verbose "Processing property $($thisProperty.Name)"
$thisPropertyValue = $thisResult."$($thisProperty.Name)"
if (($thisProperty.Name -ne 'created') -and ($thisProperty.Name -ne 'edited') -and ($thisPropertyValue -ne $null) )
{
switch ($thisPropertyValue.GetType().ToString())
{
'System.Object[]' {
$relName = ''
switch ($thisProperty.Name) {
'films' { $relName = 'FILM' }
'species' { $relName = 'SPECIES' }
'starships' { $relName = 'STARSHIP' }
'vehicles' { $relName = 'VEHICLE' }
'planets' { $relName = 'PLANET' }
'people' { $relName = 'PERSON' }
'pilots' { $relName = 'PILOT' }
'residents' { $relName = 'RESIDENT' }
'characters' { $relName = 'CHARACTER' }
default { Throw "Unknown array name of $($thisProperty.Name)" }
}
$thisPropertyValue | % {
$relationships += ($relName + "|" + $_)
}
}
'System.String' {
if (($thisPropertyValue.IndexOf('http') -ne -1) -and ($thisProperty.Name -ne 'url'))
{
$relName = ''
switch ($thisProperty.Name) {
'homeworld' { $relName = 'HOMEWORLD' }
default { Throw "Unknown single reference name of $($thisProperty.Name)" }
}
$relationships += ($relName + "|" + $thisPropertyValue)
}
else
{
$cypherSet += ",$($thisProperty.Name):'$(Safe-CypherString $thisPropertyValue)'"
}
}
'System.Int32' {
$cypherSet += ",$($thisProperty.Name):$thisPropertyValue"
}
default { Throw "Unknown object type $($thisPropertyValue.GetType().ToString())" }
}
}
}
$cypherSet = 'n = { ' + $cypherSet.SubString(1,$cypherSet.Length - 1) + '}'
$cypher = "MERGE (n {url:'$($dataobject.Url)'}) SET n:$($nodeLabel), $cypherSet;`n`r"
$relText = ''
$index = [int]0
$relationships | % {
$relName = ($_.Split('|')[0])
$url = ($_.Split('|')[1])
$relText += "MERGE (n$($index.ToString()) {url:'$($url)'})`n`rMERGE (n)-[:$($relName)]->(n$($index.ToString()))`n`r"
$index++
}
if ($relText -ne '') {
$cypher += "MATCH (n {url:'$($dataObject.url)'})`n`r" + $relText + ';'
}
Write-Output $cypher
}
@"
// Delete Everything
MATCH ()-[r]-() DELETE r;
MATCH (n) DELETE n;
"@ | Out-File -FilePath "$($PSScriptRoot)\StarWars.cypher" -Encoding "ASCII" -Force -Confirm:$false
Get-ChildItem -Path $PSScriptRoot -Filter "*.json" | % {
$thisFile = $_
Write-Verbose "Reading $($_.Fullname)"
Write-Output "// Exported from $($thisFile.Name)"
$dataFile = ConvertFrom-JSon -InputObject ([IO.File]::ReadAllText($thisFile.Fullname))
$dataFile.results | % {
$thisResult = $_
Write-Output (Convert-JSONToCypher $thisResult)
}
} | Out-File -FilePath "$($PSScriptRoot)\StarWars.cypher" -Encoding "ASCII" -Append -NoClobber -Force -Confirm:$false
$rootURL = 'http://swapi.co:80/api'
$resources = @('films','people','starships','vehicles','species','planets')
Get-ChildItem -Path $PSScriptRoot -Filter "*.json" | Remove-Item -Force -Confirm:$false | Out-Null
$resources | % {
$resource = $_
$page = [int]1
try
{
do
{
if ($page -eq 1) { $url = "$rootURL/$resource/" } else { $url = "$rootURL/$resource/?page=$page" }
Write-Host $url
Invoke-RestMethod -Uri $url -Method GET -OutFile "$PSScriptRoot\$($resource)_$($page).JSON" -ErrorAction 'Stop'
$page++
} until ($page -gt 50)
}
catch
{ # Ignore errors. It's a hack but hey...
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment