Created
December 18, 2020 19:24
-
-
Save mySYSMON/513059e917a150c561aad665fcdfded3 to your computer and use it in GitHub Desktop.
poc-domainrank-WithBugs.ps1
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #free bugs, use for reference only | |
| #free bugs, use for reference only | |
| Function poc-dns1 { | |
| [CmdletBinding()] | |
| param( | |
| #[Parameter(Mandatory=$true)] | |
| #[bool]$ = $false | |
| [String]$CommonCrawlFilePath | |
| ) | |
| # IMPORTANT PLEASE READ.. | |
| # POC-DomainRank requires Sysmon | |
| # Download link: https://docs.microsoft.com/en-us/sysinternals/downloads/sysmon | |
| # -DomainRank requires the commonCrawl domain rank file | |
| #Download link: https://commoncrawl.s3.amazonaws.com/projects/hyperlinkgraph/cc-main-2020-jul-aug-sep/host/cc-main-2020-jul-aug-sep-host-ranks.txt.gz' | |
| date | |
| $numDomains = 151000000 # 151 million | |
| $continue = $true | |
| write-host -NoNewline 'Checking for C:\Windows\sysmon*.exe... ' | |
| if (test-path "C:\Windows\sysmon*.exe") | |
| { | |
| write-host 'yes' -ForegroundColor Green | |
| } | |
| else | |
| { | |
| $continue = $false | |
| write-host 'no' -ForegroundColor Red | |
| write-host 'POC-DomainRank requires Sysmon' -ForegroundColor Red | |
| write-host 'Sysmon is not installed on this system.' -ForegroundColor Red | |
| write-host 'Download link: https://docs.microsoft.com/en-us/sysinternals/downloads/sysmon' -ForegroundColor Red | |
| } | |
| write-host -NoNewline 'Checking if sysmon* is running... ' | |
| if (Get-Process "sysmon*") | |
| { | |
| write-host 'yes' -ForegroundColor Green | |
| } | |
| else | |
| { | |
| $continue = $false | |
| write-host 'no' -ForegroundColor Red | |
| write-host 'Sysmon is not running' -foregroundcolor red | |
| write-host -NoNewline 'Please start Sysmon before installing -DomainRank' -foregroundcolor red | |
| } | |
| if ($CommonCrawlFilePath -eq "" -or $CommonCrawlFilePath -eq $null) | |
| { | |
| #write-host -NoNewline 'Checking for '$home'\Downloads\cc-main*-host-ranks.txt... ' | |
| write-host -NoNewline 'Checking for D:\data\cc-main*-host-ranks.txt' | |
| #if (test-path "$home\Downloads\cc-main*-host-ranks.txt") | |
| if (test-path "D:\data\cc-main*-host-ranks.txt") | |
| { | |
| write-host 'yes' -ForegroundColor Green | |
| $CommonCrawlFilePath = Get-ChildItem("D:\data\cc-main*-host-ranks.txt") | |
| #$CommonCrawlFilePath = Get-ChildItem("$home\Downloads\cc-main*-host-ranks.txt") | |
| } | |
| else | |
| { | |
| $continue = $false | |
| write-host 'no' -ForegroundColor Red | |
| write-host 'Checking for '$home'\Downloads\cc-main*-host-ranks.txt.gz ' | |
| if (test-path "$home\Downloads\cc-main*-host-ranks.txt.gz") | |
| { | |
| write-host 'yes' -ForegroundColor Green | |
| write-host 'Error, please unzip cc-main*-host-ranks.txt.gz using 7zip' -foregroundcolor red | |
| } | |
| else | |
| { | |
| $continue = $false | |
| write-host 'Unable to find cc-main*-host-ranks.txt.gz or cc-main*-host-ranks.txt' -foregroundcolor red | |
| write-host 'Download link:' -foregroundcolor red | |
| write-host 'https://commoncrawl.s3.amazonaws.com/projects/hyperlinkgraph/cc-main-2020-jul-aug-sep/host/cc-main-2020-jul-aug-sep-host-ranks.txt.gz' | |
| } | |
| } | |
| } | |
| else | |
| { | |
| write-host 'Checking CommonCrawlFilePath' $CommonCrawlFilePath | |
| if (!($CommonCrawlFilePath.EndsWith('.txt'))) | |
| { | |
| $continue = $false | |
| write-host 'CommonCrawlFilePath should be a txt file' -foregroundcolor red | |
| } | |
| } | |
| if ($continue) | |
| { | |
| $i = 0 | |
| write-host -NoNewline 'Reading file ' $CommonCrawlFilePath'...' | |
| $domains1 = new-object system.collections.generic.List[string] | |
| $domains2 = new-object system.collections.generic.List[string] | |
| $streamreader = New-Object IO.StreamReader $CommonCrawlFilePath | |
| while(($line = $streamreader.ReadLine()) -ne $null) | |
| { | |
| $i++ | |
| if ($i -eq 1) {continue} #skip header | |
| $tmp = $line.split("`t") | |
| $numberOfDots = $tmp[4].Split(".").GetUpperBound(0) | |
| #write-host $tmp[4] | |
| $tmp = $tmp[4] + "," + $tmp[0] | |
| #$t = $mylist.Add($tmp) | |
| if ($numberOfDots -lt 4) | |
| { | |
| if ($i -lt ($numDomains / 2)) | |
| { | |
| $t = $domains1.Add($tmp) | |
| } | |
| else | |
| { | |
| $t = $domains2.Add($tmp) | |
| } | |
| } | |
| if ($i -gt $numDomains) | |
| { | |
| $streamreader.Close() | |
| break | |
| } | |
| } | |
| write-host 'DONE' -ForegroundColor Green | |
| del $home\Downloads\BigDomainList.txt | |
| write-host -NoNewline 'Writing to file '$home'\Downloads\BigDomainList.txt...' | |
| $streamwriter = New-Object IO.StreamWriter $home\Downloads\BigDomainList.txt, $true | |
| foreach ($domain in $domains1) | |
| { | |
| $streamwriter.WriteLine($domain) | |
| } | |
| $domains1 = '' | |
| foreach ($domain in $domains2) | |
| { | |
| $streamwriter.WriteLine($domain) | |
| } | |
| $streamwriter.Close() | |
| write-host 'DONE' -ForegroundColor Green | |
| $streamwriter.Close() | |
| write-host 'DONE' -ForegroundColor Green | |
| date | |
| #del $home\Downloads\cc-main-tmp-temp.zip | |
| #write-host 'Zipping file...' | |
| #Compress-Archive -Path $home'\Downloads\cc-main-tmp-temp.txt -DestinationPath' $home'\Downloads\cc-main-tmp-temp.zip | |
| #Compress-Archive -LiteralPath "$home\Downloads\cc-main-tmp-temp.txt" -DestinationPath "$home\Downloads\cc-main-tmp-temp.zip" | |
| ######Compress-7Zip -CompressionLevel Fast -Path "$home\Downloads\cc-main-tmp-temp.txt" -ArchiveFileName "$home\Downloads\cc-main-tmp-temp.zip" | |
| } | |
| } | |
| Function GetBestMatch ([string]$querynameReversed, [string]$datafile,[int]$print) | |
| { | |
| $out = '' | |
| $ripgrep = "$home\Downloads\ripgrep.exe" | |
| $out = .\ripgrep.exe -m 1 $querynameReversed $datafile | |
| if (($out.length -gt 0 -or ([regex]::Matches($querynameReversed,"\.").count -lt 1))) | |
| { | |
| if ($print -gt 0) | |
| { | |
| write-host $querynameReversed | |
| write-host $out | |
| } | |
| write-host 12 | |
| write-host $out | |
| return $out | |
| } | |
| else | |
| { | |
| if ([regex]::Matches($querynameReversed,"\.").count -gt 1) | |
| { | |
| if ($print -gt 0) {write-host 18;write-host $querynameReversed} | |
| $querynameReversed = $querynameReversed.Substring(0, $querynameReversed.lastIndexOf('\.')) | |
| $querynameReversed = $querynameReversed + ',' | |
| if ($print -gt 0) {write-host 21;write-host $querynameReversed} | |
| if ($print -eq 0) | |
| {GetBestMatch $querynameReversed $datafile 0} #Recursive} | |
| else | |
| {GetBestMatch $querynameReversed $datafile 99} #Recursive} | |
| } | |
| } | |
| } | |
| function GetPartialMatch ([string]$querynameReversed, [string]$datafile) #, [int]$level) | |
| { | |
| $out = '' | |
| # using -m 1 to get first match only. | |
| # this prevents many matches. | |
| $ripgrep = "$home\Downloads\ripgrep.exe" | |
| $out = .\ripgrep.exe -m 1 $querynameReversed $datafile | |
| #if (($out.length -gt 0 -or ([regex]::Matches($querynameReversed,"\.").count -lt 2))) | |
| if (($out.length -gt 0 -or ([regex]::Matches($querynameReversed,"\.").count -lt 2))) | |
| { | |
| if ($out.length -lt 1) | |
| { | |
| write-host $querynameReversed | |
| write-host $out | |
| } | |
| write-host 45 | |
| write-host $out | |
| return $out | |
| } | |
| else | |
| { | |
| if ([regex]::Matches($querynameReversed,"\.").count -gt 2) | |
| { | |
| write-host $querynameReversed | |
| $querynameReversed = $querynameReversed.Substring(0, $querynameReversed.lastIndexOf('\.')) | |
| $querynameReversed = $querynameReversed.Substring(0, $querynameReversed.lastIndexOf('\.')) | |
| $querynameReversed = $querynameReversed + '\.' | |
| GetPartialMatch $querynameReversed $datafile #Recursive | |
| } | |
| return $out | |
| } | |
| } | |
| cls;$d = date;write-host $d | |
| $events = Get-WinEvent -FilterHashtable @{logname='Microsoft-Windows-Sysmon/Operational';id=22;StartTime=(Get-Date).AddHours(-1);EndTime=get-date} | |
| #$events = Get-WinEvent -FilterHashtable @{logname='Microsoft-Windows-Sysmon/Operational';id=22;StartTime=(Get-Date).AddDays(-20);EndTime=(Get-Date).AddHours(-5)} #-MaxEvents 10 | |
| write-host $events.Count | |
| $d = date;write-host $d | |
| $UniqDomains = [System.Collections.ArrayList]@() | |
| $AlreadySearched = [System.Collections.ArrayList]@() | |
| if (test-path $home\Downloads\poc-dnsrank-matches.txt) | |
| {del $home\Downloads\poc-dnsrank-matches.txt} | |
| if (test-path $home\Downloads\try-exact-match.txt) | |
| {del $home\Downloads\try-exact-match.txt} | |
| if (test-path $home\Downloads\try-partial-match.txt) | |
| {del $home\Downloads\try-partial-match.txt} | |
| $eventcount = 0 | |
| foreach ($log in $events) | |
| { | |
| [xml]$xmllog = $log.toXml() | |
| $queryname = $xmllog.event.eventdata.data | where Name -eq 'queryname' | select '#text' | |
| $querynameStr = $queryname.'#text'.ToString().ToLower() | |
| #write-host $querynameStr | |
| #dns queries must have at least 1 letter | |
| $pattern = ".*[a-zA-Z].*" | |
| if ($querynameStr -match $pattern) | |
| { | |
| #for our checks dns queries must have at least 1 dot | |
| if ([regex]::Matches($querynameStr,"\.").count -gt 0) | |
| { | |
| $tokens = $querynameStr.Split('.') | |
| [array]::Reverse($tokens) | |
| $querynameReversed = '^' | |
| foreach($token in $tokens) | |
| { | |
| $querynameReversed = $querynameReversed + $token + '\.' | |
| } | |
| #remove trailing '\.' | |
| $querynameReversed = $querynameReversed -replace “\\\.$” | |
| $querynameReversed = $querynameReversed -replace '\\\.www$','' | |
| if (!($UniqDomains.Contains("$querynameReversed,"))) | |
| { | |
| [void]$UniqDomains.Add("$querynameReversed,") | |
| Add-Content -Path $home\Downloads\try-exact-match.txt ` | |
| -Value "$querynameReversed," | |
| #write-output "1" | |
| } | |
| if (([regex]::Matches($querynameReversed,"\.")).count -eq 1) | |
| { | |
| #comma is delimiter in file between domain name and rank | |
| #comma "," is similar to regex "$" in this case | |
| if (!($UniqDomains.Contains("$querynameReversed,"))) | |
| { | |
| [void]$UniqDomains.Add("$querynameReversed,") | |
| Add-Content -Path $home\Downloads\try-exact-match.txt ` | |
| -Value "$querynameReversed," | |
| #write-output "1" | |
| } | |
| } | |
| elseif (([regex]::Matches($querynameReversed,"\.")).count -eq 2) | |
| { | |
| if (!($UniqDomains.Contains("$querynameReversed,"))) | |
| { | |
| [void]$UniqDomains.Add("$querynameReversed,") | |
| Add-Content -Path $home\Downloads\try-exact-match.txt ` | |
| -Value "$querynameReversed," | |
| #write-output "2" | |
| } | |
| $querynameReversed = $querynameReversed.Substring(0, $querynameReversed.lastIndexOf('\.')) | |
| $querynameReversed = $querynameReversed + '\.' | |
| if (!($UniqDomains.Contains("$querynameReversed"))) | |
| { | |
| [void]$UniqDomains.Add("$querynameReversed") | |
| Add-Content -Path $home\Downloads\try-partial-match.txt ` | |
| -Value "$querynameReversed" | |
| #write-output "3" | |
| } | |
| #remove trailing '\.' in search for "exact" partial match | |
| #IE missing on *.google.com so match on google.com | |
| #comma is delimiter in file between domain name and rank | |
| $querynameReversed = $querynameReversed -replace “\\\.$” | |
| if (!($UniqDomains.Contains("$querynameReversed,"))) | |
| { | |
| [void]$UniqDomains.Add("$querynameReversed,") | |
| Add-Content -Path $home\Downloads\try-exact-match.txt ` | |
| -Value "$querynameReversed," | |
| Add-Content -Path $home\Downloads\try-partial-match.txt ` | |
| -Value "$querynameReversed," | |
| #write-output "4" | |
| } | |
| } | |
| } | |
| else | |
| { | |
| write-host -NoNewline "INFO: " | |
| write-host "no dots " $querynameStr | |
| } | |
| } | |
| else | |
| { | |
| write-host -NoNewline "INFO: " | |
| write-host "no letters " $querynameStr | |
| } | |
| } | |
| $d = date | |
| $out1 = 'j' | |
| $out2 = 'j' | |
| write-host $d | |
| $ripgrep = "$home\Downloads\ripgrep.exe" | |
| if (test-path $home\Downloads\try-exact-match.txt) | |
| { | |
| $out1 = .\ripgrep.exe -f $home\Downloads\try-exact-match.txt $home\Downloads\BigDomainList.txt | |
| Add-Content -Path $home\Downloads\poc-dnsrank-matches.txt -Value $out1 | |
| Add-Content -Path $home\Downloads\Xpoc-dnsrank-matches.txt -Value $out1 | |
| foreach ($item in $out1) | |
| { | |
| $tmpItem = "^" + $item.Split(',')[0] + "," | |
| $tmpItem = $tmpItem -replace '\.', '\.' | |
| #write-host $tmpItem | |
| [void]$AlreadySearched.Add($tmpitem) | |
| } | |
| } | |
| if (test-path $home\Downloads\try-partial-match.txt) | |
| { | |
| $out2 = .\ripgrep.exe -f $home\Downloads\try-partial-match.txt $home\Downloads\try-exact-match.txt | |
| Add-Content -Path $home\Downloads\poc-dnsrank-matches.txt -Value $out2 | |
| } | |
| $d = date | |
| write-host $d | |
| $datafile = "$home\Downloads\poc-dnsrank-matches.txt" | |
| $bigDomainList = $datafile = "$home\Downloads\BigDomainList.txt" | |
| $MediumDomainList = $datafile = "$home\Downloads\MediumDomainList.txt" | |
| foreach ($log in $events) | |
| { | |
| #write-host 265 | |
| [xml]$xmllog = $log.toXml() | |
| <# | |
| $eventrecordid = $log.RecordId | |
| $computer = $log.MachineName | |
| $userid = $log.UserId | |
| $timecreated = $log.TimeCreated | |
| $utctime = $xmllog.event.eventdata.data | where Name -eq 'utctime' | select '#text' | |
| $processguid = $xmllog.event.eventdata.data | where Name -eq 'processguid' | select '#text' | |
| $processid = $xmllog.event.eventdata.data | where Name -eq 'processid' | select '#text' | |
| #$queryname = $xmllog.event.eventdata.data | where Name -eq 'queryname' | select '#text' | |
| $querystatus = $xmllog.event.eventdata.data | where Name -eq 'querystatus' | select '#text' | |
| $queryresults = $xmllog.event.eventdata.data | where Name -eq 'queryresults' | select '#text' | |
| $image = $xmllog.event.eventdata.data | where Name -eq 'image' | select '#text' | |
| #$querynameStr = $queryname.'#text'.ToString() | |
| #$userid_querynameStr = $userid.ToString() + ':' + $querynameStr + 'END' | |
| #> | |
| $queryname = $xmllog.event.eventdata.data | where Name -eq 'queryname' | select '#text' | |
| $querynameStr = $queryname.'#text'.ToString().ToLower() | |
| $pattern = ".*[a-zA-Z].*" | |
| if ($querynameStr -match $pattern) | |
| { | |
| #for our checks dns queries must have at least 1 dot | |
| if ([regex]::Matches($querynameStr,"\.").count -gt 0) | |
| { | |
| $tokens = $querynameStr.Split('.') | |
| [array]::Reverse($tokens) | |
| $querynameReversed = '^' | |
| foreach($token in $tokens) | |
| { | |
| $querynameReversed = $querynameReversed + $token + '\.' | |
| } | |
| #edge case cleanup | |
| $querynameReversed = $querynameReversed -replace '\^\\\.','^' | |
| #remove trailing '\.' | |
| $querynameReversed = $querynameReversed -replace '\\\.$' | |
| #remove trailing '\.www' | |
| $querynameReversed = $querynameReversed -replace '\\\.www$','' | |
| if (([regex]::Matches($querynameReversed,"\.")).count -lt 20) #number of dots less than 20! :) | |
| { | |
| $out1 = '' | |
| $outdot = '' | |
| $level = 0 | |
| #write-host 309 | |
| #$querynameReversed = '^com\.associates-amazon' | |
| $querynameReversed = $querynameReversed + ',' | |
| #write-host '---' | |
| #write-host $querynameReversed | |
| if (!($AlreadySearched.Contains("$querynameReversed"))) | |
| { | |
| write-host 328 | |
| $out1 = GetBestMatch $querynameReversed $datafile 0 # $level | |
| write-host -NoNewline 'line 331' | |
| #write-host $querynameReversed | |
| #sleep 2 | |
| [void]$AlreadySearched.Add("$querynameReversed") | |
| if ($out1.length -eq 0) | |
| { | |
| write-host -NoNewline 'line 338' | |
| #remove trailing comma and dot | |
| $querynameReversed = $querynameReversed -replace “,$” | |
| $querynameReversed = $querynameReversed + '\.' | |
| write-host $querynameReversed | |
| $outdot = GetPartialMatch $querynameReversed $datafile | |
| #$outdot = $home\Downloads\ripgrep.exe -m 1 $querynameReversed $home\Downloads\poc-dnsrank-matches.txt | |
| } | |
| if ($out1.length -eq 0 -and $outdot.length -eq 0) | |
| { | |
| write-host -NoNewline 'line 514' | |
| #remove trailing dot | |
| $querynameReversed = $querynameReversed -replace '\\\.$' | |
| write-host 'GetBestMatch with MediumDomainList' | |
| write-host $querynameReversed | |
| #write-output 'GetBestMatch with MediumDomainList' | |
| #write-output $querynameReversed | |
| $out1 = GetBestMatch $querynameReversed $MediumDomainList 99 | |
| } | |
| if ($out1.length -eq 0 -and $outdot.length -eq 0) | |
| { | |
| write-host -NoNewline 'line 525' | |
| #remove trailing dot | |
| $querynameReversed = $querynameReversed -replace '\\\.$' | |
| write-host 'GetBestMatch with bigDomainList' | |
| write-host $querynameReversed | |
| #write-output 'GetBestMatch with bigDomainList' | |
| #write-output $querynameReversed | |
| $out1 = GetBestMatch $querynameReversed $bigDomainList 99 | |
| } | |
| if ($out1.length -eq 0 -and $outdot.length -eq 0) | |
| { | |
| write-host -NoNewline 'line 536' | |
| write-host 'GetPartialMatch with MediumDomainList' | |
| write-host $querynameReversed | |
| #write-output 'GetPartialMatch with MediumDomainList' | |
| #write-output $querynameReversed | |
| $outdot = GetPartialMatch $querynameReversed $MediumDomainList | |
| write-host 'return ok' | |
| } | |
| if ($out1.length -eq 0 -and $outdot.length -eq 0) | |
| { | |
| write-host -NoNewline 380 | |
| write-host 'GetPartialMatch with bigDomainList' | |
| write-host $querynameReversed | |
| #write-output 'GetPartialMatch with bigDomainList' | |
| write-output $querynameReversed | |
| $outdot = GetPartialMatch $querynameReversed $bigDomainList | |
| write-host 'return ok' | |
| } | |
| if ($out1.length -gt 0) | |
| { | |
| write-host 355 | |
| Add-Content -Path $home\Downloads\poc-dnsrank-matches.txt -Value $out1 | |
| } | |
| else | |
| { | |
| write-host 'line 567' | |
| write-host $querynameReversed | |
| write-host 'out1 is empty' | |
| } | |
| if ($outdot.length -gt 0) | |
| { | |
| write-host 'line 575' | |
| write-host $outdot | |
| Add-Content -Path $home\Downloads\poc-dnsrank-matches.txt -Value $outdot | |
| } | |
| } | |
| else | |
| { | |
| write-host -NoNewline '.' | |
| } | |
| } | |
| else | |
| { | |
| write-host "many dots #" | |
| write-output $querynameReversed | |
| } | |
| } | |
| } | |
| } | |
| $d = date | |
| write-host $d | |
| #free bugs, use for reference only | |
| #free bugs, use for reference only | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment