Find New Domains For Today On Cisco's Top 1 Million Lists.

newtoday.r

library(leaflet)
library(ipapi)
#devtools::install_github("hrbrmstr/ipapi")
library(tidyverse)
 
# Get Todays List
if (file.exists(fn)) file.remove(fn)
temp <- tempfile()
download.file("http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip",temp)
unzip(temp, "top-1m.csv")
today <- read_csv("top-1m.csv", col_names = FALSE)
unlink(temp)
file.remove("top-1m.csv")

# Get Yesterdays List
if (file.exists(fn)) file.remove(fn)
temp <- tempfile()
#Change This Date To Yesterday.
download.file("http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m-2017-04-28.csv.zip",temp)
unzip(temp, "top-1m.csv")
yesterday <- read_csv("top-1m.csv", col_names = FALSE)
unlink(temp)
file.remove("top-1m.csv")

#Find Difrences:
sites <- dplyr::anti_join(data.frame(today), data.frame(yesterday), by = "X2", copy = TRUE_)

#Build Map:
locations <- mutate(geolocate(sites$X2))
sites <- merge(sites, locations, by.x = 0, by.y = 0)
m <- leaflet(sites) %>% addTiles('http://{s}.basemaps.cartocdn.com/light_all/{z}/{x}/{y}.png') 
m %>% addMarkers(clusterOptions = markerClusterOptions(), ~lon, ~lat, popup = paste("DNS:", sites$X2, "<br>",
                           "RANK:", sites$X1, "<br>",
                           "IP:", sites$query, "<br>",
                           "ISP", sites$isp, "<br>",
                           "ASN:", sites$as))

write.csv(sites, file = "newdomains.csv")