Create a gist now

Instantly share code, notes, and snippets.

@jgamblin /newtoday.r
Last active Apr 30, 2017

What would you like to do?
Find New Domains For Today On Cisco's Top 1 Million Lists.
library(leaflet)
library(ipapi)
#devtools::install_github("hrbrmstr/ipapi")
library(tidyverse)
# Get Todays List
if (file.exists(fn)) file.remove(fn)
temp <- tempfile()
download.file("http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip",temp)
unzip(temp, "top-1m.csv")
today <- read_csv("top-1m.csv", col_names = FALSE)
unlink(temp)
file.remove("top-1m.csv")
# Get Yesterdays List
if (file.exists(fn)) file.remove(fn)
temp <- tempfile()
#Change This Date To Yesterday.
download.file("http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m-2017-04-28.csv.zip",temp)
unzip(temp, "top-1m.csv")
yesterday <- read_csv("top-1m.csv", col_names = FALSE)
unlink(temp)
file.remove("top-1m.csv")
#Find Difrences:
sites <- dplyr::anti_join(data.frame(today), data.frame(yesterday), by = "X2", copy = TRUE_)
#Build Map:
locations <- mutate(geolocate(sites$X2))
sites <- merge(sites, locations, by.x = 0, by.y = 0)
m <- leaflet(sites) %>% addTiles('http://{s}.basemaps.cartocdn.com/light_all/{z}/{x}/{y}.png')
m %>% addMarkers(clusterOptions = markerClusterOptions(), ~lon, ~lat, popup = paste("DNS:", sites$X2, "<br>",
"RANK:", sites$X1, "<br>",
"IP:", sites$query, "<br>",
"ISP", sites$isp, "<br>",
"ASN:", sites$as))
write.csv(sites, file = "newdomains.csv")
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment