Normalize threat feed data

generateMasterList.py

#!/usr/bin/env python3

import requests
import pandas as pd

df = pd.read_csv("https://raw.githubusercontent.com/drb-ra/C2IntelFeeds/master/C2_configs/cobaltstrike.csv", usecols=["FirstSeen", "ip"])
df.drop_duplicates(subset="ip").to_csv("C2_masterlist.csv", index=False)

pullNewIps.py

#!/usr/bin/env python3

import requests
import pandas as pd
from datetime import date

new_df = pd.read_csv("https://raw.githubusercontent.com/drb-ra/C2IntelFeeds/master/C2_configs/cobaltstrike.csv", usecols=["FirstSeen", "ip"])
old_df = pd.read_csv("C2_masterlist.csv", usecols=["ip"])
new_ips = open("newIps.txt", "a")

newIPs_df = new_df[~new_df["ip"].isin(old_df["ip"])]
newIPs_list = newIPs_df["ip"].unique().tolist()
new_ips.write(f"New IPs seen for {date.today()}\n=====\n{chr(10).join(newIPs_list)}\n\n")
newIPs_df.to_csv("C2_masterlist.csv", mode="a", index=False)