Medium Modeling News 2 - Check GDELT files exist

gistfile1.txt

!pip install gdelt
import gdelt
gd = gdelt.gdelt(version=1)

import os, datetime

os.makedirs("data",exist_ok=True)

#starting 60 days before Oct7
cur_date = datetime.datetime(2019,10,7)-datetime.timedelta(days=60)
end_date = datetime.datetime(2019,10,7)

#check if already have each day; if not, pull it
while cur_date < end_date:
  print("%s-%s-%s"%(cur_date.year, cur_date.month, cur_date.day))
  
  if not os.path.exists("data/%s-%s-%s.pkl"%(cur_date.year, cur_date.month, cur_date.day)):
  
    year = cur_date.year
    month = str(cur_date.month)
    day = str(cur_date.day)
    
    if cur_date.month < 10:
      month = "0"+month
    if cur_date.day < 10:
      day = "0"+day

    results = gd.Search(['%s %s %s'%(year, month, day)],table='gkg',coverage=True, translation=False)
    results.to_pickle("data/%s-%s-%s.pkl"%(cur_date.year, cur_date.month, cur_date.day))
  
  cur_date+=datetime.timedelta(days=1)