crawl_delay_pe.py

#first convert the dates to datetime objects
crawled["lastmod"] = pd.to_datetime(crawled["lastmod"])
crawled["date"] = pd.to_datetime(crawled["date"])

crawled["crawl_delay"] = crawled["date"] - crawled["lastmod"]