Crawler For FBO.gov

FBOSpider.py

# -*- coding: utf-8 -*-
import datetime
import scrapy
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor

class FBOSpider(scrapy.Spider):
  name = "FBOSpider"
  start_url = "https://www.fbo.gov/index?s=opportunity&mode=list&tab=list&pageID={page_no}"

  def start_requests(self):
    for i in range(20):
      yield scrapy.Request(url=self.start_url.format(page_no=i), callback=self.parse)

  def parse(self, response):
    for opportunity in response.xpath('//*[starts-with(@id, "row_")]/td[1]/a/@href'):
      yield scrapy.Request(url="https://www.fbo.gov/index"+opportunity.extract(), callback=self.parse_opportunity)

  def parse_opportunity(self, response):
    yield {
      "response": response.xpath('//*[@id="so_formfield_dnf_class_values_procurement_notice__description_"]').extract()
    }