Crawling web pages with scrapy

scrapy.py

import scrapy
crawled=set('https://ves.ac.in/')
class MySpider(scrapy.Spider):
    name = 'my_spider'
    allowed_domains = ['ves.ac.in']
    def start_requests(self):yield scrapy.Request("https://ves.ac.in/",self.parse)
    def parse(self, response):
        for url in response.xpath('//a/@href').extract():
            if url not in crawled:
                crawled.add(url)
                yield {'url':response.urljoin(url)}
                if url[-4:] not in [".pdf" ,".png" ,".jpg",".gif"] :
                    yield scrapy.Request(url=response.urljoin(url), callback=self.parse)