Download RoundTable mp3 files from http://english.cri.cn/4926/more/11680/more11680.htm

audio.py

# -*- coding: utf-8 -*-
import os
import scrapy
import urllib.request


class AudioSpider(scrapy.Spider):
    name = 'audio'
    allowed_domains = ['english.cri.cn']
    start_urls = ['http://english.cri.cn/4926/more/11680/more11680.htm']
    prefix_url = 'http://english.cri.cn'

    def parse(self, response):
        audio_names = response.xpath('//tr[*]/td[*]/a[contains(text(), "RoundTable")]/text()').extract()
        audio_jump_url_tails = response.xpath('//tr[*]/td[*]/a[contains(text(), "RoundTable")]/@href').extract()
        for i, j in zip(audio_names, audio_jump_url_tails):
            audio_jump_url = self.prefix_url + j
            yield scrapy.Request(audio_jump_url, callback=self.parse_jump_link, dont_filter=True, meta={'name': i})

        next_page_tail = response.xpath('//a[contains(text(), "Next")]/@href').extract_first()
        if next_page_tail:
            next_page = self.prefix_url + next_page_tail
            yield scrapy.Request(next_page, callback=self.parse)

    def parse_jump_link(self, response):
        audio_link = response.xpath('//*[@id="ccontent"]/a/@href').extract_first()
        file_name = response.meta.get('name') + '.mp3'
        if not os.path.isfile(file_name):
            urllib.request.urlretrieve(audio_link, file_name)

1. Following https://letslearnabout.net/tutorial/scrapy-tutorial/python-scrapy-tutorial-for-beginners-01-creating-your-first-spider/ to setup the env
2. replace the content of the audio.py
3. scrapy startproject audio
4. scrapy crawl audio