freelancer.py

# -*- coding: utf-8 -*-
import scrapy
import json


class FreelancerComRuSpider(scrapy.Spider):
    name = 'freelancer_com_ru'
    allowed_domains = ['www.freelancer.com.ru']

    iDisplayStart = 0

    def start_requests(self):
        yield scrapy.Request(
            url=f'https://www.freelancer.com.ru/ajax/table/project_contest_datatable.php?tag=&type=false&budget_min=false&budget_max=false&contest_budget_min=false&contest_budget_max=false&hourlyrate_min=false&hourlyrate_max=false&hourlyProjectDuration=2&skills_chosen=57&languages=en%2Cru&status=open&vicinity=false&countries=false&lat=false&lon=false&iDisplayStart={self.iDisplayStart}&iDisplayLength=50&iSortingCols=1&iSortCol_0=9&sSortDir_0=desc&format_version=3&',
            callback=self.parse
        )

    def parse(self, response):
        json_response = json.loads(response.body)
        projects = json_response.get('aaData')
        for project in projects:
            yield {
                'project_name': project.get('project_name'),
                'seo_url': project.get('seo_url'),
                'description': project.get('project_desc'),
                'bid_average': project.get('bid_avg'),
                'bid_count': project.get('bid_count'),
                'project_id': project.get('project_id'),
                'min_budget': project.get('minbudget'),
                'max_budget': project.get('maxbudget')
            }
        
        if self.iDisplayStart <= 1850:
            self.iDisplayStart += 50
            yield scrapy.Request(
                url=f'https://www.freelancer.com.ru/ajax/table/project_contest_datatable.php?tag=&type=false&budget_min=false&budget_max=false&contest_budget_min=false&contest_budget_max=false&hourlyrate_min=false&hourlyrate_max=false&hourlyProjectDuration=2&skills_chosen=57&languages=en%2Cru&status=open&vicinity=false&countries=false&lat=false&lon=false&iDisplayStart={self.iDisplayStart}&iDisplayLength=50&iSortingCols=1&iSortCol_0=9&sSortDir_0=desc&format_version=3&',
                callback=self.parse
            )