Skip to content

Instantly share code, notes, and snippets.

@rafikahmed
Created October 29, 2019 17:45
Show Gist options
  • Save rafikahmed/c9f0e6b813a15ad09870d80bd79cd016 to your computer and use it in GitHub Desktop.
Save rafikahmed/c9f0e6b813a15ad09870d80bd79cd016 to your computer and use it in GitHub Desktop.
# -*- coding: utf-8 -*-
import scrapy
import json
class FreelancerComRuSpider(scrapy.Spider):
name = 'freelancer_com_ru'
allowed_domains = ['www.freelancer.com.ru']
iDisplayStart = 0
def start_requests(self):
yield scrapy.Request(
url=f'https://www.freelancer.com.ru/ajax/table/project_contest_datatable.php?tag=&type=false&budget_min=false&budget_max=false&contest_budget_min=false&contest_budget_max=false&hourlyrate_min=false&hourlyrate_max=false&hourlyProjectDuration=2&skills_chosen=57&languages=en%2Cru&status=open&vicinity=false&countries=false&lat=false&lon=false&iDisplayStart={self.iDisplayStart}&iDisplayLength=50&iSortingCols=1&iSortCol_0=9&sSortDir_0=desc&format_version=3&',
callback=self.parse
)
def parse(self, response):
json_response = json.loads(response.body)
projects = json_response.get('aaData')
for project in projects:
yield {
'project_name': project.get('project_name'),
'seo_url': project.get('seo_url'),
'description': project.get('project_desc'),
'bid_average': project.get('bid_avg'),
'bid_count': project.get('bid_count'),
'project_id': project.get('project_id'),
'min_budget': project.get('minbudget'),
'max_budget': project.get('maxbudget')
}
if self.iDisplayStart <= 1850:
self.iDisplayStart += 50
yield scrapy.Request(
url=f'https://www.freelancer.com.ru/ajax/table/project_contest_datatable.php?tag=&type=false&budget_min=false&budget_max=false&contest_budget_min=false&contest_budget_max=false&hourlyrate_min=false&hourlyrate_max=false&hourlyProjectDuration=2&skills_chosen=57&languages=en%2Cru&status=open&vicinity=false&countries=false&lat=false&lon=false&iDisplayStart={self.iDisplayStart}&iDisplayLength=50&iSortingCols=1&iSortCol_0=9&sSortDir_0=desc&format_version=3&',
callback=self.parse
)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment