Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
# -*- coding: utf-8 -*-
import scrapy
import json
import requests
import re
from time import sleep
import sys
class LetgoSpider(scrapy.Spider):
name = 'letgo'
allowed_domains = ['letgo.com/en', 'search-products-pwa.letgo.com']
start_urls = ['https://search-products-pwa.letgo.com/api/products?country_code=US&offset=0&quadkey=0320030123201&num_results=50&distance_radius=50&distance_type=mi']
offset = 0
def parse(self, response):
data = json.loads(response.text)
if len(data) == 0:
sys.exit()
else:
for used_item in data:
try:
if used_item['name'] == None:
title = used_item['image_information']
else:
title = used_item['name']
id_number = used_item['id']
price = used_item['price']
description = used_item['description']
date = used_item['updated_at']
images = [img['url'] for img in used_item['images']]
latitude = used_item['geo']['lat']
longitude = used_item['geo']['lng']
link = 'https://us.letgo.com/en/i/' + re.sub(r'\W+', '-', title) + '_' + id_number
location = used_item['geo']['city']
except:
pass
yield {'Title': title,
'Url': link,
'Price': price,
'Description': description,
'Date': date,
'Images': images,
'Latitude': latitude,
'Longitude': longitude,
'Location': location,
}
self.offset += 50
new_request = 'https://search-products-pwa.letgo.com/api/products?country_code=US&offset=' + str(self.offset) + \
'&quadkey=0320030123201&num_results=50&distance_radius=50&distance_type=mi'
print('new request is: ' + new_request)
sleep(0.5)
yield scrapy.Request(new_request, callback=self.parse)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.