Skip to content

Instantly share code, notes, and snippets.

@scrapehero
Last active August 14, 2022 03:03
Show Gist options
  • Save scrapehero/0b8b4aeea00ff3abf3bc72a9e9d26849 to your computer and use it in GitHub Desktop.
Save scrapehero/0b8b4aeea00ff3abf3bc72a9e9d26849 to your computer and use it in GitHub Desktop.
from lxml import html
import csv
import os
import requests
from exceptions import ValueError
from time import sleep
from random import randint
def parse(url):
headers = {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.90 Safari/537.36'
}
try:
# Retrying for failed requests
for i in range(20):
# Generating random delays
sleep(randint(1,3))
# Adding verify=False to avold ssl related issues
response = requests.get(url, headers=headers, verify=False)
if response.status_code == 200:
doc = html.fromstring(response.content)
XPATH_NAME = '//h1[@id="title"]//text()'
XPATH_SALE_PRICE = '//span[contains(@id,"ourprice") or contains(@id,"saleprice")]/text()'
XPATH_ORIGINAL_PRICE = '//td[contains(text(),"List Price") or contains(text(),"M.R.P") or contains(text(),"Price")]/following-sibling::td/text()'
XPATH_CATEGORY = '//a[@class="a-link-normal a-color-tertiary"]//text()'
XPATH_AVAILABILITY = '//div[@id="availability"]//text()'
RAW_NAME = doc.xpath(XPATH_NAME)
RAW_SALE_PRICE = doc.xpath(XPATH_SALE_PRICE)
RAW_CATEGORY = doc.xpath(XPATH_CATEGORY)
RAW_ORIGINAL_PRICE = doc.xpath(XPATH_ORIGINAL_PRICE)
RAw_AVAILABILITY = doc.xpath(XPATH_AVAILABILITY)
NAME = ' '.join(''.join(RAW_NAME).split()) if RAW_NAME else None
SALE_PRICE = ' '.join(''.join(RAW_SALE_PRICE).split()).strip() if RAW_SALE_PRICE else None
CATEGORY = ' > '.join([i.strip() for i in RAW_CATEGORY]) if RAW_CATEGORY else None
ORIGINAL_PRICE = ''.join(RAW_ORIGINAL_PRICE).strip() if RAW_ORIGINAL_PRICE else None
AVAILABILITY = ''.join(RAw_AVAILABILITY).strip() if RAw_AVAILABILITY else None
if not ORIGINAL_PRICE:
ORIGINAL_PRICE = SALE_PRICE
# retrying in case of captcha
if not NAME:
raise ValueError('captcha')
data = {
'NAME': NAME,
'SALE_PRICE': SALE_PRICE,
'CATEGORY': CATEGORY,
'ORIGINAL_PRICE': ORIGINAL_PRICE,
'AVAILABILITY': AVAILABILITY,
'URL': url,
}
return data
elif response.status_code==404:
break
except Exception as e:
print e
def ReadAsin():
# AsinList = csv.DictReader(open(os.path.join(os.path.dirname(__file__),"Asinfeed.csv")))
AsinList = ['B075QLRSPK',
'B0731KJVGG',
'B075KQ622T',
'B071YC3G5V',
'B01MDMJGYT',
'B071VZKDDP',
'B075FLG6MV',
'B075TZQRHN',
'B01MQUB0QN',
'B075XKFMQC' ]
extracted_data = []
for i in AsinList:
url = "http://www.amazon.com/dp/" + i
print "Processing: " + url
# Calling the parser
parsed_data = parse(url)
if parsed_data:
extracted_data.append(parsed_data)
# Writing scraped data to csv file
with open('scraped_data.csv', 'w') as csvfile:
fieldnames = ['NAME','SALE_PRICE','CATEGORY','ORIGINAL_PRICE','AVAILABILITY','URL']
writer = csv.DictWriter(csvfile, fieldnames=fieldnames, quoting=csv.QUOTE_ALL)
writer.writeheader()
for data in extracted_data:
writer.writerow(data)
if __name__ == "__main__":
ReadAsin()
@Raiden911
Copy link

capture
Getting this error when ran?

@liamgens
Copy link

@Raiden911 You're probably using python3 because print has actually become a function in 3 and requires parenthesis. I've updated the code to python3, here you go.

from lxml import html
import csv,os,json
import requests
from time import sleep

def parse(url):
    headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.90 Safari/537.36'}
    page = requests.get(url,headers=headers)
    for i in range(20):
        sleep(3)
        try:
            doc = html.fromstring(page.content)
            XPATH_NAME = '//h1[@id="title"]//text()'
            XPATH_SALE_PRICE = '//span[contains(@id,"ourprice") or contains(@id,"saleprice")]/text()'
            XPATH_ORIGINAL_PRICE = '//td[contains(text(),"List Price") or contains(text(),"M.R.P") or contains(text(),"Price")]/following-sibling::td/text()'
            XPATH_CATEGORY = '//a[@class="a-link-normal a-color-tertiary"]//text()'
            XPATH_AVAILABILITY = '//div[@id="availability"]//text()'

            RAW_NAME = doc.xpath(XPATH_NAME)
            RAW_SALE_PRICE = doc.xpath(XPATH_SALE_PRICE)
            RAW_CATEGORY = doc.xpath(XPATH_CATEGORY)
            RAW_ORIGINAL_PRICE = doc.xpath(XPATH_ORIGINAL_PRICE)
            RAw_AVAILABILITY = doc.xpath(XPATH_AVAILABILITY)

            NAME = ' '.join(''.join(RAW_NAME).split()) if RAW_NAME else None
            SALE_PRICE = ' '.join(''.join(RAW_SALE_PRICE).split()).strip() if RAW_SALE_PRICE else None
            CATEGORY = ' > '.join([i.strip() for i in RAW_CATEGORY]) if RAW_CATEGORY else None
            ORIGINAL_PRICE = ''.join(RAW_ORIGINAL_PRICE).strip() if RAW_ORIGINAL_PRICE else None
            AVAILABILITY = ''.join(RAw_AVAILABILITY).strip() if RAw_AVAILABILITY else None

            if not ORIGINAL_PRICE:
                ORIGINAL_PRICE = SALE_PRICE
            #retrying in case of caotcha
            if not NAME :
                raise ValueError('captcha')

            data = {
                    'NAME':NAME,
                    'SALE_PRICE':SALE_PRICE,
                    'CATEGORY':CATEGORY,
                    'ORIGINAL_PRICE':ORIGINAL_PRICE,
                    'AVAILABILITY':AVAILABILITY,
                    'URL':url,
                    }

            return data
        except Exception as e:
            print (e)

def ReadAsin():
    # AsinList = csv.DictReader(open(os.path.join(os.path.dirname(__file__),"Asinfeed.csv")))
    AsinList = ['B0046UR4F4',
    'B00UZKG8QU',]
    extracted_data = []
    for i in AsinList:
        url = "http://www.amazon.com/dp/"+i
        print ("Processing: "+url)
        extracted_data.append(parse(url))
        sleep(5)
    f=open('data.json','w')
    json.dump(extracted_data,f,indent=4)


if __name__ == "__main__":
    ReadAsin()

@z3dm4n
Copy link

z3dm4n commented Jan 11, 2018

Thanks for sharing the code. I used your code above to write a simple Python script, which saves the scraped data in an JSON (or CSV) output file, reads and compares wish prices from an JSON (or CSV) input file to that file and sends an email alert if the scraped price drops to (or below) the given wish price. Here is the link to it on GitHub.

@3bner
Copy link

3bner commented Jun 18, 2018

Getting this error when run...
image
can someone hel?

@MoMoMoMoMoMoMoMoMoMoMo
Copy link

MoMoMoMoMoMoMoMoMoMoMo commented Oct 12, 2018

``
C:\WINDOWS\system32>C:\Users\snyde\Documents\amazonscraper\Amazon_Scraper\amazon_product.py
File "C:\Users\snyde\Documents\amazonscraper\Amazon_Scraper\amazon_product.py", line 52
AsinList = [''B077XM396G',
^
SyntaxError: invalid syntax

C:\WINDOWS\system32>"
The error is with one of the ASINs for a product.... Anyone have any tips to correct this?
If I were smarter I'd actually error check my code.... I see what I did now. Whoops.
Thank you for the code! Hopefully this makes my job a little bit easier.

@karthikeyan622
Copy link

How can I change this script to work for amazon.in site.

Thanks in advance.

@tchow25
Copy link

tchow25 commented Aug 15, 2019

Getting this error:

Capture

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment