Skip to content

Instantly share code, notes, and snippets.

@0x1F602

0x1F602/amazon_scraper.js

Created April 19, 2018 23:38
Show Gist options
  • Star 1 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save 0x1F602/d3e4bbca82a0688f76c98ff037727fe3 to your computer and use it in GitHub Desktop.
Save 0x1F602/d3e4bbca82a0688f76c98ff037727fe3 to your computer and use it in GitHub Desktop.
Download ZIP
amzn - ebay arbys
Raw
amazon_scraper.js
const Xray = require('x-ray');
const xray = Xray();
const Promise = require('promise');
const request = require('request');
const numeral = require('numeral');
function scrape_best_sellers(language, best_seller_url) {
return new Promise(function (resolve, reject) {
var web_url_base = 'https://www.amazon.' + language;
xray(web_url_base + best_seller_url, {
products: xray('div.zg_itemWrapper', [{
product_name: 'div.a-section.a-spacing-none.p13n-asin div.p13n-sc-truncated-hyphen',
product_url: 'div.a-section.a-spacing-none.p13n-asin a.a-link-normal@href'
}])
})(function(err, data) {
if (err) reject(err);
console.log(err)
//console.log(data)
data.products.forEach(function (ii) {
ii.product_name = ii.product_name.replace(/[\$\t\n]*/g, '').trim();
ii.product_url = ii.product_url.replace(web_url_base, '');
ii.product_url = ii.product_url.replace(/[\$\t\n]*/g, '').trim();
//console.log("replacing");
//console.log(ii);
});
var data_to_return = {
datetime: new Date().toLocaleDateString('da'),
lang: [language],
products: data.products
};
return resolve(data_to_return);
})
})
}
function sortByPrice(a, b) {
if (a.price_eur < b.price_eur) return -1;
if (a.price_eur > b.price_eur) return 1;
return 0;
}
module.exports = {
scrape_best_sellers: scrape_best_sellers
};
Raw
arbitrage.js
'use strict';
const sqlite3 = require('sqlite3').verbose();
const util = require('util');
// This program detects possible price arbitrages
// open the database
let db = new sqlite3.Database('MMdb.db');
let sql = `select distinct
amazon.url as amazon_url,
amazon.description,
ebay.price - amazon.price as price_diff,
ebay.url as ebay_url
from amazon
left join ebay using (amazon_id)
where ebay.price - amazon.price > 10
and ebay.selling_state LIKE 'EndedWithSales'
`;
db.each(sql, [], (err, row) => {
if (err) {
throw err;
}
var output_string_format = "%s\t$%s\n%s\n%s\n\n";
var price = row.price_diff.toFixed(2);
console.log(util.format(output_string_format, row.description, price, "https://www.amazon.com" + row.amazon_url, row.ebay_url));
});
// close the database connection
db.close();
Raw
example output
Sennheiser HD280PRO Headphone (new model) $40.04 https://www.amazon.com/Sennheiser-HD280PRO-Headphone-new-model/dp/B00IT0IHOY/ref=sr_1_4?ie=UTF8&qid=1508111309&sr=8-4&keywords=sennheiser+headphone http://www.ebay.com/itm/Sennheiser-HD280PRO-Headphone-new-model-/182790745400
HTC VIVE - Virtual Reality System $51.00 https://www.amazon.com/HTC-VIVE-Virtual-Reality-System-pc/dp/B00VF5NT4I/ref=sr_1_1?ie=UTF8&qid=1508104989&sr=8-1&keywords=htc+vive http://www.ebay.com/itm/HTC-VIVE-VR-Gaming-Headset-Virtual-Reality-System-Brand-New-and-Sealed-Box-/122793560321
HTC VIVE - Virtual Reality System $243.95 https://www.amazon.com/HTC-VIVE-Virtual-Reality-System-pc/dp/B00VF5NT4I/ref=sr_1_1?ie=UTF8&qid=1508104989&sr=8-1&keywords=htc+vive http://www.ebay.com/itm/HTC-VIVE-Virtual-Reality-System-/122807709183
HTC VIVE - Virtual Reality System $142.83 https://www.amazon.com/HTC-VIVE-Virtual-Reality-System-pc/dp/B00VF5NT4I/ref=sr_1_1?ie=UTF8&qid=1508104989&sr=8-1&keywords=htc+vive http://www.ebay.com/itm/NEW-HTC-Vive-VR-Headset-Virtual-Reality-System-/152724394186
HTC VIVE - Virtual Reality System $101.00 https://www.amazon.com/HTC-VIVE-Virtual-Reality-System-pc/dp/B00VF5NT4I/ref=sr_1_1?ie=UTF8&qid=1508104989&sr=8-1&keywords=htc+vive http://www.ebay.com/itm/HTC-VIVE-Business-Edition-Virtual-Reality-System-/172972251625
HTC VIVE - Virtual Reality System $592.00 https://www.amazon.com/HTC-VIVE-Virtual-Reality-System-pc/dp/B00VF5NT4I/ref=sr_1_1?ie=UTF8&qid=1508104989&sr=8-1&keywords=htc+vive http://www.ebay.com/itm/HTC-VIVE-VR-Virtual-Reality-System-Headset-CybertronPC-Palladium-1070X-VR-R-/132393657533
Brother PE770 5x7 inch Embroidery machine with built-in memory, USB port, 6 lettering fonts and 136 built-in designs $134.31 https://www.amazon.com/gp/product/B002MQI2NM/ref=s9_acsd_top_hd_bw_bs7eB_c_x_2_w?pf_rd_m=ATVPDKIKX0DER&pf_rd_s=merchandised-search-3&pf_rd_r=37JVMKTX2G2TFNY7YBG0&pf_rd_t=101&pf_rd_p=fc16d61e-ad04-5238-a7c6-1cfc6cd183d7&pf_rd_i=12899111 http://www.ebay.com/itm/Brother-PE770-5x7-inch-Embroidery-machine-built-in-memory-USB-port-6-font-/311955238981
Brother PE770 5x7 inch Embroidery machine with built-in memory, USB port, 6 lettering fonts and 136 built-in designs $95.48 https://www.amazon.com/gp/product/B002MQI2NM/ref=s9_acsd_top_hd_bw_bs7eB_c_x_2_w?pf_rd_m=ATVPDKIKX0DER&pf_rd_s=merchandised-search-3&pf_rd_r=37JVMKTX2G2TFNY7YBG0&pf_rd_t=101&pf_rd_p=fc16d61e-ad04-5238-a7c6-1cfc6cd183d7&pf_rd_i=12899111 http://www.ebay.com/itm/Brother-PE770-5x7-inch-Embroidery-only-machine-built-in-memory-USB-port-6-/142087981168
Brother PE770 5x7 inch Embroidery machine with built-in memory, USB port, 6 lettering fonts and 136 built-in designs $225.00 https://www.amazon.com/gp/product/B002MQI2NM/ref=s9_acsd_top_hd_bw_bs7eB_c_x_2_w?pf_rd_m=ATVPDKIKX0DER&pf_rd_s=merchandised-search-3&pf_rd_r=37JVMKTX2G2TFNY7YBG0&pf_rd_t=101&pf_rd_p=fc16d61e-ad04-5238-a7c6-1cfc6cd183d7&pf_rd_i=12899111 http://www.ebay.com/itm/PE770-5x7-Inch-Embroidery-Machine-Built-in-Memory-USB-Port-Lettering-And-/322569805048
Echo Show - Black $120.00 https://www.amazon.com/Amazon-Echo-Show-Alexa-Enabled-Black/dp/B01J24C0TI/ref=zg_bs_electronics_7/137-6903280-2367816?_encoding=UTF8&psc=1&refRID=BN5WMKYZQWAKGR5SP1DS http://www.ebay.com/itm/Amazon-Echo-Show-Black-Latest-Touchscreen-Video-version-factory-b-S21012359-/311999396178
Kindle Paperwhite E-reader - Black, 6" High-Resolution Display (300 ppi) with Built-in Light, Wi-Fi - Includes Special Offers $31.73 https://www.amazon.com/Amazon-Kindle-Paperwhite-6-Inch-4GB-eReader/dp/B00OQVZDJM/ref=zg_bs_electronics_12/137-6903280-2367816?_encoding=UTF8&psc=1&refRID=BN5WMKYZQWAKGR5SP1DS http://www.ebay.com/itm/Kindle-Paperwhite-E-Reader-Black-6-High-Resolution-Display-300-Ppi-Bu-/253240590915
Kindle Paperwhite E-reader - Black, 6" High-Resolution Display (300 ppi) with Built-in Light, Wi-Fi - Includes Special Offers $80.01 https://www.amazon.com/Amazon-Kindle-Paperwhite-6-Inch-4GB-eReader/dp/B00OQVZDJM/ref=zg_bs_electronics_12/137-6903280-2367816?_encoding=UTF8&psc=1&refRID=BN5WMKYZQWAKGR5SP1DS http://www.ebay.com/itm/Kindle-Paperwhite-E-reader-Black-6-High-Resolution-Display-300-ppi-/263261648517
Kindle Paperwhite E-reader - Black, 6" High-Resolution Display (300 ppi) with Built-in Light, Wi-Fi - Includes Special Offers $35.96 https://www.amazon.com/Amazon-Kindle-Paperwhite-6-Inch-4GB-eReader/dp/B00OQVZDJM/ref=zg_bs_electronics_12/137-6903280-2367816?_encoding=UTF8&psc=1&refRID=BN5WMKYZQWAKGR5SP1DS http://www.ebay.com/itm/Kindle-Paperwhite-E-reader-Black-6-High-Resolution-Display-300-ppi-/322807015853
Kindle Paperwhite E-reader - Black, 6" High-Resolution Display (300 ppi) with Built-in Light, Wi-Fi - Includes Special Offers $31.20 https://www.amazon.com/Amazon-Kindle-Paperwhite-6-Inch-4GB-eReader/dp/B00OQVZDJM/ref=zg_bs_electronics_12/137-6903280-2367816?_encoding=UTF8&psc=1&refRID=BN5WMKYZQWAKGR5SP1DS http://www.ebay.com/itm/Kindle-Paperwhite-E-reader-Black-6-High-Resolution-Display-300-ppi-Bu-/302171435738
Kindle Paperwhite E-reader - Black, 6" High-Resolution Display (300 ppi) with Built-in Light, Wi-Fi - Includes Special Offers $33.66 https://www.amazon.com/Amazon-Kindle-Paperwhite-6-Inch-4GB-eReader/dp/B00OQVZDJM/ref=zg_bs_electronics_12/137-6903280-2367816?_encoding=UTF8&psc=1&refRID=BN5WMKYZQWAKGR5SP1DS http://www.ebay.com/itm/Kindle-Paperwhite-E-reader-Black-6-High-Resolution-Display-300-ppi-NO-TAX-/162624785166
Fire TV Stick with Alexa Voice Remote | Streaming Media Player $35.00 https://www.amazon.com/Amazon-Fire-TV-Stick-With-Alexa-Voice-Remote-Streaming-Media-Player/dp/B00ZV9RDKK/ref=zg_bs_electronics_1/137-6903280-2367816?_encoding=UTF8&psc=1&refRID=BN5WMKYZQWAKGR5SP1DS http://www.ebay.com/itm/Amazon-Fire-TV-Stick-Alexa-Voice-Remote-Streaming-Media-Player-Gen-2-/272599614449
Fire TV Stick with Alexa Voice Remote | Streaming Media Player $40.00 https://www.amazon.com/Amazon-Fire-TV-Stick-With-Alexa-Voice-Remote-Streaming-Media-Player/dp/B00ZV9RDKK/ref=zg_bs_electronics_1/137-6903280-2367816?_encoding=UTF8&psc=1&refRID=BN5WMKYZQWAKGR5SP1DS http://www.ebay.com/itm/Brand-New-Fire-TV-Stick-Alexa-Voice-Remote-Streaming-Media-Player-/112628500192
Fire TV Stick with Alexa Voice Remote | Streaming Media Player $35.00 https://www.amazon.com/Amazon-Fire-TV-Stick-With-Alexa-Voice-Remote-Streaming-Media-Player/dp/B00ZV9RDKK/ref=zg_bs_electronics_1/137-6903280-2367816?_encoding=UTF8&psc=1&refRID=BN5WMKYZQWAKGR5SP1DS http://www.ebay.com/itm/Fire-TV-Stick-Alexa-Voice-Remote-Streaming-Media-Player-/282719469535?var=581971256036
Fujifilm INSTAX Mini Instant Film Twin Pack (White) $30.51 https://www.amazon.com/Fujifilm-INSTAX-Instant-Twin-Pack/dp/B00EB4ADQW/ref=zg_bs_electronics_3/137-6903280-2367816?_encoding=UTF8&psc=1&refRID=BN5WMKYZQWAKGR5SP1DS http://www.ebay.com/itm/2-Days-Delivery-Fujifilm-INSTAX-Mini-Instant-Film-Twin-Pack-White-/282724533347
Fujifilm INSTAX Mini Instant Film Twin Pack (White) $32.51 https://www.amazon.com/Fujifilm-INSTAX-Instant-Twin-Pack/dp/B00EB4ADQW/ref=zg_bs_electronics_3/137-6903280-2367816?_encoding=UTF8&psc=1&refRID=BN5WMKYZQWAKGR5SP1DS http://www.ebay.com/itm/FujiFilm-Instax-Mini-25-Instant-Camera-2x-Film-Twin-Packs-40-Exposures-/282616613962
Fujifilm INSTAX Mini Instant Film Twin Pack (White) $72.46 https://www.amazon.com/Fujifilm-INSTAX-Instant-Twin-Pack/dp/B00EB4ADQW/ref=zg_bs_electronics_3/137-6903280-2367816?_encoding=UTF8&psc=1&refRID=BN5WMKYZQWAKGR5SP1DS http://www.ebay.com/itm/FujiFilm-Instax-Mini-25-Instant-Camera-2x-Film-Twin-Packs-40-Exposures-/292009231107
Fujifilm INSTAX Mini Instant Film Twin Pack (White) $92.50 https://www.amazon.com/Fujifilm-INSTAX-Instant-Twin-Pack/dp/B00EB4ADQW/ref=zg_bs_electronics_3/137-6903280-2367816?_encoding=UTF8&psc=1&refRID=BN5WMKYZQWAKGR5SP1DS http://www.ebay.com/itm/Fujifilm-Instax-Mini-70-White-Instant-Film-Camera-Blue-w-Twin-Pack-Film-/132311205577
Fujifilm INSTAX Mini Instant Film Twin Pack (White) $49.83 https://www.amazon.com/Fujifilm-INSTAX-Instant-Twin-Pack/dp/B00EB4ADQW/ref=zg_bs_electronics_3/137-6903280-2367816?_encoding=UTF8&psc=1&refRID=BN5WMKYZQWAKGR5SP1DS http://www.ebay.com/itm/Fujifilm-Instax-Mini-70-White-Instant-Film-Camera-Blue-w-Twin-Pack-Film-/292321937853
Fujifilm INSTAX Mini Instant Film Twin Pack (White) $50.50 https://www.amazon.com/Fujifilm-INSTAX-Instant-Twin-Pack/dp/B00EB4ADQW/ref=zg_bs_electronics_3/137-6903280-2367816?_encoding=UTF8&psc=1&refRID=BN5WMKYZQWAKGR5SP1DS http://www.ebay.com/itm/Fujifilm-Instax-Mini-8-9-70-90-Film-Camera-Sheets-Fuji-Instant-White-Photos-/122795055427?var=423229223680
Fujifilm INSTAX Mini Instant Film Twin Pack (White) $53.50 https://www.amazon.com/Fujifilm-INSTAX-Instant-Twin-Pack/dp/B00EB4ADQW/ref=zg_bs_electronics_3/137-6903280-2367816?_encoding=UTF8&psc=1&refRID=BN5WMKYZQWAKGR5SP1DS http://www.ebay.com/itm/Fujifilm-Instax-Mini-8-9-Camera-Film-Sheets-Fuji-Instant-White-Frame-Photos-/122791011939?var=423222881821
Fujifilm INSTAX Mini Instant Film Twin Pack (White) $101.25 https://www.amazon.com/Fujifilm-INSTAX-Instant-Twin-Pack/dp/B00EB4ADQW/ref=zg_bs_electronics_3/137-6903280-2367816?_encoding=UTF8&psc=1&refRID=BN5WMKYZQWAKGR5SP1DS http://www.ebay.com/itm/Fujifilm-Instax-Mini-9-Instant-Polaroid-Camera-Case-Film-Twin-Pack-White-/302396880054
Fujifilm INSTAX Mini Instant Film Twin Pack (White) $81.42 https://www.amazon.com/Fujifilm-INSTAX-Instant-Twin-Pack/dp/B00EB4ADQW/ref=zg_bs_electronics_3/137-6903280-2367816?_encoding=UTF8&psc=1&refRID=BN5WMKYZQWAKGR5SP1DS http://www.ebay.com/itm/Fujifilm-Instax-Mini-9-Instant-Polaroid-Camera-Film-Twin-Pack-Smokey-White-/302396881838
All-New Fire 7 Tablet with Alexa, 7" Display, 8 GB, Black - with Special Offers $31.99 https://www.amazon.com/All-New-Amazon-Fire-7-Tablet/dp/B01GEW27DA/ref=zg_bs_electronics_8/137-6903280-2367816?_encoding=UTF8&psc=1&refRID=BN5WMKYZQWAKGR5SP1DS http://www.ebay.com/itm/All-New-Fire-7-Tablet-Alexa-7-Display-8-GB-Black-Special-Offers-/142447357160
All-New Fire HD 8 Tablet with Alexa, 8" HD Display, 16 GB, Black - with Special Offers $41.00 https://www.amazon.com/All-New-Amazon-Fire-HD-8-Tablet-With-Alexa/dp/B01J94SWWU/ref=zg_bs_electronics_14/137-6903280-2367816?_encoding=UTF8&psc=1&refRID=BN5WMKYZQWAKGR5SP1DS http://www.ebay.com/itm/All-New-Fire-HD-8-Tablet-Alexa-8-HD-Display-16-GB-BLACK-/172890840715
All-New Fire HD 8 Tablet with Alexa, 8" HD Display, 16 GB, Black - with Special Offers $30.01 https://www.amazon.com/All-New-Amazon-Fire-HD-8-Tablet-With-Alexa/dp/B01J94SWWU/ref=zg_bs_electronics_14/137-6903280-2367816?_encoding=UTF8&psc=1&refRID=BN5WMKYZQWAKGR5SP1DS http://www.ebay.com/itm/All-New-Fire-HD-8-Tablet-Alexa-8-HD-Display-16-GB-Black-/202064905694
All-New Fire HD 8 Tablet with Alexa, 8" HD Display, 16 GB, Black - with Special Offers $80.00 https://www.amazon.com/All-New-Amazon-Fire-HD-8-Tablet-With-Alexa/dp/B01J94SWWU/ref=zg_bs_electronics_14/137-6903280-2367816?_encoding=UTF8&psc=1&refRID=BN5WMKYZQWAKGR5SP1DS http://www.ebay.com/itm/All-New-Fire-HD-8-Tablet-Alexa-8-HD-Display-16-GB-Black-Special-Offer-/162683316887
Raw
index.js
const sqlite3 = require('sqlite3').verbose();
const Ebay = require('ebay')
const AmazonScraper = require('amazon-scraper')
const config = require('./config.json')
const BestSellers = require('./amazon_scraper.js');
var ebay = new Ebay({
// get this from eBay's DevZone. Use prod API keys
app_id: config.ebay_prod_api_key
})
process.on('uncaughtException', function (exception) {
console.log(exception); // to see your exception details in the console
// if you are on production, maybe you can send the exception details to your
// email as well ?
});
//console.log(config)
//var number_of_amazon_products = config.amazon_products.length;
[
'/Best-Sellers-Home-Improvement-Air-Powered-Tools/zgbs/hi/552684/ref=zg_bs_nav_hi_1_hi',
'/gp/bestsellers/electronics/ref=sv_e_1',
'/Best-Sellers-Industrial-Scientific/zgbs/industrial/ref=zg_bs_nav_0'
].forEach(function(best_seller_link) {
BestSellers.scrape_best_sellers('com', best_seller_link).then(function (scrape_results) {
var amazon_products_list = scrape_results.products;
console.log(scrape_results);
console.log("amazon_products_list");
console.log(amazon_products_list);
var number_of_amazon_products = amazon_products_list.length;
for (var ii = 0; ii < number_of_amazon_products; ii++) {
try {
var amazon_product = amazon_products_list[ii];
console.log(amazon_product.product_url);
var amazon_scraper = AmazonScraper();
amazon_scraper.scraper('com', amazon_product.product_url).then(function(data) {
console.log('amazon_scraper data');
console.log(data);
var db = new sqlite3.Database('MMdb.db', sqlite3.OPEN_READWRITE, (err) => {
if (err) {
console("Had error");
console.error(err);
}
console.log('Connected to the MoneyMaker database.');
});
db.on("error", function(error) {
console.log("Getting an error : ", error);
});
db.configure("busyTimeout", 10000);
//amazon_scraper.printTable(data)
var amazon_price = data.price
//console.log("Current Amazon Price")
// if the amazon price has a period in it, don't divide by 100
var amazon_price_has_period = amazon_price.match(/\./)
var amazon_divisor = 1;
//console.log(amazon_price_has_period)
if (amazon_price_has_period == null) {
amazon_divisor = 100
}
amazon_price = amazon_price.replace(/[$$\s\t\n]*/g, '').trim() / amazon_divisor
//console.log(amazon_price);
var params = {
'OPERATION-NAME': 'findCompletedItems',
//'OPERATION-NAME': 'findItemsByKeywords',
'keywords': data.name.substring(0,64)
};
console.log("eBay finding API search keyword: " + data.name.substring(0,32));
var ins = "INSERT INTO amazon(url, description, price, asin) VALUES(?,?,?,?)";
//var amazon_url = config.amazon_products[ii].product_url;
var amazon_url = data.product_url;
console.log("Inserting now!!!");
//var amazon_url = config.amazon_products[ii].product_url;
db.run(ins, [amazon_url, data.name, amazon_price, data.asin], function (err) {
if (err) {
console.log(err);
}
console.log(this);
var amazon_row = this;
ebay.get('finding', params, function (err, data) {
console.log("Running ebay finding API");
if (err) {
console.log(err);
throw err;
}
////console.log(data.findItemsByKeywordsResponse[0].errorMessage[0].error[0])
//console.log("Top eBay Price")
//console.log(data.findItemsByKeywordsResponse[0].searchResult[0].item[0])
var searchResult = data.findCompletedItemsResponse[0].searchResult[0]
console.log(params.keywords);
console.log(data.findCompletedItemsResponse[0]);
var jj = searchResult.item.length;
for (var ii = 0; ii < jj; ii++) {
var item = searchResult.item[ii];
//console.log("\t" + item.viewItemURL[0])
console.log("\t" + item.title[0])
console.log(item.sellingStatus[0].sellingState[0])
var ebay_selling_state = item.sellingStatus[0].sellingState[0];
var ebay_price = item.sellingStatus[0].currentPrice[0].__value__;
if (ebay_price > amazon_price) {
//console.log("[ARB] ebay_price " + ebay_price + " > " + amazon_price)
}
else {
//console.log("[BAD] ebay_price " + ebay_price + " <= " + amazon_price)
}
//console.log("\n")
ins = "INSERT INTO ebay (url, description, price, selling_state, amazon_id) VALUES (?, ?, ?, ?, ?)";
db.run(ins, [item.viewItemURL[0], item.title[0], ebay_price, ebay_selling_state, amazon_row.lastID]);
}
db.close();
})
});
})
} catch (err) {
console.log(err.stack);
}
}
});
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment