Skip to content

Instantly share code, notes, and snippets.

@adrianhajdin
Created September 30, 2023 10:15
Show Gist options
  • Save adrianhajdin/686326bc20e24810128637a9053c49a0 to your computer and use it in GitHub Desktop.
Save adrianhajdin/686326bc20e24810128637a9053c49a0 to your computer and use it in GitHub Desktop.
Web Scraping Full Course 2023 | Build and Deploy eCommerce Price Tracker
import { NextResponse } from "next/server";
import { getLowestPrice, getHighestPrice, getAveragePrice, getEmailNotifType } from "@/lib/utils";
import { connectToDB } from "@/lib/mongoose";
import Product from "@/lib/models/product.model";
import { scrapeAmazonProduct } from "@/lib/scraper";
import { generateEmailBody, sendEmail } from "@/lib/nodemailer";
export const maxDuration = 300; // This function can run for a maximum of 300 seconds
export const dynamic = "force-dynamic";
export const revalidate = 0;
export async function GET(request: Request) {
try {
connectToDB();
const products = await Product.find({});
if (!products) throw new Error("No product fetched");
// ======================== 1 SCRAPE LATEST PRODUCT DETAILS & UPDATE DB
const updatedProducts = await Promise.all(
products.map(async (currentProduct) => {
// Scrape product
const scrapedProduct = await scrapeAmazonProduct(currentProduct.url);
if (!scrapedProduct) return;
const updatedPriceHistory = [
...currentProduct.priceHistory,
{
price: scrapedProduct.currentPrice,
},
];
const product = {
...scrapedProduct,
priceHistory: updatedPriceHistory,
lowestPrice: getLowestPrice(updatedPriceHistory),
highestPrice: getHighestPrice(updatedPriceHistory),
averagePrice: getAveragePrice(updatedPriceHistory),
};
// Update Products in DB
const updatedProduct = await Product.findOneAndUpdate(
{
url: product.url,
},
product
);
// ======================== 2 CHECK EACH PRODUCT'S STATUS & SEND EMAIL ACCORDINGLY
const emailNotifType = getEmailNotifType(
scrapedProduct,
currentProduct
);
if (emailNotifType && updatedProduct.users.length > 0) {
const productInfo = {
title: updatedProduct.title,
url: updatedProduct.url,
};
// Construct emailContent
const emailContent = await generateEmailBody(productInfo, emailNotifType);
// Get array of user emails
const userEmails = updatedProduct.users.map((user: any) => user.email);
// Send email notification
await sendEmail(emailContent, userEmails);
}
return updatedProduct;
})
);
return NextResponse.json({
message: "Ok",
data: updatedProducts,
});
} catch (error: any) {
throw new Error(`Failed to get all products: ${error.message}`);
}
}
export async function generateEmailBody(
product: EmailProductInfo,
type: NotificationType
) {
const THRESHOLD_PERCENTAGE = 40;
// Shorten the product title
const shortenedTitle =
product.title.length > 20
? `${product.title.substring(0, 20)}...`
: product.title;
let subject = "";
let body = "";
switch (type) {
case Notification.WELCOME:
subject = `Welcome to Price Tracking for ${shortenedTitle}`;
body = `
<div>
<h2>Welcome to PriceWise 🚀</h2>
<p>You are now tracking ${product.title}.</p>
<p>Here's an example of how you'll receive updates:</p>
<div style="border: 1px solid #ccc; padding: 10px; background-color: #f8f8f8;">
<h3>${product.title} is back in stock!</h3>
<p>We're excited to let you know that ${product.title} is now back in stock.</p>
<p>Don't miss out - <a href="${product.url}" target="_blank" rel="noopener noreferrer">buy it now</a>!</p>
<img src="https://i.ibb.co/pwFBRMC/Screenshot-2023-09-26-at-1-47-50-AM.png" alt="Product Image" style="max-width: 100%;" />
</div>
<p>Stay tuned for more updates on ${product.title} and other products you're tracking.</p>
</div>
`;
break;
case Notification.CHANGE_OF_STOCK:
subject = `${shortenedTitle} is now back in stock!`;
body = `
<div>
<h4>Hey, ${product.title} is now restocked! Grab yours before they run out again!</h4>
<p>See the product <a href="${product.url}" target="_blank" rel="noopener noreferrer">here</a>.</p>
</div>
`;
break;
case Notification.LOWEST_PRICE:
subject = `Lowest Price Alert for ${shortenedTitle}`;
body = `
<div>
<h4>Hey, ${product.title} has reached its lowest price ever!!</h4>
<p>Grab the product <a href="${product.url}" target="_blank" rel="noopener noreferrer">here</a> now.</p>
</div>
`;
break;
case Notification.THRESHOLD_MET:
subject = `Discount Alert for ${shortenedTitle}`;
body = `
<div>
<h4>Hey, ${product.title} is now available at a discount more than ${THRESHOLD_PERCENTAGE}%!</h4>
<p>Grab it right away from <a href="${product.url}" target="_blank" rel="noopener noreferrer">here</a>.</p>
</div>
`;
break;
default:
throw new Error("Invalid notification type.");
}
return { subject, body };
}
@tailwind base;
@tailwind components;
@tailwind utilities;
* {
margin: 0;
padding: 0;
box-sizing: border-box;
scroll-behavior: smooth;
}
@layer base {
body {
@apply font-inter;
}
}
@layer utilities {
.btn {
@apply py-4 px-4 bg-secondary hover:bg-opacity-70 rounded-[30px] text-white text-lg font-semibold;
}
.head-text {
@apply mt-4 text-6xl leading-[72px] font-bold tracking-[-1.2px] text-gray-900;
}
.section-text {
@apply text-secondary text-[32px] font-semibold;
}
.small-text {
@apply flex gap-2 text-sm font-medium text-primary;
}
.paragraph-text {
@apply text-xl leading-[30px] text-gray-600;
}
.hero-carousel {
@apply relative sm:px-10 py-5 sm:pt-20 pb-5 max-w-[560px] h-[700px] w-full bg-[#F2F4F7] rounded-[30px] sm:mx-auto;
}
.carousel {
@apply flex flex-col-reverse h-[700px];
}
.carousel .control-dots {
@apply static !important;
}
.carousel .control-dots .dot {
@apply w-[10px] h-[10px] bg-[#D9D9D9] rounded-full bottom-0 !important;
}
.carousel .control-dots .dot.selected {
@apply bg-[#475467] !important;
}
.trending-section {
@apply flex flex-col gap-10 px-6 md:px-20 py-24;
}
/* PRODUCT DETAILS PAGE STYLES */
.product-container {
@apply flex flex-col gap-16 flex-wrap px-6 md:px-20 py-24;
}
.product-image {
@apply flex-grow xl:max-w-[50%] max-w-full py-16 border border-[#CDDBFF] rounded-[17px];
}
.product-info {
@apply flex items-center flex-wrap gap-10 py-6 border-y border-y-[#E4E4E4];
}
.product-hearts {
@apply flex items-center gap-2 px-3 py-2 bg-[#FFF0F0] rounded-10;
}
.product-stars {
@apply flex items-center gap-2 px-3 py-2 bg-[#FBF3EA] rounded-[27px];
}
.product-reviews {
@apply flex items-center gap-2 px-3 py-2 bg-white-200 rounded-[27px];
}
/* MODAL */
.dialog-container {
@apply fixed inset-0 z-10 overflow-y-auto bg-black bg-opacity-60;
}
.dialog-content {
@apply p-6 bg-white inline-block w-full max-w-md my-8 overflow-hidden text-left align-middle transition-all transform shadow-xl rounded-2xl;
}
.dialog-head_text {
@apply text-secondary text-lg leading-[24px] font-semibold mt-4;
}
.dialog-input_container {
@apply px-5 py-3 mt-3 flex items-center gap-2 border border-gray-300 rounded-[27px];
}
.dialog-input {
@apply flex-1 pl-1 border-none text-gray-500 text-base focus:outline-none border border-gray-300 rounded-[27px] shadow-xs;
}
.dialog-btn {
@apply px-5 py-3 text-white text-base font-semibold border border-secondary bg-secondary rounded-lg mt-8;
}
/* NAVBAR */
.nav {
@apply flex justify-between items-center px-6 md:px-20 py-4;
}
.nav-logo {
@apply font-spaceGrotesk text-[21px] text-secondary font-bold;
}
/* PRICE INFO */
.price-info_card {
@apply flex-1 min-w-[200px] flex flex-col gap-2 border-l-[3px] rounded-10 bg-white-100 px-5 py-4;
}
/* PRODUCT CARD */
.product-card {
@apply sm:w-[292px] sm:max-w-[292px] w-full flex-1 flex flex-col gap-4 rounded-md;
}
.product-card_img-container {
@apply flex-1 relative flex flex-col gap-5 p-4 rounded-md;
}
.product-card_img {
@apply max-h-[250px] object-contain w-full h-full bg-transparent;
}
.product-title {
@apply text-secondary text-xl leading-6 font-semibold truncate;
}
/* SEARCHBAR INPUT */
.searchbar-input {
@apply flex-1 min-w-[200px] w-full p-3 border border-gray-300 rounded-lg shadow-xs text-base text-gray-500 focus:outline-none;
}
.searchbar-btn {
@apply bg-gray-900 border border-gray-900 rounded-lg shadow-xs px-5 py-3 text-white text-base font-semibold hover:opacity-90 disabled:pointer-events-none disabled:cursor-not-allowed disabled:opacity-40;
}
}
"use server"
import axios from 'axios';
import * as cheerio from 'cheerio';
import { extractCurrency, extractDescription, extractPrice } from '../utils';
export async function scrapeAmazonProduct(url: string) {
if(!url) return;
// BrightData proxy configuration
const username = String(process.env.BRIGHT_DATA_USERNAME);
const password = String(process.env.BRIGHT_DATA_PASSWORD);
const port = 22225;
const session_id = (1000000 * Math.random()) | 0;
const options = {
auth: {
username: `${username}-session-${session_id}`,
password,
},
host: 'brd.superproxy.io',
port,
rejectUnauthorized: false,
}
try {
// Fetch the product page
const response = await axios.get(url, options);
const $ = cheerio.load(response.data);
// Extract the product title
const title = $('#productTitle').text().trim();
const currentPrice = extractPrice(
$('.priceToPay span.a-price-whole'),
$('.a.size.base.a-color-price'),
$('.a-button-selected .a-color-base'),
);
const originalPrice = extractPrice(
$('#priceblock_ourprice'),
$('.a-price.a-text-price span.a-offscreen'),
$('#listPrice'),
$('#priceblock_dealprice'),
$('.a-size-base.a-color-price')
);
const outOfStock = $('#availability span').text().trim().toLowerCase() === 'currently unavailable';
const images =
$('#imgBlkFront').attr('data-a-dynamic-image') ||
$('#landingImage').attr('data-a-dynamic-image') ||
'{}'
const imageUrls = Object.keys(JSON.parse(images));
const currency = extractCurrency($('.a-price-symbol'))
const discountRate = $('.savingsPercentage').text().replace(/[-%]/g, "");
const description = extractDescription($)
// Construct data object with scraped information
const data = {
url,
currency: currency || '$',
image: imageUrls[0],
title,
currentPrice: Number(currentPrice) || Number(originalPrice),
originalPrice: Number(originalPrice) || Number(currentPrice),
priceHistory: [],
discountRate: Number(discountRate),
category: 'category',
reviewsCount:100,
stars: 4.5,
isOutOfStock: outOfStock,
description,
lowestPrice: Number(currentPrice) || Number(originalPrice),
highestPrice: Number(originalPrice) || Number(currentPrice),
averagePrice: Number(currentPrice) || Number(originalPrice),
}
return data;
} catch (error: any) {
console.log(error);
}
}
/** @type {import('next').NextConfig} */
const nextConfig = {
experimental: {
serverActions: true,
serverComponentsExternalPackages: ['mongoose']
},
images: {
domains: ['m.media-amazon.com']
}
}
module.exports = nextConfig
https://drive.google.com/file/d/1v6h993BgYX6axBoIXFbZ9HQAgqbR4PSH/view?usp=sharing
/** @type {import('tailwindcss').Config} */
module.exports = {
content: [
"./pages/**/*.{js,ts,jsx,tsx,mdx}",
"./components/**/*.{js,ts,jsx,tsx,mdx}",
"./app/**/*.{js,ts,jsx,tsx,mdx}",
],
theme: {
extend: {
colors: {
primary: {
DEFAULT: "#E43030",
"orange": "#D48D3B",
"green": "#3E9242"
},
secondary: "#282828",
"gray-200": "#EAECF0",
"gray-300": "D0D5DD",
"gray-500": "#667085",
"gray-600": "#475467",
"gray-700": "#344054",
"gray-900": "#101828",
"white-100": "#F4F4F4",
"white-200": "#EDF0F8",
"black-100": "#3D4258",
"neutral-black": "#23263B",
},
boxShadow: {
xs: "0px 1px 2px 0px rgba(16, 24, 40, 0.05)",
},
maxWidth: {
"10xl": '1440px'
},
fontFamily: {
inter: ['Inter', 'sans-serif'],
spaceGrotesk: ['Space Grotesk', 'sans-serif'],
},
borderRadius: {
10: "10px"
}
},
},
plugins: [],
};
export type PriceHistoryItem = {
price: number;
};
export type User = {
email: string;
};
export type Product = {
_id?: string;
url: string;
currency: string;
image: string;
title: string;
currentPrice: number;
originalPrice: number;
priceHistory: PriceHistoryItem[] | [];
highestPrice: number;
lowestPrice: number;
averagePrice: number;
discountRate: number;
description: string;
category: string;
reviewsCount: number;
stars: number;
isOutOfStock: Boolean;
users?: User[];
};
export type NotificationType =
| "WELCOME"
| "CHANGE_OF_STOCK"
| "LOWEST_PRICE"
| "THRESHOLD_MET";
export type EmailContent = {
subject: string;
body: string;
};
export type EmailProductInfo = {
title: string;
url: string;
};
import { PriceHistoryItem, Product } from "@/types";
const Notification = {
WELCOME: 'WELCOME',
CHANGE_OF_STOCK: 'CHANGE_OF_STOCK',
LOWEST_PRICE: 'LOWEST_PRICE',
THRESHOLD_MET: 'THRESHOLD_MET',
}
const THRESHOLD_PERCENTAGE = 40;
// Extracts and returns the price from a list of possible elements.
export function extractPrice(...elements: any) {
for (const element of elements) {
const priceText = element.text().trim();
if(priceText) {
const cleanPrice = priceText.replace(/[^\d.]/g, '');
let firstPrice;
if (cleanPrice) {
firstPrice = cleanPrice.match(/\d+\.\d{2}/)?.[0];
}
return firstPrice || cleanPrice;
}
}
return '';
}
// Extracts and returns the currency symbol from an element.
export function extractCurrency(element: any) {
const currencyText = element.text().trim().slice(0, 1);
return currencyText ? currencyText : "";
}
// Extracts description from two possible elements from amazon
export function extractDescription($: any) {
// these are possible elements holding description of the product
const selectors = [
".a-unordered-list .a-list-item",
".a-expander-content p",
// Add more selectors here if needed
];
for (const selector of selectors) {
const elements = $(selector);
if (elements.length > 0) {
const textContent = elements
.map((_: any, element: any) => $(element).text().trim())
.get()
.join("\n");
return textContent;
}
}
// If no matching elements were found, return an empty string
return "";
}
export function getHighestPrice(priceList: PriceHistoryItem[]) {
let highestPrice = priceList[0];
for (let i = 0; i < priceList.length; i++) {
if (priceList[i].price > highestPrice.price) {
highestPrice = priceList[i];
}
}
return highestPrice.price;
}
export function getLowestPrice(priceList: PriceHistoryItem[]) {
let lowestPrice = priceList[0];
for (let i = 0; i < priceList.length; i++) {
if (priceList[i].price < lowestPrice.price) {
lowestPrice = priceList[i];
}
}
return lowestPrice.price;
}
export function getAveragePrice(priceList: PriceHistoryItem[]) {
const sumOfPrices = priceList.reduce((acc, curr) => acc + curr.price, 0);
const averagePrice = sumOfPrices / priceList.length || 0;
return averagePrice;
}
export const getEmailNotifType = (
scrapedProduct: Product,
currentProduct: Product
) => {
const lowestPrice = getLowestPrice(currentProduct.priceHistory);
if (scrapedProduct.currentPrice < lowestPrice) {
return Notification.LOWEST_PRICE as keyof typeof Notification;
}
if (!scrapedProduct.isOutOfStock && currentProduct.isOutOfStock) {
return Notification.CHANGE_OF_STOCK as keyof typeof Notification;
}
if (scrapedProduct.discountRate >= THRESHOLD_PERCENTAGE) {
return Notification.THRESHOLD_MET as keyof typeof Notification;
}
return null;
};
export const formatNumber = (num: number = 0) => {
return num.toLocaleString(undefined, {
minimumFractionDigits: 0,
maximumFractionDigits: 0,
});
};
@ra8200
Copy link

ra8200 commented Oct 28, 2023

the favicon.icon he used in the replacement folder provided in the github gist is missing. It is no longer in the folder. Has anyone else had this problem?

Yes there isnt a favicon in the folder. just make your own

@bmartinis12
Copy link

If anyone is having trouble getting the emails to send in production change your sendEmail function to this:
`export const sendEmail = async (emailContent: EmailContent, sentTo: string[]) => {
const mailOptions = {
from: 'trackazon@outlook.com',
to: sentTo,
html: emailContent.body,
subject: emailContent.subject,
};

await transporter.sendMail(mailOptions);

return {};
};`
You need to remove the callback from the nodemailer sendEmail function otherwise it does not return a promise... this cause the script in vercel to be terminated earlier hence not email being sent!

@STAHollmichel
Copy link

favicon is missing from the G Drive public folder... :-( @adrianhajdin

@gknanhe
Copy link

gknanhe commented Nov 8, 2023

i got 401 when test cron jobs

@STAHollmichel
Copy link

STAHollmichel commented Nov 9, 2023

t's because the price appears twice within another span within the span.a-price.a-textprice. Need to find a way to ignore the second span class called aria-hidden.

Hey, hey! I had the same issue... It's funny how in the tutorial video this is not happening.

So some Amazon Pages have multiple prices:
Used price, Discounted, and then the replicated hidden prices.

I took me a whole afternoon to figure this out. This part could be more developed in the tutorial. But hey, at least I went on my own to fix it, with the help of bing chat (aka ChatGPT)!

I worked for pages without any discount and pages with discount as well as amazon.de and amazon.com

const currentPrice = extractPrice(
$('span[data-a-color=price] span.a-offscreen').first(),
$('.priceToPay span.a-price-whole').first(),
$('.a.size.base.a-color-price'),
$('.a-button-selected .a-color-base'),
);

    const originalPrice = extractPrice(
        $('span[data-a-strike=true] span.a-offscreen').first()
    );

The ".first()" method helps to remove the same price duplicates or the used price.

Also, if you need to update to Next.js version 14 comment out or remove "serverActions: true", from next.config.js

@STAHollmichel
Copy link

Does anyone know how to fix this error when trying to run the dev?

image0

I think you are not runing the project from a proper repository folder.

You should be running from your main repository location folder. Did you install .git and nextjs?

@STAHollmichel
Copy link

the favicon.icon he used in the replacement folder provided in the github gist is missing. It is no longer in the folder. Has anyone else had this problem?

Yes there isnt a favicon in the folder. just make your own

LOL don't make your own go take it from the original repo: https://github.com/adrianhajdin/pricewise/tree/main/app

@STAHollmichel
Copy link

the favicon.icon he used in the replacement folder provided in the github gist is missing. It is no longer in the folder. Has anyone else had this problem?

take it from the original repo: https://github.com/adrianhajdin/pricewise/tree/main/app

@STAHollmichel
Copy link

STAHollmichel commented Nov 9, 2023

image I'm getting started with this project and while creating the Navbar, my styles are not reflecting on website. I did copy the globals.css and tailwind-config-ts from here and I'm attaching my Navbar.tsx code too. I don't know what's wrong but the styles are not reflecting in the website.

import React from 'react'
import Link from 'next/link'
import Image from 'next/image'
const NavIcons=[
    {src:'/assets/icons/search.svg',alt:'search'},
    {src:'/assets/icons/black-heart.svg',alt:'heart'},
    {src:'/assets/icons/user.svg',alt:'user'},
]
const Navbar = () => {
  return (
    <header className='w-full'>
        <nav className="nav">
            <Link href="/" className="flex items-center gap-1">
                <Image 
                src="/assets/icons/logo.svg"
                width={27}
                height={27}
                alt="logo"
                />
                <p className="nav-logo">
                    Price<span className='text-primary'>Wise</span>
                </p>
            </Link>
            <div className='flex items-center gap-5'>
                {NavIcons.map((icon)=>(
                    <Image
                     key={icon.alt}
                     src={icon.src}
                     alt={icon.alt}
                     width={28}
                     height={28}
                     className='object-contain'
                    />
                ))}
            </div>
        </nav>
    </header>
  )
}

export default Navbar```

Also the console of the browser is showing this error
```app-index.js:31 Warning: Prop `className` did not match. Server: "__className_725fdb vsc-initialized" Client: "__className_725fdb"
    at body
    at html
    at RedirectErrorBoundary (webpack-internal:///(app-pages-browser)/./node_modules/next/dist/client/components/redirect-boundary.js:72:9)
    at RedirectBoundary (webpack-internal:///(app-pages-browser)/./node_modules/next/dist/client/components/redirect-boundary.js:80:11)
    at NotFoundErrorBoundary (webpack-internal:///(app-pages-browser)/./node_modules/next/dist/client/components/not-found-boundary.js:54:9)
    at NotFoundBoundary (webpack-internal:///(app-pages-browser)/./node_modules/next/dist/client/components/not-found-boundary.js:62:11)
    at DevRootNotFoundBoundary (webpack-internal:///(app-pages-browser)/./node_modules/next/dist/client/components/dev-root-not-found-boundary.js:32:11)
    at ReactDevOverlay (webpack-internal:///(app-pages-browser)/./node_modules/next/dist/client/components/react-dev-overlay/internal/ReactDevOverlay.js:66:9)
    at HotReload (webpack-internal:///(app-pages-browser)/./node_modules/next/dist/client/components/react-dev-overlay/hot-reloader-client.js:294:11)
    at Router (webpack-internal:///(app-pages-browser)/./node_modules/next/dist/client/components/app-router.js:157:11)
    at ErrorBoundaryHandler (webpack-internal:///(app-pages-browser)/./node_modules/next/dist/client/components/error-boundary.js:82:9)
    at ErrorBoundary (webpack-internal:///(app-pages-browser)/./node_modules/next/dist/client/components/error-boundary.js:110:11)
    at AppRouter (webpack-internal:///(app-pages-browser)/./node_modules/next/dist/client/components/app-router.js:440:13)
    at ServerRoot (webpack-internal:///(app-pages-browser)/./node_modules/next/dist/client/app-index.js:126:11)
    at RSCComponent
    at Root (webpack-internal:///(app-pages-browser)/./node_modules/next/dist/client/app-index.js:142:11)```

I think your problem lies on

"const NavIcons=["

it should be navIcons and "{navIcons.map((icon) => ("

Also, your spacing is part of the issue It should be "const navIcons = [

You must be precise with the spacing, and you need to fix your indentation.

My << className="object-contain" >> is at line 35 for instance.

@Spyware007
Copy link

MongooseServerSelectionError: Could not connect to any servers in your MongoDB Atlas cluster. One common reason is that you're trying to access the database from an IP that isn't whitelisted. Make sure your current IP address is on your Atlas cluster's IP whitelist: https://www.mongodb.com/docs/atlas/security-whitelist/
    at _handleConnectionErrors (/Users/spider/Desktop/Scrapping/node_modules/mongoose/lib/connection.js:805:11)
    at NativeConnection.openUri (/Users/spider/Desktop/Scrapping/node_modules/mongoose/lib/connection.js:780:11)
    at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
    at async connectToDB (webpack-internal:///(rsc)/./lib/mongoose.ts:14:9) {
  reason: TopologyDescription {
    type: 'ReplicaSetNoPrimary',
    servers: Map(3) {
      'cluster0-shard-00-01.h2he2.mongodb.net:27017' => [ServerDescription],
      'cluster0-shard-00-02.h2he2.mongodb.net:27017' => [ServerDescription],
      'cluster0-shard-00-00.h2he2.mongodb.net:27017' => [ServerDescription]
    },
    stale: false,
    compatible: true,
    heartbeatFrequencyMS: 10000,
    localThresholdMS: 15,
    setName: 'atlas-12oka2-shard-0',
    maxElectionId: null,
    maxSetVersion: null,
    commonWireVersion: 0,
    logicalSessionTimeoutMinutes: null
  },
  code: undefined
}

I am getting this error, Anyone please help!
I checked my .env.local variables as well as mongodb configuration still nothing!
ALthough it was working before but now without making any changes it is not working!

@Spyware007
Copy link

Spyware007 commented Nov 14, 2023

@bmartinis12 @DailyDisco Did you guys found anything on the above error?
I am also getting the same one along with the MongooseServerSelectionError

@Xeven777
Copy link

I am getting error 503 when I try to scrape a product. Anyone else dealing with this?

=> Already connected to MongoDB
  lib\actions\index.ts (50:10) @ scrapeAndStoreProductError: Failed to create/update product: Failed to scrape product: Request failed with status code 503
    at scrapeAndStoreProduct (./lib/actions/index.ts:60:15)
  48 |     revalidatePath(`/products/${newProduct._id}`);
  49 |   } catch (error: any) {
> 50 |     throw new Error(`Failed to create/update product: ${error.message}`);
     |          ^
  51 |   }
  52 | }
  53 |

Did you find a solution I am experiencing the same problem right now. It worked the first time, but now I get a 503 error.

Not yet, I will post one here if I find it, or maybe someone will post one here when they find it.

Alright thanks, he mentions in the video that it may not work during development when you are using a localhost, so I am hoping that is what is going wrong and it works in production.

i got its solution. it might be due to the imgUrls it was returning.

image: imgUrls[0]||productImg,

changing to this line helped solve the error

@Savion92
Copy link

Savion92 commented Dec 9, 2023

For those of you who've mentioned the missing fav icons, the .svg files are all available in the completed project code - pricewise public assets folder. There you'll find two folders that contain all of the svgs you'll need.

@vivek9124vivek
Copy link

cors
I am facing this issue and I am using this in India. Does it create problems due to different country? Please anybody suggest the solution, how you successfully did it?

@Kapelo256
Copy link

Help, i can not reach images in assets with extention .svg

@Kapelo256
Copy link

Capture

@MikeeBuilds
Copy link

the favicon.icon he used in the replacement folder provided in the github gist is missing. It is no longer in the folder. Has anyone else had this problem?

Yes i just made my own

@MikeeBuilds
Copy link

Screenshot 2023-12-23 at 3 44 50 PM

import axios from "axios";
import * as cheerio from "cheerio";
import { extractPrice } from "../utils";

export async function scrapeAmazonProduct(url: string) {
    if(!url) return;

    

    const username = String(process.env.BRIGHT_DATA_USERNAME);
    const password = String(process.env.BRIGHT_DATA_PASSWORD);
    const port = 2225;
    const session_id = (1000000 * Math.random()) | 0;

    const options = {
        auth: {
            username: `${username}-session${session_id}`,
            password,
        },
        host: 'brd.superproxy.io',
        port,
        rejectUnauthorized: false,
    }

    try {
        // Fetch product page
        const response = await axios.get(url, options);
        const $ = cheerio.load(response.data);

        // Get product title
        const title = $('#productTitle').text().trim();
        const currentPrice = extractPrice(
            $('.priceToPay span.a-price-whole'),
            $('a.size.base.a-color-price'),
            $('.a-button-selected .a-color-base'),
        );

        console.log({title, currentPrice});
    } catch (error: any) {
        throw new Error(`Failed to scrape product: ${error.message}`);
    }
}

Can anyone help me figure out why price price is displaying these extra numbers? I cannot figure this out

@Harshkr75
Copy link

I've created and hosted my website on vercel but the website does not always work , it works sometimes properly and becomes unresponsive sometimes, It does not show any error in the console . Even the cron job is failing as i've recieved a few mails of cron jobs failure.

@Allan2000-Git
Copy link

Screenshot 2023-12-23 at 3 44 50 PM

import axios from "axios";
import * as cheerio from "cheerio";
import { extractPrice } from "../utils";

export async function scrapeAmazonProduct(url: string) {
    if(!url) return;

    

    const username = String(process.env.BRIGHT_DATA_USERNAME);
    const password = String(process.env.BRIGHT_DATA_PASSWORD);
    const port = 2225;
    const session_id = (1000000 * Math.random()) | 0;

    const options = {
        auth: {
            username: `${username}-session${session_id}`,
            password,
        },
        host: 'brd.superproxy.io',
        port,
        rejectUnauthorized: false,
    }

    try {
        // Fetch product page
        const response = await axios.get(url, options);
        const $ = cheerio.load(response.data);

        // Get product title
        const title = $('#productTitle').text().trim();
        const currentPrice = extractPrice(
            $('.priceToPay span.a-price-whole'),
            $('a.size.base.a-color-price'),
            $('.a-button-selected .a-color-base'),
        );

        console.log({title, currentPrice});
    } catch (error: any) {
        throw new Error(`Failed to scrape product: ${error.message}`);
    }
}

Can anyone help me figure out why price price is displaying these extra numbers? I cannot figure this out

You need to parse it properly. Try the below code
export function extractPrice(...args: any){ for(const element of args){ const price = element.text().trim(); if(price){ return price.replace(/\D/g, ''); } } return ""; }

@harshbajani
Copy link

I am not able to get any mails in my inbox after submitting even after doing everything correctly

"use server";

import { EmailContent, EmailProductInfo, NotificationType } from "@/types";
import nodemailer from "nodemailer";

const Notification = {
WELCOME: "WELCOME",
CHANGE_OF_STOCK: "CHANGE_OF_STOCK",
LOWEST_PRICE: "LOWEST_PRICE",
THRESHOLD_MET: "THRESHOLD_MET",
};

export async function generateEmailBody(
product: EmailProductInfo,
type: NotificationType
) {
const THRESHOLD_PERCENTAGE = 40;
// Shorten the product title
const shortenedTitle =
product.title.length > 20
? ${product.title.substring(0, 20)}...
: product.title;

let subject = "";
let body = "";

switch (type) {
case Notification.WELCOME:
subject = Welcome to Price Tracking for ${shortenedTitle};
body = <div> <h2>Welcome to PriceWise 🚀</h2> <p>You are now tracking ${product.title}.</p> <p>Here's an example of how you'll receive updates:</p> <div style="border: 1px solid #ccc; padding: 10px; background-color: #f8f8f8;"> <h3>${product.title} is back in stock!</h3> <p>We're excited to let you know that ${product.title} is now back in stock.</p> <p>Don't miss out - <a href="${product.url}" target="_blank" rel="noopener noreferrer">buy it now</a>!</p> <img src="https://i.ibb.co/pwFBRMC/Screenshot-2023-09-26-at-1-47-50-AM.png" alt="Product Image" style="max-width: 100%;" /> </div> <p>Stay tuned for more updates on ${product.title} and other products you're tracking.</p> </div>;
break;

case Notification.CHANGE_OF_STOCK:
  subject = `${shortenedTitle} is now back in stock!`;
  body = `
    <div>
      <h4>Hey, ${product.title} is now restocked! Grab yours before they run out again!</h4>
      <p>See the product <a href="${product.url}" target="_blank" rel="noopener noreferrer">here</a>.</p>
    </div>
  `;
  break;

case Notification.LOWEST_PRICE:
  subject = `Lowest Price Alert for ${shortenedTitle}`;
  body = `
    <div>
      <h4>Hey, ${product.title} has reached its lowest price ever!!</h4>
      <p>Grab the product <a href="${product.url}" target="_blank" rel="noopener noreferrer">here</a> now.</p>
    </div>
  `;
  break;

case Notification.THRESHOLD_MET:
  subject = `Discount Alert for ${shortenedTitle}`;
  body = `
    <div>
      <h4>Hey, ${product.title} is now available at a discount more than ${THRESHOLD_PERCENTAGE}%!</h4>
      <p>Grab it right away from <a href="${product.url}" target="_blank" rel="noopener noreferrer">here</a>.</p>
    </div>
  `;
  break;

default:
  throw new Error("Invalid notification type.");

}

return { subject, body };
}

const transporter = nodemailer.createTransport({
pool: true,
service: "hotmail",
port: 2525,
auth: {
user: process.env.EMAIL,
pass: process.env.EMAIL_PASSWORD,
},
maxConnections: 1,
});

export const sendEmail = async (
emailContent: EmailContent,
sendTo: string[]
) => {
const mailOptions = {
from: "harsh.xira@outlook.com",
to: sendTo,
html: emailContent.body,
subject: emailContent.subject,
};

transporter.sendMail(mailOptions, (error: any, info: any) => {
if (error) return console.log(error);

console.log("Email sent: ", info);

});
};

@Abubaker6898
Copy link

where is a zipped folder

@tito-Saptarshi
Copy link

can only receive mails on localhost, but after deploying, it's not working

@DillBal
Copy link

DillBal commented Mar 12, 2024

I am getting error 503 when I try to scrape a product. Anyone else dealing with this?

=> Already connected to MongoDB
  lib\actions\index.ts (50:10) @ scrapeAndStoreProductError: Failed to create/update product: Failed to scrape product: Request failed with status code 503
    at scrapeAndStoreProduct (./lib/actions/index.ts:60:15)
  48 |     revalidatePath(`/products/${newProduct._id}`);
  49 |   } catch (error: any) {
> 50 |     throw new Error(`Failed to create/update product: ${error.message}`);
     |          ^
  51 |   }
  52 | }
  53 |

Did you find a solution I am experiencing the same problem right now. It worked the first time, but now I get a 503 error.

Not yet, I will post one here if I find it, or maybe someone will post one here when they find it.

Alright thanks, he mentions in the video that it may not work during development when you are using a localhost, so I am hoping that is what is going wrong and it works in production.

i got its solution. it might be due to the imgUrls it was returning.

image: imgUrls[0]||productImg,

changing to this line helped solve the error

Brother where do you change that line?

@wilsonlaww
Copy link

After deployment, how can the public open the website? I tried opening the link and it says I am not a member of the team and therefore cant view the page.

@viibhuGupta
Copy link

viibhuGupta commented May 19, 2024

Hy Guys i am facing the problem that In DB Image and Descreption is not storing other than all the details are storing

` product = {
...scrapedProduct,
priceHistory : updatePriceHistory,
lowestPrice : getLowestPrice(updatePriceHistory),
highestPrice : getHighestPrice(updatePriceHistory),
averagePrice : getAveragePrice(updatePriceHistory),

        }
        console.log(product); //  here i am getting all the details `   

**When i Console log this i get all the details about the product **

` const newProduct = await Product.findOneAndUpdate(
{ url : scrapedProduct.url},
product,
{ upsert : true , new : true}
)

       console.log(newProduct); // here i am not getting all details `   

**When i log this i am not getting getting details like Images , Descreption and users details **

This is the Db Images what i am getting

Screenshot_20240519_125924

this is code and repo link

https://github.com/viibhuGupta/eCommerce-Price-Tracker/blob/main/lib/actions/index.ts

@neelp03
Copy link

neelp03 commented May 22, 2024

I was able to extract the description and categories!

My next goal is to get all the images of the product and show it similar to how the carousel is on home page but without the autoplay
Here is my repo if anyone want to see how: https://github.com/neelp03/pricetracker

Image 1 Image 2

FYI headless UI deprecated a lot of the old components.

It easy to transition to the new version but might be a little tricky to find the correct components since a few have been renamed completely. See my repo linked above to see how to implement the track model with the new version!

@shashankxrm
Copy link

MongooseError: Operation products.find()buffering timed out after 10000ms at Timeout.<anonymous> (/vercel/path0/node_modules/mongoose/lib/drivers/node-mongodb-native/collection.js:185:23) at listOnTimeout (node:internal/timers:573:17) at process.processTimers (node:internal/timers:514:7) MongooseError: Operationproducts.find() buffering timed out after 10000ms at Timeout.<anonymous> (/vercel/path0/node_modules/mongoose/lib/drivers/node-mongodb-native/collection.js:185:23) at listOnTimeout (node:internal/timers:573:17) at process.processTimers (node:internal/timers:514:7)

I am getting this error when i try to scrape any product.

@ManuelAlejandroG
Copy link

Anyone having issues with the logos and favicon? It is not displaying properly on VS Code for me....

@nsaicharan
Copy link

cors I am facing this issue and I am using this in India. Does it create problems due to different country? Please anybody suggest the solution, how you successfully did it?

@vivek9124vivek Maybe you missed "use server" in the file where you made axios request. Or you didn't enable serverActions in next.config.js.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment