Skip to content

Instantly share code, notes, and snippets.

@zomars
Forked from agungjk/[slug].js
Created April 6, 2021 20:43
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save zomars/f97f829c5066829ad86ea822a26edb9a to your computer and use it in GitHub Desktop.
Save zomars/f97f829c5066829ad86ea822a26edb9a to your computer and use it in GitHub Desktop.
Crawler example on Vercel using Puppeteer and NextJS API routes
const puppeteer = require('puppeteer-core');
const cheerio = require('cheerio');
const chrome = require('chrome-aws-lambda');
export default async (req, res) => {
const slug = req?.query?.slug;
if (!slug) {
res.statusCode = 200
res.setHeader('Content-Type', 'application/json')
res.end(JSON.stringify({ id: null }))
return;
}
const browser = await puppeteer.launch(
process.env.NODE_ENV === 'production'
? {
args: chrome.args,
executablePath: await chrome.executablePath,
headless: chrome.headless,
}
: {}
);
const page = await browser.newPage();
page.setUserAgent('Opera/9.80 (J2ME/MIDP; Opera Mini/5.1.21214/28.2725; U; ru) Presto/2.8.119 Version/11.10');
await page.goto(`https://m.youtube.com/${slug}/videos`);
let content = await page.content();
var $ = cheerio.load(content);
$.prototype.exists = function (selector) {
return this.find(selector).length > 0;
}
let id = null;
const isLive = $('body').exists('[data-style="LIVE"]');
if (isLive) {
const url = $('ytm-compact-video-renderer .compact-media-item-image').attr('href');
const arr = url.split('?v=');
id = arr[1];
}
await browser.close();
res.statusCode = 200
res.setHeader('Content-Type', 'application/json')
res.end(JSON.stringify({ id }))
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment