JadsonLucena

async function HlsScraper(rootUrl) {
  const urls = new Set()

  if (!URL.canParse(rootUrl)) {
    return urls
  }

  rootUrl = new URL(rootUrl)

  if (!rootUrl.pathname.endsWith('.m3u8')) {

JadsonLucena

async function SitemapScraper(rootUrl) {
  const urls = new Set()

  if (!URL.canParse(rootUrl)) {
    return urls
  }

  const res = await fetch(rootUrl)

  if (![200, 301, 302, 303, 304, 307, 308].includes(res.status)) {

JadsonLucena

// https://www.rfc-editor.org/rfc/rfc9309.html
// https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt
// https://developers.google.com/search/docs/advanced/crawling/overview-google-crawlers
// https://github.com/google/robotstxt

const RobotsTxt = {
  parse (robotsTxt) {
    const res = {
      userAgent: {},
      sitemap: [],

JadsonLucena

#!/bin/bash

# https://git-scm.com/docs/githooks#_commit_msg

if [[ $(git branch | grep '*' | sed 's/* //') == *"no branch"* ]]
then
  echo "Rebase or merge in progress. Skipping commit-msg hook."
  exit 0
fi

JadsonLucena

const crypto = require('crypto')


function urlPrefixEncoded(url) {

  url = new URL(url)

  // A URL-safe base64 encoded URL prefix that encompasses all paths that the signature should be valid for
  // The prefix shouldn't include query parameters or fragments such as ? or #
  return Buffer.from(url.origin + url.pathname).toString('base64url')

JadsonLucena

/*
	https://nodejs.org/api/fs.html#fsreaddirsyncpath-options
*/

const fs = require('fs');
const path = require('path');

const readdirFilesSync = (_path, { encoding = 'utf8', withFileTypes = false, recursive = false } = {}) => [].concat(fs.readdirSync(_path, { encoding, withFileTypes }).reduce((acc, item) => (item => fs.lstatSync(item).isDirectory() ? (recursive ? acc.concat(readdirFilesSync(item, { encoding, withFileTypes, recursive })) : acc) : acc.concat(item))(path.join(_path, item)), []));

module.exports = readdirFilesSync;

JadsonLucena

const Cookie = {
  parse: (cookies = '') => Object.fromEntries(cookies.split(';').map(e => e.trim()).filter(Boolean).map(cookie => (cookie => [cookie.shift(), cookie.join('=')])(cookie.split('=')))),
  stringify: cookies => Object.entries(cookies).map(([key, value]) => `${key}=${value}`).join(';')
}

export default Cookie

JadsonLucena

class HTTPError extends Error {

	constructor(message, statusCode, ...options) {

		// Needs to pass both `message` and `options` to install the "cause" property.
		super(message, ...options);

		// Maintains proper stack trace for where our error was thrown (only available on V8)
		if (Error.captureStackTrace) {

JadsonLucena

/*
	https://nodejs.org/api/url.html#url-strings-and-url-objects
	https://datatracker.ietf.org/doc/html/rfc4291
	https://datatracker.ietf.org/doc/html/rfc3986
	https://datatracker.ietf.org/doc/html/rfc2396
	https://datatracker.ietf.org/doc/html/rfc1738
	https://datatracker.ietf.org/doc/html/rfc1035
	https://datatracker.ietf.org/doc/html/rfc1034
	https://datatracker.ietf.org/doc/html/rfc791