dwsmart

function fixedEncodeURI(str) {
    const prepURL = new URL(str);
    const origin = prepURL.origin;
    let raw = str;
    let decoded = decodeURI(str);
    while (raw !== decoded) {
        decoded = decodeURI(decoded);
        raw = decodeURI(raw);
    }
    justPath = decoded.replace(origin, '');

dwsmart

// require libs
// run npm install @google-cloud/bigquery
const { BigQuery } = require('@google-cloud/bigquery');
const fs = require('fs');
// BigQuery Config - see https://cloud.google.com/docs/authentication/production#create_service_account
const options = {
  keyFilename: '{path_to_key_file}',
  projectId: '{project_id}',
};

dwsmart

function normEncoding(string) {
    try {
        return urlEncodeCaseCorrect(encodeURI(path).replace(/%25/g, '%'));
    } catch (e) {
        return string;
    }
}

function urlEncodeCaseCorrect(string) {
    return path.replace(/%[0-9a-fA-F]{2}/g, function(match) {

dwsmart

// update existing robots tag if one is there 
const oldRobots = document.querySelector("[name='robots']");
if (oldRobots) {
    oldRobots.content = "noindex";
} else {
    // add one if none existed
    const newRobots = document.createElement('meta');
    newRobots.name = "robots";
    newRobots.content = "noindex";
    document.getElementsByTagName('head')[0].prepend(newRobots);

dwsmart

const puppeteer = require('puppeteer');
// throttle network and cpu?
const throttle = true;
// viewport sizes
const viewportw = 412;
const viewporth = 732;
// url to test
const theurl = 'https://tamethebots.com';

(async() => {

dwsmart

const puppeteer = require('puppeteer');
// throttle network and cpu?
const throttle = true;
// viewport sizes
const viewportw = 412;
const viewporth = 732;
// url to test
const theurl = 'https://tamethebots.com';

(async() => {

dwsmart

const currURL = `${window.location.protocol}//${window.location.host}${window.location.pathname}${window.location.search}`
let outputString = ''
for (var idx in document.links) {
    const ancString = `${document.links[idx]}`
    if (ancString.includes('#')) {
        const anchParts = ancString.split('#')
        if (anchParts[0] === currURL) {
​
            if (!document.getElementById(anchParts[1])) {
                outputString += `${document.links[idx]}\n`

dwsmart

/*
usage:
node usedcss url=<page_to_test> filename=<required_file_name> viewport=<widthxheight> ua='<my user agent>' 
filename, viewport & ua optional,
Example:
node usedcss url=https://tamethebots.com filename=ttb_used.css viewport=360x640 ua='css coverage script' 

Updated to print original stylesheet URL

*/

dwsmart

//Run this in the console, will cooy the shadow DOM to the light. Will only work for open ones, and will not recurse nested shadow DOMs
for (let elMnt of document.getElementsByTagName('*')) {
   if (elMnt.shadowRoot) elMnt.innerHTML = elMnt.shadowRoot.innerHTML;
}

dwsmart

//Run this in the console, will cooy the shadow DOM to the light. Will only work for open ones, and will not recurse nested shadow DOMs
for (let elMnt of document.getElementsByTagName('*')) {
   if (elMnt.shadowRoot) elMnt.innerHTML = elMnt.shadowRoot.innerHTML;
}