Skip to content

Instantly share code, notes, and snippets.

Avatar

Dave Smart dwsmart

View GitHub Profile
@dwsmart
dwsmart / logflare_to_common_log_format.js
Last active Aug 3, 2021
Logflare to Common Log Format nodejs Script
View logflare_to_common_log_format.js
// require libs
// run npm install @google-cloud/bigquery
const { BigQuery } = require('@google-cloud/bigquery');
const fs = require('fs');
// BigQuery Config - see https://cloud.google.com/docs/authentication/production#create_service_account
const options = {
keyFilename: '{path_to_key_file}',
projectId: '{project_id}',
};
View unicode_normal.js
function normEncoding(string) {
try {
return urlEncodeCaseCorrect(encodeURI(path).replace(/%25/g, '%'));
} catch (e) {
return string;
}
}
function urlEncodeCaseCorrect(string) {
return path.replace(/%[0-9a-fA-F]{2}/g, function(match) {
View add_noindex.js
// update existing robots tag if one is there
const oldRobots = document.querySelector("[name='robots']");
if (oldRobots) {
oldRobots.content = "noindex";
} else {
// add one if none existed
const newRobots = document.createElement('meta');
newRobots.name = "robots";
newRobots.content = "noindex";
document.getElementsByTagName('head')[0].prepend(newRobots);
View syntheticFid-targetInject.js
const puppeteer = require('puppeteer');
// throttle network and cpu?
const throttle = true;
// viewport sizes
const viewportw = 412;
const viewporth = 732;
// url to test
const theurl = 'https://tamethebots.com';
(async() => {
View syntheticFid.js
const puppeteer = require('puppeteer');
// throttle network and cpu?
const throttle = true;
// viewport sizes
const viewportw = 412;
const viewporth = 732;
// url to test
const theurl = 'https://tamethebots.com';
(async() => {
View missing-anchor-ids.js
const currURL = `${window.location.protocol}//${window.location.host}${window.location.pathname}${window.location.search}`
let outputString = ''
for (var idx in document.links) {
const ancString = `${document.links[idx]}`
if (ancString.includes('#')) {
const anchParts = ancString.split('#')
if (anchParts[0] === currURL) {
if (!document.getElementById(anchParts[1])) {
outputString += `${document.links[idx]}\n`
View usedcss.js
/*
usage:
node usedcss url=<page_to_test> filename=<required_file_name> viewport=<widthxheight> ua='<my user agent>'
filename, viewport & ua optional,
Example:
node usedcss url=https://tamethebots.com filename=ttb_used.css viewport=360x640 ua='css coverage script'
Updated to print original stylesheet URL
*/
View move_shadow_to_light.js
//Run this in the console, will cooy the shadow DOM to the light. Will only work for open ones, and will not recurse nested shadow DOMs
for (let elMnt of document.getElementsByTagName('*')) {
if (elMnt.shadowRoot) elMnt.innerHTML = elMnt.shadowRoot.innerHTML;
}
View move_shadow_to_light.js
//Run this in the console, will cooy the shadow DOM to the light. Will only work for open ones, and will not recurse nested shadow DOMs
for (let elMnt of document.getElementsByTagName('*')) {
if (elMnt.shadowRoot) elMnt.innerHTML = elMnt.shadowRoot.innerHTML;
}
View move_shadow_to_light.js
//Run this in the console, will cooy the shadow DOM to the light. Will only work for open ones, and will not recurse nested shadow DOMs
for (let elMnt of document.getElementsByTagName('*')) {
if (elMnt.shadowRoot) elMnt.innerHTML = elMnt.shadowRoot.innerHTML;
}