dwsmart

const puppeteer = require('puppeteer');
// throttle network and cpu?
const throttle = true;
// viewport sizes
const viewportw = 412;
const viewporth = 732;
// url to test
const theurl = 'https://tamethebots.com';

(async() => {

dwsmart

// update existing robots tag if one is there 
const oldRobots = document.querySelector("[name='robots']");
if (oldRobots) {
    oldRobots.content = "noindex";
} else {
    // add one if none existed
    const newRobots = document.createElement('meta');
    newRobots.name = "robots";
    newRobots.content = "noindex";
    document.getElementsByTagName('head')[0].prepend(newRobots);

dwsmart

function normEncoding(string) {
    try {
        return urlEncodeCaseCorrect(encodeURI(path).replace(/%25/g, '%'));
    } catch (e) {
        return string;
    }
}

function urlEncodeCaseCorrect(string) {
    return path.replace(/%[0-9a-fA-F]{2}/g, function(match) {

dwsmart

function fixedEncodeURI(str) {
    const prepURL = new URL(str);
    const origin = prepURL.origin;
    let raw = str;
    let decoded = decodeURI(str);
    while (raw !== decoded) {
        decoded = decodeURI(decoded);
        raw = decodeURI(raw);
    }
    justPath = decoded.replace(origin, '');

dwsmart

/*
* a cloudflare worker to remove the script tag they inject into the head if you have a cloudflare app installed.
* WARNING: some apps may not work if you remove this script tag!!! But things like logflare that add no js of
* their own should be fine.
*/

addEventListener('fetch', event => {
    const request = event.request
    event.respondWith(handleRequest(request))
});

dwsmart

// config
const sitemapFile = '/var/www/html/mastodon_sitemap.xml'; // path / name of the created sitemap
const mastodonServer = 'https://yourmastodon.domain'; // your mastodon server, no trailing slash
const numberofPosts = 10000; // number of posts to include in the sitemap
const pingGoogle = false; // true or false
const pingURL = 'enter_full_final_accessible_url'; // URL to ping Google with, if you're putting on differet domain, see cross-site sitemaps https://developers.google.com/search/docs/crawling-indexing/sitemaps/large-sitemaps#manage-sitemaps-for-multiple-sites
// end config


const fs = require('fs');

dwsmart

# before server {} block

# if you're not mapping $request_path already, you'll need to this next block
map $request_uri $request_path {
    ~(?<captured_path>[^?]*) $captured_path;
}

map $arg_resource $valid_mastodon {
    # If you want any account at your domain to resolve to just one mastodon account, i.e bob@tamethebots.com, mary@tamethebots.com

dwsmart

# google sheets app script to get file size in bytes from a url

function getSizes(theurl) {
  const response = UrlFetchApp.fetch(theurl);
  const blob = response.getBlob()
  const size = blob.getBytes().length
  return size
}

dwsmart

def Jaccard_Similarity(doc1, doc2): 
    
    # List the unique words in a document
    words_doc1 = set(doc1.lower().split()) 
    words_doc2 = set(doc2.lower().split())
    
    # Find the intersection of words list of doc1 & doc2
    intersection = words_doc1.intersection(words_doc2)

    # Find the union of words list of doc1 & doc2

dwsmart

// require libs
// run npm install @google-cloud/bigquery
const { BigQuery } = require('@google-cloud/bigquery');
const fs = require('fs');
// BigQuery Config - see https://cloud.google.com/docs/authentication/production#create_service_account
const options = {
  keyFilename: '{path_to_key_file}',
  projectId: '{project_id}',
};