Skip to content

Instantly share code, notes, and snippets.

Avatar

Dave Smart dwsmart

View GitHub Profile
@dwsmart
dwsmart / mastodonsitemap.js
Last active Nov 15, 2022
Node.js script to create a sitemap from mastodon API
View mastodonsitemap.js
// config
const sitemapFile = '/var/www/html/mastodon_sitemap.xml'; // path / name of the created sitemap
const mastodonServer = 'https://yourmastodon.domain'; // your mastodon server, no trailing slash
const numberofPosts = 10000; // number of posts to include in the sitemap
const pingGoogle = false; // true or false
const pingURL = 'enter_full_final_accessible_url'; // URL to ping Google with, if you're putting on differet domain, see cross-site sitemaps https://developers.google.com/search/docs/crawling-indexing/sitemaps/large-sitemaps#manage-sitemaps-for-multiple-sites
// end config
const fs = require('fs');
@dwsmart
dwsmart / nginx_redirect_mastodon_well-known.config
Created Nov 9, 2022
nginx conf for redirecting .well-known for mastodon
View nginx_redirect_mastodon_well-known.config
# before server {} block
# if you're not mapping $request_path already, you'll need to this next block
map $request_uri $request_path {
~(?<captured_path>[^?]*) $captured_path;
}
map $arg_resource $valid_mastodon {
# If you want any account at your domain to resolve to just one mastodon account, i.e bob@tamethebots.com, mary@tamethebots.com
View remove_cloudflare_apps_script.js
/*
* a cloudflare worker to remove the script tag they inject into the head if you have a cloudflare app installed.
* WARNING: some apps may not work if you remove this script tag!!! But things like logflare that add no js of
* their own should be fine.
*/
addEventListener('fetch', event => {
const request = event.request
event.respondWith(handleRequest(request))
});
@dwsmart
dwsmart / decode-encode.js
Last active Oct 27, 2021
decode-encode.js
View decode-encode.js
function fixedEncodeURI(str) {
const prepURL = new URL(str);
const origin = prepURL.origin;
let raw = str;
let decoded = decodeURI(str);
while (raw !== decoded) {
decoded = decodeURI(decoded);
raw = decodeURI(raw);
}
justPath = decoded.replace(origin, '');
@dwsmart
dwsmart / logflare_to_common_log_format.js
Last active Aug 3, 2021
Logflare to Common Log Format nodejs Script
View logflare_to_common_log_format.js
// require libs
// run npm install @google-cloud/bigquery
const { BigQuery } = require('@google-cloud/bigquery');
const fs = require('fs');
// BigQuery Config - see https://cloud.google.com/docs/authentication/production#create_service_account
const options = {
keyFilename: '{path_to_key_file}',
projectId: '{project_id}',
};
View unicode_normal.js
function normEncoding(string) {
try {
return urlEncodeCaseCorrect(encodeURI(path).replace(/%25/g, '%'));
} catch (e) {
return string;
}
}
function urlEncodeCaseCorrect(string) {
return path.replace(/%[0-9a-fA-F]{2}/g, function(match) {
View add_noindex.js
// update existing robots tag if one is there
const oldRobots = document.querySelector("[name='robots']");
if (oldRobots) {
oldRobots.content = "noindex";
} else {
// add one if none existed
const newRobots = document.createElement('meta');
newRobots.name = "robots";
newRobots.content = "noindex";
document.getElementsByTagName('head')[0].prepend(newRobots);
View syntheticFid-targetInject.js
const puppeteer = require('puppeteer');
// throttle network and cpu?
const throttle = true;
// viewport sizes
const viewportw = 412;
const viewporth = 732;
// url to test
const theurl = 'https://tamethebots.com';
(async() => {
View syntheticFid.js
const puppeteer = require('puppeteer');
// throttle network and cpu?
const throttle = true;
// viewport sizes
const viewportw = 412;
const viewporth = 732;
// url to test
const theurl = 'https://tamethebots.com';
(async() => {
View missing-anchor-ids.js
const currURL = `${window.location.protocol}//${window.location.host}${window.location.pathname}${window.location.search}`
let outputString = ''
for (var idx in document.links) {
const ancString = `${document.links[idx]}`
if (ancString.includes('#')) {
const anchParts = ancString.split('#')
if (anchParts[0] === currURL) {
if (!document.getElementById(anchParts[1])) {
outputString += `${document.links[idx]}\n`