Skip to content

Instantly share code, notes, and snippets.

@dwsmart
Last active November 15, 2022 11:04
Show Gist options
  • Save dwsmart/fc8bee06361063f3e7bbd8c551eafc65 to your computer and use it in GitHub Desktop.
Save dwsmart/fc8bee06361063f3e7bbd8c551eafc65 to your computer and use it in GitHub Desktop.
Node.js script to create a sitemap from mastodon API
// config
const sitemapFile = '/var/www/html/mastodon_sitemap.xml'; // path / name of the created sitemap
const mastodonServer = 'https://yourmastodon.domain'; // your mastodon server, no trailing slash
const numberofPosts = 10000; // number of posts to include in the sitemap
const pingGoogle = false; // true or false
const pingURL = 'enter_full_final_accessible_url'; // URL to ping Google with, if you're putting on differet domain, see cross-site sitemaps https://developers.google.com/search/docs/crawling-indexing/sitemaps/large-sitemaps#manage-sitemaps-for-multiple-sites
// end config
const fs = require('fs');
const fetch = require('node-fetch');
async function getAllToots() {
let hasNextPage = true;
let after = "";
try {
fs.writeFileSync(sitemapFile, `<?xml version="1.0" encoding="UTF-8" ?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n`);
} catch (err) {
console.error(err);
}
while (hasNextPage) {
const tootList = await getTootList(after);
let locCount = 0;
if (tootList.length === 0) {
hasNextPage = false;
} else {
tootList.forEach(toot => {
if(locCount < numberofPosts) {
fs.appendFileSync(sitemapFile, `<url>\n <loc>${toot.url}</loc>\n <lastmod>${toot.created_at}</lastmod></url>\n`);
locCount++;
} else {
hasNextPage = false;
}
});
const lastItem = tootList.pop();
console.log("lastItem", lastItem.url);
after = lastItem.id;
}
}
fs.appendFileSync(sitemapFile, `</urlset>`);
return 'done';
}
async function getTootList(after = "") {
console.log("after", after);
const posts = await fetch(`${mastodonServer}/api/v1/timelines/public?local=true&max_id=${after}`)
.then(res => res.json())
return posts;
}
(async () => {
const done = await getAllToots();
if(pingGoogle) {
fetch(pingURL);
}
console.log("done?", done);
})();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment