Skip to content

Instantly share code, notes, and snippets.

@Dan-Q
Created October 9, 2018 10:23
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save Dan-Q/1129e91db2c74cdd0c057b477702bfca to your computer and use it in GitHub Desktop.
Save Dan-Q/1129e91db2c74cdd0c057b477702bfca to your computer and use it in GitHub Desktop.
// Basic Reddit Content Exporter
// Author: Dan Q <https://danq.me/>
// License: The Unlicense <https://unlicense.org/> / Public Domain
// 1. Go to www.reddit.com and log in (logging in means that we don't have to worry about a modhash)
// 2. Edit this script to include YOUR username and the content you want to download
// 3. Paste this script into your browser console
(()=>{
const username = 'avapoet'; // <-- your username goes here (be sure to be logged in as it, too!)
const content = 'submitted'; // <-- valid options: overview (everything), submitted (posts), comments,
// upvoted, downvoted, hidden, saved, gilded
const limit = 100; // <-- how many items to fetch at a time; 100 is the upper limit
const rate = 2000; // <-- milliseconds; the Reddit API requests that you make no more than
// 30 requests per minute so we take a 2000ms break between each
const showProgress = true; // <-- should progress be shown as-we-go-along or only at the end
// (set to false to possibly maybe slightly improve performance)
// Storage
let data = [];
let after = '';
// Set the page up so that we can use it for progress/output
document.querySelectorAll('link, style').forEach(link=>link.remove()); // strip CSS
const body = document.querySelector('body');
body.innerHTML = `
<h1>Basic Content Exporter</h1>
<p>
<label for="itemCounter">Items exported:</label>
<input type="text" id="itemCounter" readonly />
<span id="itemCounterStatus"></span>
</p>
<p>
<label for="result">Output:</label>
<textarea id="result" style="width: 100%; min-height: 80vh; resize: vertical;"></textarea>
</p>
`;
const itemCounter = document.getElementById('itemCounter');
const itemCounterStatus = document.getElementById('itemCounterStatus');
const result = document.getElementById('result');
// When all fetching is done, this function is called
function allFetched(){
if(!showProgress) result.value = JSON.stringify(data);
itemCounterStatus.innerText = 'FINISHED!';
}
// Fetcher method pulls the next ${limit} (e.g. 100) entries and then queues another fetcher
// if necessary or calls allFetched if not
function fetcher(){
const url = `https://www.reddit.com/user/${username}/${content}.json?limit=${limit}&after=${after}`;
itemCounterStatus.innerText = `Fetching ${url}`;
fetch(url, { credentials: 'include' }).then(r=>r.json()).then(json=>{
// Merge new data in
data = [...data, ...json.data.children];
// Display progress
if(showProgress) result.value = JSON.stringify(data);
itemCounter.value = data.length;
// Work out whether we're done yet
if(json.data.after){
// there's still more data to fetch!
after = json.data.after;
setTimeout(fetcher, rate);
itemCounterStatus.innerText = `Waiting ${rate}ms`;
} else {
allFetched();
}
});
}
// Begin fetching!
fetcher();
})();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment