Created
October 9, 2018 10:23
-
-
Save Dan-Q/1129e91db2c74cdd0c057b477702bfca to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// Basic Reddit Content Exporter | |
// Author: Dan Q <https://danq.me/> | |
// License: The Unlicense <https://unlicense.org/> / Public Domain | |
// 1. Go to www.reddit.com and log in (logging in means that we don't have to worry about a modhash) | |
// 2. Edit this script to include YOUR username and the content you want to download | |
// 3. Paste this script into your browser console | |
(()=>{ | |
const username = 'avapoet'; // <-- your username goes here (be sure to be logged in as it, too!) | |
const content = 'submitted'; // <-- valid options: overview (everything), submitted (posts), comments, | |
// upvoted, downvoted, hidden, saved, gilded | |
const limit = 100; // <-- how many items to fetch at a time; 100 is the upper limit | |
const rate = 2000; // <-- milliseconds; the Reddit API requests that you make no more than | |
// 30 requests per minute so we take a 2000ms break between each | |
const showProgress = true; // <-- should progress be shown as-we-go-along or only at the end | |
// (set to false to possibly maybe slightly improve performance) | |
// Storage | |
let data = []; | |
let after = ''; | |
// Set the page up so that we can use it for progress/output | |
document.querySelectorAll('link, style').forEach(link=>link.remove()); // strip CSS | |
const body = document.querySelector('body'); | |
body.innerHTML = ` | |
<h1>Basic Content Exporter</h1> | |
<p> | |
<label for="itemCounter">Items exported:</label> | |
<input type="text" id="itemCounter" readonly /> | |
<span id="itemCounterStatus"></span> | |
</p> | |
<p> | |
<label for="result">Output:</label> | |
<textarea id="result" style="width: 100%; min-height: 80vh; resize: vertical;"></textarea> | |
</p> | |
`; | |
const itemCounter = document.getElementById('itemCounter'); | |
const itemCounterStatus = document.getElementById('itemCounterStatus'); | |
const result = document.getElementById('result'); | |
// When all fetching is done, this function is called | |
function allFetched(){ | |
if(!showProgress) result.value = JSON.stringify(data); | |
itemCounterStatus.innerText = 'FINISHED!'; | |
} | |
// Fetcher method pulls the next ${limit} (e.g. 100) entries and then queues another fetcher | |
// if necessary or calls allFetched if not | |
function fetcher(){ | |
const url = `https://www.reddit.com/user/${username}/${content}.json?limit=${limit}&after=${after}`; | |
itemCounterStatus.innerText = `Fetching ${url}`; | |
fetch(url, { credentials: 'include' }).then(r=>r.json()).then(json=>{ | |
// Merge new data in | |
data = [...data, ...json.data.children]; | |
// Display progress | |
if(showProgress) result.value = JSON.stringify(data); | |
itemCounter.value = data.length; | |
// Work out whether we're done yet | |
if(json.data.after){ | |
// there's still more data to fetch! | |
after = json.data.after; | |
setTimeout(fetcher, rate); | |
itemCounterStatus.innerText = `Waiting ${rate}ms`; | |
} else { | |
allFetched(); | |
} | |
}); | |
} | |
// Begin fetching! | |
fetcher(); | |
})(); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment