Skip to content

Instantly share code, notes, and snippets.

Created May 20, 2017 11:40
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save anonymous/5c20de6366977242384f56d0d1b92ee5 to your computer and use it in GitHub Desktop.
Save anonymous/5c20de6366977242384f56d0d1b92ee5 to your computer and use it in GitHub Desktop.
A script that checks the values from Duolingo's incubator site and makes update to a wiki (defaults to wikia's wikis).
{
"ar#en": {
"P1date": "2014-01-29",
"P2date": "2014-05-09",
"P3date": "2015-08-11"
},
}
// Duolingo incubator update
// By Dessamator
// Public domain (or CCO - https://creativecommons.org/publicdomain/zero/1.0/)
// Google apps script
// Instructions to use
// Follow instructions in https://developers.google.com/apps-script/overview
// Create a file, e.g. 'incubatorupdates.gs' and paste the content below
// Click run on the menu bar on top, then click updateIncubatorData()
// This will generate a new module (if it doesn't exist) or will update the existing one
// This can also run periodically using triggers (https://developers.google.com/apps-script/guides/triggers/).
// Example file incupdates.json that is created in google drive is included...
// Incubator update bot.
var iThrottler = 1000;
var wikiasToRun = ["duolingo", "fr.duolingo"];
var sStatusPage = "|"; //This is the page it checks if it can run, e.g. "User:DuolingoBot", currently set to an invalid file
//Checks if new courses have changed phases and saves to a page
function updateIncubatorData() {
//Checks to see if bot can run.
if (canRun()){
var url = 'http://incubator.duolingo.com/api/1/courses/list?'
var res = getJsonFeed(url);
var oDuoCourseList = {};
var sData = "";
var oWikiCourseInfo;
var sUpdateText;
var oCourseDates = {"1":"start_date","2":"beta_date", "3":"released_date"};
var bFlag;
var sUpdateComment = "";
try {
oWikiCourseInfo = importData("incupdates.json")
}
catch (e){
oWikiCourseInfo = {}
}
for (var i in res.directions) {
var sL1 = res.directions[i].from_language_id;
var sLearnLangId = res.directions[i].learning_language_id;
var sProgress = res.directions[i].progress;
var sPhase = res.directions[i].phase;
if (sPhase === 1) {
sProgress = sProgress * 2;
} else {
sProgress = 100;
}
sProgress = Math.round(sProgress * 100) / 100
oDuoCourseList[sL1 + '#' + sLearnLangId] = {
sLangId: sL1,
sLearnLangId: sLearnLangId,
sProgress: sProgress,
sPhase: sPhase
};
var iWPhase = 0;
if (oWikiCourseInfo) {
var oWikiCourseDetails = oWikiCourseInfo[sL1 + '#' + sLearnLangId];
var sDate = new Date().toJSON().slice(0,10);
if (oWikiCourseDetails) {
for (i=1;i<4;i++){
if (oWikiCourseDetails['P'+i+'date']) {
iWPhase++;
}
}
if ( iWPhase != sPhase) {
bFlag = true;
var oCourseDetails = getCourseDetails(sL1,sLearnLangId);
if (oCourseDetails){
if (oCourseDetails[oCourseDates[sPhase]]) {
sDate = oCourseDetails[oCourseDates[sPhase]].slice(0,10);
}
else {
Logger.log("Course phase "+ sPhase +" date is null. Incubator probably not updated yet." )
saveData("incupdatesdebug.txt", "Error: oCourseDetails[oCourseDates[sPhase]] ->" + oCourseDetails[oCourseDates[sPhase]]
+ "\noCourseDates[sPhase] ->" +oCourseDates[sPhase] + " sPhase ->" + sPhase
+ "sL1:" + sL1 + " sLearnLangId -> " + sLearnLangId );
return;
}
}
sUpdateComment = sUpdateComment + sL1 + '->' + sLearnLangId + ' - P' + sPhase + ";"
oWikiCourseInfo[sL1 + '#' + sLearnLangId]['P' + sPhase + "date"] = sDate;
}
} else {
bFlag = true;
var oCourseDetails = getCourseDetails(sL1,sLearnLangId);
if (oCourseDetails){
sDate = oCourseDetails[oCourseDates[sPhase]].slice(0,10);
}
oWikiCourseInfo[sL1 + '#' + sLearnLangId] = {};
oWikiCourseInfo[sL1 + '#' + sLearnLangId]['P' + sPhase + "date"] = sDate;
sUpdateComment = sUpdateComment + sL1 + '->' + sLearnLangId + ' - P' + sPhase + ";"
}
}
}
if (i < res.directions.length - 1) {
sSuffix = "\r\n";
}
var bTestCode = false;
if (bFlag || bTestCode) {
var sTemplateData = "local courseData = [[";
var sTemplateSuffix = "]]\nlocal json = require(\"Module:JSON\")\nreturn json:decode(courseData)";
var sBeautifiedJson = JSON.stringify(oWikiCourseInfo, null, "\t");
sUpdateText = sTemplateData + sBeautifiedJson + sTemplateSuffix;
if (!bTestCode) {
for (var wikiId in wikiasToRun) {
updateWikiPage("Module:Crow/json", sUpdateText,sUpdateComment, wikiasToRun[wikiId]);
}
try{
saveData("incupdates.json", sBeautifiedJson);
}
catch(e){
saveData("incupdatesbak.json", sBeautifiedJson);
mSend('Error');
}
}
else {
mSend(sUpdateText);
saveData("incupdatesdebug.txt", sUpdateText);
}
}
mSend("MikoBot - Nothing to update");
return sUpdateText;
}
}
//Updates a page on the wiki
//sPage (string) pagename
//sNewText (string) text to save
//sComment (string) summary or comment for the edit
//sWiki (string) name of wiki to save, e.g. duolingo
function updateWikiPage(sPage, sNewText,sComment,sWiki) {
//Only works every 1 second
var header = ""
var sMsg = sNewText || "";
var sWiki = sWiki || "duolingo";
var sDomain = ".wikia.com";
var sPage = sPage || "";
var sOptions1 = {
"method": "POST"
};
var sPageUrl = "http://" + sWiki + sDomain + "/api.php?action=login&format=json";
try {
var oLogin = UrlFetchApp.fetch(sPageUrl, sOptions1);
var oData = JSON.parse(oLogin.getContentText());
var oHeaders = oLogin.getAllHeaders();
var oCookies = {
"cookie": oHeaders['Set-Cookie']
};
if (oLogin.getResponseCode() === 200) {
var options2 = {
"method": "POST",
"headers": oCookies
}
var lconfirm = UrlFetchApp.fetch(sPageUrl, options2);
var oData1 = JSON.parse(lconfirm.getContentText());
if (lconfirm.getResponseCode() === 200) {
var oData1 = JSON.parse(lconfirm.getContentText());
var options3 = {
"method": "POST",
"payload": {
"action": "query",
"prop": "info",
"titles": sPage,
"intoken": "edit"
},
"headers": oCookies
};
var qresult = UrlFetchApp.fetch(sPageUrl, options3);
var oDatai = qresult.getContentText();
var toke = (oDatai.split("edittoken")[1]).split("\+")[0];
var token = toke.substring(3, toke.length);
var options4 = {
"method": "POST",
"headers": oCookies,
"payload": {
"action": "edit",
"title": sPage,
"text": sMsg,
"summary": header,
"format": 'json',
"token": token + "+\\",
"bot":true,
},
};
var editurl = "http://" + sWiki + sDomain + "/api.php?"
var edit = UrlFetchApp.fetch(editurl, options4);
}
}
} catch (e) {
mSend(e.toString() + ' (err)');
}
Utilities.sleep(iThrottler )
}
// Checks if the bot can run, and aborts if not
function canRun(){
// this is the page the bot checks to see if it should run
var sUrl = "http://duolingo.wikia.com/api.php?format=json&action=query&titles="+sStatusPage+"&prop=revisions&rvprop=content";
var oData = getJsonFeed(sUrl)
for (var entry in oData.query.pages) {
var oPageContent = oData.query.pages[entry]
if (oData.query.pages[entry]){
try {
var oRevisions = oPageContent["revisions"][0]["*"];
if (oRevisions){
var bStatus = oRevisions.split("=");
if (oRevisions.substring(0,18)=="botstatus=disabled"){
Logger.log("Bot disabled");
return false;
}
}
}
catch (e){
Logger.log("Bot disabled");
}
}
}
return true;
}
// Saves course data to google drive folder
function saveData(sFileName, sFileData) {
// Check that the file name entered wasn't empty
if (sFileName.length !== 0) {
// var files = DriveApp.searchFiles(sFileName)
files = DriveApp.getFilesByName(sFileName)
// Loop through the results
var bFound = false;
while (files && files.hasNext()) {
var file = files.next();
// assuming the first file we find is the one we want
bFound = file.getName() == sFileName
if (bFound) {
// get file as a string
file.setContent(sFileData)
mSend('File Saved');
break;
}
}
if (!bFound) {
DriveApp.createFile(sFileName, sFileData, MimeType.PLAIN_TEXT);
}
}
}
// Gets course details
// L1 (string) native language course code
// L2 (string) learning language course code
function getCourseDetails(L1,L2){
var oCookies = {
"X-Requested-With": "XMLHttpRequest",
};
var options = {
"method": "get",
"headers": oCookies,
}
var url = "https://www.duolingo.com/api/1/courses/"+L2+"/"+L1+"/show?ui_language_abbrev=en"
var oData;
try {
oData = getJsonFeed(url,options)
}
catch(e){
return ;
}
return oData;
}
// Gets a json feed
// url (string) site to get the json feed from
// options (table) options to fetch content { method: get/ post, "headers": cookies}
function getJsonFeed(url,options) {
Utilities.sleep(1000);
var response;
var options;
if(options) {
response = UrlFetchApp.fetch(url,options);
}
else{
response = UrlFetchApp.fetch(url);
}
var json = response.getContentText();
var oData = JSON.parse(json);
return oData;
}
// Gets data from a filename in google drive
// sFilename (string) the name of a file
function importData(sFileName) {
// https://developers.google.com/apps-script/reference/drive/drive-app#searchFiles(String)
var searchTerm = "title = '" + sFileName + "'";
// search for our file
var files = DriveApp.searchFiles(searchTerm)
var fLoadFile;
// Loop through the results
while (files.hasNext()) {
var fCurrFile = files.next();
// assuming the first file we find is the one we want
if (fCurrFile.getName() == sFileName) {
// get file as a string
fLoadFile = fCurrFile.getBlob().getDataAsString();
break;
}
}
if (fLoadFile==undefined) {
return;
} else{
var sLoadedData = JSON.parse(fLoadFile);
return sLoadedData;
}
}
//Debugging function to output text
//oMsg (string) an object containing a message
function mSend(oMsg) {
Logger.log(oMsg)
}
//Submits a request to send a post to a url
//sUrl (string) url post request is sent to
function postRequest(sUrl,oOptions){
Utilities.sleep(1000);
return UrlFetchApp.fetch(sUrl,oOptions);
}
//function deleteTrigger(triggerId) {
// // Loop over all triggers.
// var allTriggers = ScriptApp.getProjectTriggers();
// for (var i = 0; i < allTriggers.length; i++) {
// // If the current trigger is the correct one, delete it.
// if (allTriggers[i].getUniqueId() == triggerId) {
// ScriptApp.deleteTrigger(allTriggers[i]);
// break;
// }
// }
//}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment