Created
May 20, 2017 11:40
-
-
Save anonymous/5c20de6366977242384f56d0d1b92ee5 to your computer and use it in GitHub Desktop.
A script that checks the values from Duolingo's incubator site and makes update to a wiki (defaults to wikia's wikis).
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"ar#en": { | |
"P1date": "2014-01-29", | |
"P2date": "2014-05-09", | |
"P3date": "2015-08-11" | |
}, | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// Duolingo incubator update | |
// By Dessamator | |
// Public domain (or CCO - https://creativecommons.org/publicdomain/zero/1.0/) | |
// Google apps script | |
// Instructions to use | |
// Follow instructions in https://developers.google.com/apps-script/overview | |
// Create a file, e.g. 'incubatorupdates.gs' and paste the content below | |
// Click run on the menu bar on top, then click updateIncubatorData() | |
// This will generate a new module (if it doesn't exist) or will update the existing one | |
// This can also run periodically using triggers (https://developers.google.com/apps-script/guides/triggers/). | |
// Example file incupdates.json that is created in google drive is included... | |
// Incubator update bot. | |
var iThrottler = 1000; | |
var wikiasToRun = ["duolingo", "fr.duolingo"]; | |
var sStatusPage = "|"; //This is the page it checks if it can run, e.g. "User:DuolingoBot", currently set to an invalid file | |
//Checks if new courses have changed phases and saves to a page | |
function updateIncubatorData() { | |
//Checks to see if bot can run. | |
if (canRun()){ | |
var url = 'http://incubator.duolingo.com/api/1/courses/list?' | |
var res = getJsonFeed(url); | |
var oDuoCourseList = {}; | |
var sData = ""; | |
var oWikiCourseInfo; | |
var sUpdateText; | |
var oCourseDates = {"1":"start_date","2":"beta_date", "3":"released_date"}; | |
var bFlag; | |
var sUpdateComment = ""; | |
try { | |
oWikiCourseInfo = importData("incupdates.json") | |
} | |
catch (e){ | |
oWikiCourseInfo = {} | |
} | |
for (var i in res.directions) { | |
var sL1 = res.directions[i].from_language_id; | |
var sLearnLangId = res.directions[i].learning_language_id; | |
var sProgress = res.directions[i].progress; | |
var sPhase = res.directions[i].phase; | |
if (sPhase === 1) { | |
sProgress = sProgress * 2; | |
} else { | |
sProgress = 100; | |
} | |
sProgress = Math.round(sProgress * 100) / 100 | |
oDuoCourseList[sL1 + '#' + sLearnLangId] = { | |
sLangId: sL1, | |
sLearnLangId: sLearnLangId, | |
sProgress: sProgress, | |
sPhase: sPhase | |
}; | |
var iWPhase = 0; | |
if (oWikiCourseInfo) { | |
var oWikiCourseDetails = oWikiCourseInfo[sL1 + '#' + sLearnLangId]; | |
var sDate = new Date().toJSON().slice(0,10); | |
if (oWikiCourseDetails) { | |
for (i=1;i<4;i++){ | |
if (oWikiCourseDetails['P'+i+'date']) { | |
iWPhase++; | |
} | |
} | |
if ( iWPhase != sPhase) { | |
bFlag = true; | |
var oCourseDetails = getCourseDetails(sL1,sLearnLangId); | |
if (oCourseDetails){ | |
if (oCourseDetails[oCourseDates[sPhase]]) { | |
sDate = oCourseDetails[oCourseDates[sPhase]].slice(0,10); | |
} | |
else { | |
Logger.log("Course phase "+ sPhase +" date is null. Incubator probably not updated yet." ) | |
saveData("incupdatesdebug.txt", "Error: oCourseDetails[oCourseDates[sPhase]] ->" + oCourseDetails[oCourseDates[sPhase]] | |
+ "\noCourseDates[sPhase] ->" +oCourseDates[sPhase] + " sPhase ->" + sPhase | |
+ "sL1:" + sL1 + " sLearnLangId -> " + sLearnLangId ); | |
return; | |
} | |
} | |
sUpdateComment = sUpdateComment + sL1 + '->' + sLearnLangId + ' - P' + sPhase + ";" | |
oWikiCourseInfo[sL1 + '#' + sLearnLangId]['P' + sPhase + "date"] = sDate; | |
} | |
} else { | |
bFlag = true; | |
var oCourseDetails = getCourseDetails(sL1,sLearnLangId); | |
if (oCourseDetails){ | |
sDate = oCourseDetails[oCourseDates[sPhase]].slice(0,10); | |
} | |
oWikiCourseInfo[sL1 + '#' + sLearnLangId] = {}; | |
oWikiCourseInfo[sL1 + '#' + sLearnLangId]['P' + sPhase + "date"] = sDate; | |
sUpdateComment = sUpdateComment + sL1 + '->' + sLearnLangId + ' - P' + sPhase + ";" | |
} | |
} | |
} | |
if (i < res.directions.length - 1) { | |
sSuffix = "\r\n"; | |
} | |
var bTestCode = false; | |
if (bFlag || bTestCode) { | |
var sTemplateData = "local courseData = [["; | |
var sTemplateSuffix = "]]\nlocal json = require(\"Module:JSON\")\nreturn json:decode(courseData)"; | |
var sBeautifiedJson = JSON.stringify(oWikiCourseInfo, null, "\t"); | |
sUpdateText = sTemplateData + sBeautifiedJson + sTemplateSuffix; | |
if (!bTestCode) { | |
for (var wikiId in wikiasToRun) { | |
updateWikiPage("Module:Crow/json", sUpdateText,sUpdateComment, wikiasToRun[wikiId]); | |
} | |
try{ | |
saveData("incupdates.json", sBeautifiedJson); | |
} | |
catch(e){ | |
saveData("incupdatesbak.json", sBeautifiedJson); | |
mSend('Error'); | |
} | |
} | |
else { | |
mSend(sUpdateText); | |
saveData("incupdatesdebug.txt", sUpdateText); | |
} | |
} | |
mSend("MikoBot - Nothing to update"); | |
return sUpdateText; | |
} | |
} | |
//Updates a page on the wiki | |
//sPage (string) pagename | |
//sNewText (string) text to save | |
//sComment (string) summary or comment for the edit | |
//sWiki (string) name of wiki to save, e.g. duolingo | |
function updateWikiPage(sPage, sNewText,sComment,sWiki) { | |
//Only works every 1 second | |
var header = "" | |
var sMsg = sNewText || ""; | |
var sWiki = sWiki || "duolingo"; | |
var sDomain = ".wikia.com"; | |
var sPage = sPage || ""; | |
var sOptions1 = { | |
"method": "POST" | |
}; | |
var sPageUrl = "http://" + sWiki + sDomain + "/api.php?action=login&format=json"; | |
try { | |
var oLogin = UrlFetchApp.fetch(sPageUrl, sOptions1); | |
var oData = JSON.parse(oLogin.getContentText()); | |
var oHeaders = oLogin.getAllHeaders(); | |
var oCookies = { | |
"cookie": oHeaders['Set-Cookie'] | |
}; | |
if (oLogin.getResponseCode() === 200) { | |
var options2 = { | |
"method": "POST", | |
"headers": oCookies | |
} | |
var lconfirm = UrlFetchApp.fetch(sPageUrl, options2); | |
var oData1 = JSON.parse(lconfirm.getContentText()); | |
if (lconfirm.getResponseCode() === 200) { | |
var oData1 = JSON.parse(lconfirm.getContentText()); | |
var options3 = { | |
"method": "POST", | |
"payload": { | |
"action": "query", | |
"prop": "info", | |
"titles": sPage, | |
"intoken": "edit" | |
}, | |
"headers": oCookies | |
}; | |
var qresult = UrlFetchApp.fetch(sPageUrl, options3); | |
var oDatai = qresult.getContentText(); | |
var toke = (oDatai.split("edittoken")[1]).split("\+")[0]; | |
var token = toke.substring(3, toke.length); | |
var options4 = { | |
"method": "POST", | |
"headers": oCookies, | |
"payload": { | |
"action": "edit", | |
"title": sPage, | |
"text": sMsg, | |
"summary": header, | |
"format": 'json', | |
"token": token + "+\\", | |
"bot":true, | |
}, | |
}; | |
var editurl = "http://" + sWiki + sDomain + "/api.php?" | |
var edit = UrlFetchApp.fetch(editurl, options4); | |
} | |
} | |
} catch (e) { | |
mSend(e.toString() + ' (err)'); | |
} | |
Utilities.sleep(iThrottler ) | |
} | |
// Checks if the bot can run, and aborts if not | |
function canRun(){ | |
// this is the page the bot checks to see if it should run | |
var sUrl = "http://duolingo.wikia.com/api.php?format=json&action=query&titles="+sStatusPage+"&prop=revisions&rvprop=content"; | |
var oData = getJsonFeed(sUrl) | |
for (var entry in oData.query.pages) { | |
var oPageContent = oData.query.pages[entry] | |
if (oData.query.pages[entry]){ | |
try { | |
var oRevisions = oPageContent["revisions"][0]["*"]; | |
if (oRevisions){ | |
var bStatus = oRevisions.split("="); | |
if (oRevisions.substring(0,18)=="botstatus=disabled"){ | |
Logger.log("Bot disabled"); | |
return false; | |
} | |
} | |
} | |
catch (e){ | |
Logger.log("Bot disabled"); | |
} | |
} | |
} | |
return true; | |
} | |
// Saves course data to google drive folder | |
function saveData(sFileName, sFileData) { | |
// Check that the file name entered wasn't empty | |
if (sFileName.length !== 0) { | |
// var files = DriveApp.searchFiles(sFileName) | |
files = DriveApp.getFilesByName(sFileName) | |
// Loop through the results | |
var bFound = false; | |
while (files && files.hasNext()) { | |
var file = files.next(); | |
// assuming the first file we find is the one we want | |
bFound = file.getName() == sFileName | |
if (bFound) { | |
// get file as a string | |
file.setContent(sFileData) | |
mSend('File Saved'); | |
break; | |
} | |
} | |
if (!bFound) { | |
DriveApp.createFile(sFileName, sFileData, MimeType.PLAIN_TEXT); | |
} | |
} | |
} | |
// Gets course details | |
// L1 (string) native language course code | |
// L2 (string) learning language course code | |
function getCourseDetails(L1,L2){ | |
var oCookies = { | |
"X-Requested-With": "XMLHttpRequest", | |
}; | |
var options = { | |
"method": "get", | |
"headers": oCookies, | |
} | |
var url = "https://www.duolingo.com/api/1/courses/"+L2+"/"+L1+"/show?ui_language_abbrev=en" | |
var oData; | |
try { | |
oData = getJsonFeed(url,options) | |
} | |
catch(e){ | |
return ; | |
} | |
return oData; | |
} | |
// Gets a json feed | |
// url (string) site to get the json feed from | |
// options (table) options to fetch content { method: get/ post, "headers": cookies} | |
function getJsonFeed(url,options) { | |
Utilities.sleep(1000); | |
var response; | |
var options; | |
if(options) { | |
response = UrlFetchApp.fetch(url,options); | |
} | |
else{ | |
response = UrlFetchApp.fetch(url); | |
} | |
var json = response.getContentText(); | |
var oData = JSON.parse(json); | |
return oData; | |
} | |
// Gets data from a filename in google drive | |
// sFilename (string) the name of a file | |
function importData(sFileName) { | |
// https://developers.google.com/apps-script/reference/drive/drive-app#searchFiles(String) | |
var searchTerm = "title = '" + sFileName + "'"; | |
// search for our file | |
var files = DriveApp.searchFiles(searchTerm) | |
var fLoadFile; | |
// Loop through the results | |
while (files.hasNext()) { | |
var fCurrFile = files.next(); | |
// assuming the first file we find is the one we want | |
if (fCurrFile.getName() == sFileName) { | |
// get file as a string | |
fLoadFile = fCurrFile.getBlob().getDataAsString(); | |
break; | |
} | |
} | |
if (fLoadFile==undefined) { | |
return; | |
} else{ | |
var sLoadedData = JSON.parse(fLoadFile); | |
return sLoadedData; | |
} | |
} | |
//Debugging function to output text | |
//oMsg (string) an object containing a message | |
function mSend(oMsg) { | |
Logger.log(oMsg) | |
} | |
//Submits a request to send a post to a url | |
//sUrl (string) url post request is sent to | |
function postRequest(sUrl,oOptions){ | |
Utilities.sleep(1000); | |
return UrlFetchApp.fetch(sUrl,oOptions); | |
} | |
//function deleteTrigger(triggerId) { | |
// // Loop over all triggers. | |
// var allTriggers = ScriptApp.getProjectTriggers(); | |
// for (var i = 0; i < allTriggers.length; i++) { | |
// // If the current trigger is the correct one, delete it. | |
// if (allTriggers[i].getUniqueId() == triggerId) { | |
// ScriptApp.deleteTrigger(allTriggers[i]); | |
// break; | |
// } | |
// } | |
//} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment