Last active
November 1, 2018 23:02
Star
You must be signed in to star a gist
Lua Scraping Function
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// http://novicelab.org/jsonabc/ is a good resource to sort and beautify the json so that commits are clean. | |
function scrapeit() | |
print("Downloading...") | |
downloadFile(getMudletHomeDir().."/functions.html", "https://wiki.mudlet.org/w/Manual:Lua_Functions") | |
end | |
function trim(s) | |
return (s:gsub("^%s*(.-)%s*$", "%1")) | |
end | |
function magiclines(s) | |
if s:sub(-1)~="\n" then s=s.."\n" end | |
return s:gmatch("(.-)\n") | |
end | |
function scrapeLuaFunctions(_, filename) | |
-- is the file that downloaded ours? | |
if not filename:match("functions", 1, true) then return end | |
print("Done this time!") | |
-- parse our downloaded file for the player count | |
io.input(filename) | |
local s = io.read("*all") | |
local funcs = {} | |
local funcsHash = {} | |
--for match in string.gmatch(s, "") do | |
local count = 0 | |
local state = 0 | |
local line = nil | |
local match = nil | |
local name, usage, definition | |
for line in magiclines(s) do | |
if state == 0 then | |
--print("testing match on " .. line) | |
name = string.match(line, '<h2><span class="mw%-headline" id="(.-)">.-</span></h2>') | |
if name then | |
state = 1 | |
--print("Name: " .. name) | |
end | |
elseif state == 1 then | |
usage = string.match(line, '<dl><dt>(.-)</dt>') | |
if usage then | |
state = 0 | |
--print("Usage: " .. usage) | |
func = {} | |
func.name = trim(name) | |
func.usage = trim(usage) | |
table.insert(funcs, func) | |
end | |
end | |
end | |
table.sort(funcs, function (a,b) return a.name < b.name end) | |
funcsHash = {} | |
for i, v in ipairs(funcs) do | |
--print(v.name .. " - " .. v.usage) | |
if not string.match(v.name, "[%:%.]") then | |
funcsHash[v.name] = v.usage | |
end | |
--funcsHash[v.name] = {["usage"] = v.usage, ["definition"] = v.definition} | |
end | |
openUserWindow("test userwindow") | |
clearWindow("test userwindow") | |
local str = "" | |
for k, v in pairs(funcsHash) do | |
str = str .. v .. '|' | |
end | |
str = string.sub(str, 1, #str - 2) | |
echo("test userwindow", str) | |
print(table.size(funcsHash)) | |
jsonText = yajl.to_string(funcsHash) | |
print(#jsonText) | |
os.remove(filename) | |
local f = io.open(getMudletHomeDir() .. "..\\..\\..\\lua-function-list.json", "w") | |
io.output(f) | |
io.write(jsonText) | |
io.close(f) | |
return true | |
end | |
registerAnonymousEventHandler("sysDownloadDone", "scrapeLuaFunctions") |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment