Skip to content

Instantly share code, notes, and snippets.

@dicene
Last active November 1, 2018 23:02
  • Star 1 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
Star You must be signed in to star a gist
Save dicene/b714660b13da4bd7767f89182476af4c to your computer and use it in GitHub Desktop.
Lua Scraping Function
// http://novicelab.org/jsonabc/ is a good resource to sort and beautify the json so that commits are clean.
function scrapeit()
print("Downloading...")
downloadFile(getMudletHomeDir().."/functions.html", "https://wiki.mudlet.org/w/Manual:Lua_Functions")
end
function trim(s)
return (s:gsub("^%s*(.-)%s*$", "%1"))
end
function magiclines(s)
if s:sub(-1)~="\n" then s=s.."\n" end
return s:gmatch("(.-)\n")
end
function scrapeLuaFunctions(_, filename)
-- is the file that downloaded ours?
if not filename:match("functions", 1, true) then return end
print("Done this time!")
-- parse our downloaded file for the player count
io.input(filename)
local s = io.read("*all")
local funcs = {}
local funcsHash = {}
--for match in string.gmatch(s, "") do
local count = 0
local state = 0
local line = nil
local match = nil
local name, usage, definition
for line in magiclines(s) do
if state == 0 then
--print("testing match on " .. line)
name = string.match(line, '<h2><span class="mw%-headline" id="(.-)">.-</span></h2>')
if name then
state = 1
--print("Name: " .. name)
end
elseif state == 1 then
usage = string.match(line, '<dl><dt>(.-)</dt>')
if usage then
state = 0
--print("Usage: " .. usage)
func = {}
func.name = trim(name)
func.usage = trim(usage)
table.insert(funcs, func)
end
end
end
table.sort(funcs, function (a,b) return a.name < b.name end)
funcsHash = {}
for i, v in ipairs(funcs) do
--print(v.name .. " - " .. v.usage)
if not string.match(v.name, "[%:%.]") then
funcsHash[v.name] = v.usage
end
--funcsHash[v.name] = {["usage"] = v.usage, ["definition"] = v.definition}
end
openUserWindow("test userwindow")
clearWindow("test userwindow")
local str = ""
for k, v in pairs(funcsHash) do
str = str .. v .. '|'
end
str = string.sub(str, 1, #str - 2)
echo("test userwindow", str)
print(table.size(funcsHash))
jsonText = yajl.to_string(funcsHash)
print(#jsonText)
os.remove(filename)
local f = io.open(getMudletHomeDir() .. "..\\..\\..\\lua-function-list.json", "w")
io.output(f)
io.write(jsonText)
io.close(f)
return true
end
registerAnonymousEventHandler("sysDownloadDone", "scrapeLuaFunctions")
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment