Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Stream logs in an HTTP format from section.io platform (Use cases - External log storage, Machine learning, data processing etc)
--Requires section.io OpenResty module
--Requires an "Alternate Origin" configured that points to an HTTP log ingestion endpoint: https://www.section.io/docs/how-to/multiple-origins/
--There are several files that need to be created in the "openresty" folder in the section.io configuration. Recommend cloning the git repository from "Advanced Config" menu in section.io portal.
--server.conf should contain:
location / {
#https://github.com/openresty/lua-nginx-module#log_by_lua_file
log_by_lua_file /opt/proxy_config/log.lua;
proxy_pass "http://next_hop_upstream";
}
--init.lua should contain the following to gather the correct next-hop IP for the nginx subrequest we make later:
-- Configure `require()` to find custom Lua modules
package.path = package.path .. ";/opt/proxy_config/?.lua"
local hostsFile = io.open("/etc/hosts", "r")
nextHop = "1.1.1.1"
for line in hostsFile:lines() do
local etcHostsEntry = line:match("(%d+%.%d+%.%d+%.%d+).*next%-hop")
if etcHostsEntry then
nextHop = etcHostsEntry
end
end
ngx.log(ngx.ERR, "Final next-hop value: ", nextHop)
hostsFile:close()
--log.lua, Generates the Async HTTP request that is streamed and defines content. The file should contain:
local http = require ("http")
local function push_data(premature, uri, args, status, nextHopIP)
--Default variables
uri = uri or ""
args = args or ""
status = status or ""
local httpc = http.new()
--Example using Sumo Logic, Adjust to your requirements (Adjust request_uri function,HTTP header: Host, HTTP header: section-origin and the "body" to contain the data you want to send)
local res, err = httpc:request_uri("http://" .. nextHopIP .. "/receiver/v1/http/YOURENDPOINTDETAILHERE", {
method = "POST",
ssl_verify = false,
headers = {
["Accept"] = "*/*",
["Host"] = "collectors.au.sumologic.com",
["section-origin"] = "log_streaming_endpoint",
["X-Forwarded-Proto"] = "https",
},
body = [[{data:{"url": "]]..uri..[[", "args": "]]..args..[[", "status": "]]..status..[["}}]]
})
if not res then
ngx.log(ngx.ERR, "Log Stream Call failed : ", err)
return
end
end
local ok, err = ngx.timer.at(0, push_data, ngx.var.uri, ngx.var.args, ngx.status, nextHop)
if not ok then
ngx.log(ngx.ERR, "failed to create timer: ", err)
return
end
-- Include the file http.lua and http_headers.lua inside the openresty folder. Source: https://github.com/pintsized/lua-resty-http
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment