-
-
Save danopia/c0c4313b4809d565af7c7738bcdbeec7 to your computer and use it in GitHub Desktop.
export * from "./deps.ts"; | |
import { | |
DatadogApi, | |
MetricSubmission, | |
fixedInterval, | |
} from "./deps.ts"; | |
const datadog = DatadogApi.fromEnvironment(Deno.env); | |
export function headers(accept = 'text/html') { | |
return { | |
headers: { | |
'Accept': accept, | |
'User-Agent': `Deno/${Deno.version} (+https://p.datadoghq.com/sb/5c2fc00be-393be929c9c55c3b80b557d08c30787a)`, | |
}, | |
}; | |
} | |
export async function runMetricsLoop( | |
gather: () => Promise<MetricSubmission[]>, | |
intervalMinutes: number, | |
loopName: string, | |
) { | |
for await (const dutyCycle of fixedInterval(intervalMinutes * 60 * 1000)) { | |
try { | |
const data = await gather(); | |
// Our own loop-health metric | |
data.push({ | |
metric_name: `ercot.app.duty_cycle`, | |
points: [{value: dutyCycle*100}], | |
tags: [`app:${loopName}`], | |
interval: 60, | |
metric_type: 'gauge', | |
}); | |
// Submit all metrics | |
try { | |
await datadog.v1Metrics.submit(data); | |
} catch (err) { | |
console.log(new Date().toISOString(), 'eh', err.message); | |
await datadog.v1Metrics.submit(data); | |
} | |
} catch (err) { | |
console.log(new Date().toISOString(), '!!', err.message); | |
} | |
} | |
}; |
// deno run --allow-net --allow-env examples/emit-metrics.ts | |
import { runMetricsLoop, MetricSubmission, headers, fetch } from "./_lib.ts"; | |
export async function start() { | |
await runMetricsLoop(grabUserMetrics, 5, 'ercot_ancillary'); | |
} | |
if (import.meta.main) start(); | |
async function grabUserMetrics(): Promise<MetricSubmission[]> { | |
const body = await fetch('http://127.0.0.1:5102/content/cdr/html/as_capacity_monitor.html', headers('text/html')).then(x => x.text()); | |
const sections = body.split('an="2">').slice(1); | |
const metrics = new Array<MetricSubmission>(); | |
for (const section of sections) { | |
const label = section.slice(0, section.indexOf('<')); | |
const boxes = section.match(/ <td class="tdLeft">[^<]+<\/td>\r\n <td class="labelClassCenter">[^<]+<\/td>/g) ?? []; | |
for (const box of boxes) { | |
const parts = box.split(/[<>]/); | |
const field = parts[2] | |
.replace(/Controllable Load Resource/g, 'CLR') | |
.replace(/Load Resource/g, 'LR') | |
.replace(/Generation Resource/g, 'GR') | |
.replace(/Energy Offer Curve/g, 'EOC') | |
.replace(/Output Schedule/g, 'OS') | |
.replace(/Base Point/g, 'BP') | |
.replace(/Resource Status/g, 'RS') | |
.replace(/ \(energy consumption\)/g, '') | |
.replace(/telemetered/g, 'TMd') | |
.replace(/Fast Frequency Response/g, 'FFR') | |
.replace(/available to decrease/g, 'to decr') | |
.replace(/available to increase/g, 'to incr') | |
.replace(/in the next 5 minutes/g, 'in 5min') | |
.replace(/Physical Responsive Capability \(PRC\)/g, 'PRC') | |
.replace(/^Real-Time /, '') | |
.replace(/[ ()-]+/g, ' ').trim().replace(/ /g, '_'); | |
// console.log(label, field, parts[6]); | |
metrics.push({ | |
metric_name: `ercot_ancillary.${field}`, | |
points: [{value: parseFloat(parts[6].replace(/,/g, ''))}], | |
interval: 60, | |
metric_type: 'gauge', | |
}); | |
} | |
} | |
console.log(new Date, 'ancillary', metrics | |
.find(x => x.metric_name.endsWith('PRC')) | |
?.points[0]?.value); | |
return metrics; | |
} |
export { default as DatadogApi } from "https://deno.land/x/datadog_api@v0.1.3/mod.ts"; | |
export type { MetricSubmission } from "https://deno.land/x/datadog_api@v0.1.3/v1/metrics.ts"; | |
export { fixedInterval } from "https://crux.land/4MC9JG#fixed-interval@v1"; | |
export { Sha256 } from "https://deno.land/std@0.95.0/hash/sha256.ts"; | |
export { runMetricsServer } from "https://deno.land/x/observability@v0.1.0/sinks/openmetrics/server.ts"; | |
export { replaceGlobalFetch, fetch } from "https://deno.land/x/observability@v0.1.0/sources/fetch.ts"; |
FROM hayd/alpine-deno:1.10.1 | |
WORKDIR /src/app | |
ADD deps.ts ./ | |
RUN ["deno", "cache", "deps.ts"] | |
ADD *.ts ./ | |
RUN ["deno", "cache", "mod.ts"] | |
ENTRYPOINT ["deno", "run", "--unstable", "--allow-net", "--allow-hrtime", "--allow-env", "--cached-only", "--no-check", "mod.ts"] |
// deno run --allow-net --allow-env examples/emit-metrics.ts | |
import { runMetricsLoop, MetricSubmission, headers, fetch } from "./_lib.ts"; | |
export async function start() { | |
await runMetricsLoop(grabUserMetrics, 10, 'ercot_eea'); | |
} | |
if (import.meta.main) start(); | |
async function grabUserMetrics(): Promise<MetricSubmission[]> { | |
const body = await fetch(`http://127.0.0.1:5102/content/alerts/conservation_state.js`, headers('application/javascript')).then(x => x.text()); | |
const line = body.split(/\r?\n/).find(x => x.startsWith('eeaLevel = ')); | |
if (!line) { | |
console.log(new Date, 'EEA Unknown'); | |
return []; | |
} | |
const level = parseInt(line.split('=')[1].trim()); | |
console.log(new Date, 'EEA Level', level); | |
return [{ | |
metric_name: `ercot.eea_level`, | |
points: [{value: level}], | |
interval: 60*10, | |
metric_type: 'gauge', | |
}]; | |
} |
// deno run --allow-net --allow-env examples/emit-metrics.ts | |
import { runMetricsLoop, MetricSubmission, headers, fetch } from "./_lib.ts"; | |
export async function start() { | |
await runMetricsLoop(grabUserMetrics, 1, 'ercot_realtime'); | |
} | |
if (import.meta.main) start(); | |
async function grabUserMetrics(): Promise<MetricSubmission[]> { | |
const body = await fetch('http://127.0.0.1:5102/content/cdr/html/real_time_system_conditions.html', headers('text/html')).then(x => x.text()); | |
const sections = body.split('an="2">').slice(1); | |
const metrics = new Array<MetricSubmission>(); | |
for (const section of sections) { | |
const label = section.slice(0, section.indexOf('<')); | |
const boxes = section.match(/ <td class="tdLeft">[^<]+<\/td>\r\n <td class="labelClassCenter">[^<]+<\/td>/g) ?? []; | |
for (const box of boxes) { | |
const parts = box.split(/[<>]/); | |
// console.log(label, parts[2], parts[6]); | |
if (label === 'DC Tie Flows') { | |
metrics.push({ | |
metric_name: `ercot.${label}`.replace(/[ -]+/g, '_'), | |
tags: [`ercot_dc_tie:${parts[2].split('(')[0].trim()}`], | |
points: [{value: parseFloat(parts[6])}], | |
interval: 60, | |
metric_type: 'gauge', | |
}); | |
} else { | |
metrics.push({ | |
metric_name: `ercot.${label}.${parts[2].split('(')[0].trim()}`.replace(/[ -]+/g, '_'), | |
points: [{value: parseFloat(parts[6])}], | |
interval: 60, | |
metric_type: 'gauge', | |
}); | |
} | |
} | |
} | |
console.log(new Date, 'grid', metrics[0]?.points[0]?.value); | |
return metrics; | |
} |
Licensed under the MIT license: | |
http://www.opensource.org/licenses/mit-license.php | |
Permission is hereby granted, free of charge, to any person obtaining a copy | |
of this software and associated documentation files (the "Software"), to deal | |
in the Software without restriction, including without limitation the rights | |
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | |
copies of the Software, and to permit persons to whom the Software is | |
furnished to do so, subject to the following conditions: | |
The above copyright notice and this permission notice shall be included in | |
all copies or substantial portions of the Software. | |
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | |
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | |
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN | |
THE SOFTWARE. |
// deno run --allow-net --allow-env examples/emit-metrics.ts | |
const knownTexts = new Map<string,string>(); | |
// https://www.faa.gov/air_traffic/weather/asos/?state=TX | |
// https://en.wikipedia.org/wiki/List_of_power_stations_in_Texas#Wind_farms | |
const ids = [ | |
'KABI', // Abilene (near Roscoe Wind Farm) | |
'KAUS', | |
'KDFW', | |
'KEFD', // Houston/Ellington Ar | |
'KGLS', // Galveston/Scholes In | |
'KHOU', // Houston/Hobby Arpt | |
'KIAH', | |
'KLBX', // Angleton/Texas Gulf | |
'KLRD', // Laredo (nearish Javelina Wind Energy Center) | |
'KLVJ', // Houston/Pearland Rgn | |
'KMAF', | |
'KSAT', | |
'KSGR', // Houston/Sugar Land R | |
'KTKI', | |
]; | |
import { runMetricsLoop, MetricSubmission, headers, fetch } from "./_lib.ts"; | |
export async function start() { | |
await runMetricsLoop(grabUserMetrics, 30, 'metar'); | |
} | |
if (import.meta.main) start(); | |
async function grabUserMetrics(): Promise<MetricSubmission[]> { | |
const body = await fetch(`https://www.aviationweather.gov/metar/data?ids=${ids.join('%2C')}&format=decoded`, headers('text/html')).then(resp => resp.text()); | |
const sections = body.split(/<!-- Data (?:starts|ends) here -->/)[1].split(`METAR for:</span></td><td>`).slice(1); | |
const stations = new Array<MetricSubmission[]>(); | |
for (const sect of sections) { | |
const title = sect.slice(0, sect.indexOf('<')); | |
const code = title.split(' ')[0]; | |
const name = title.split(/[\(\)]/)[1].replace(/[ ,]+/g, '_'); | |
const metrics = new Array<MetricSubmission>(); | |
let text = ''; | |
for (const row of sect.match(/>[^<]+<\/span><\/td><td[^>]*>[^<]+<\/td>/g) ?? []) { | |
const cells = row.split(/[<>]/); | |
const metric = cells[1].split(/[:(]/)[0].toLowerCase().trim(); | |
const value = cells[7].trim(); | |
// console.log([code, name, cells[1].slice(0, -1), cells[7]]); | |
if (metric === 'text') { | |
text = value; | |
} | |
if ([ | |
'temperature', | |
'dewpoint', | |
'pressure', | |
].includes(metric)) { | |
metrics.push({ | |
metric_name: `metar.${metric}`, | |
tags: [ | |
`metar_code:${code}`, | |
`metar_location:${name}`, | |
], | |
points: [{value: parseFloat(value)}], | |
interval: 60, | |
metric_type: 'gauge', | |
}); | |
} | |
if (metric === 'winds' && value.includes('MPH')) { | |
const [speed, gusts] = value.match(/([0-9.]+) MPH/g) ?? []; | |
if (speed) metrics.push({ | |
metric_name: `metar.winds.speed`, | |
tags: [ | |
`metar_code:${code}`, | |
`metar_location:${name}`, | |
], | |
points: [{value: parseFloat(speed)}], | |
interval: 60, | |
metric_type: 'gauge', | |
}); | |
if (gusts) metrics.push({ | |
metric_name: `metar.winds.gusts`, | |
tags: [ | |
`metar_code:${code}`, | |
`metar_location:${name}`, | |
], | |
points: [{value: parseFloat(gusts)}], | |
interval: 60, | |
metric_type: 'gauge', | |
}); | |
} | |
} | |
// console.log(code, text, knownTexts.get(code), metrics.length); | |
if (!text) continue; | |
if (knownTexts.get(code) === text) continue; | |
stations.push(metrics); | |
knownTexts.set(code, text); | |
} | |
console.log(new Date, 'METAR', (stations[0] ?? [])[0]?.tags); | |
return stations.flat(); | |
} |
import { start as startAncillary } from "./ancillary.ts"; | |
import { start as startEea } from "./eea.ts"; | |
import { start as startGrid } from "./grid.ts"; | |
import { start as startMetar } from "./metar.ts"; | |
import { start as startOutages } from "./outages.ts"; | |
import { start as startPrices } from "./prices.ts"; | |
import { | |
runMetricsServer, replaceGlobalFetch, | |
} from './deps.ts'; | |
if (Deno.args.includes('--serve-metrics')) { | |
runMetricsServer({ port: 9090 }); | |
console.log("Now serving OpenMetrics @ :9090/metrics"); | |
} | |
if (import.meta.main) { | |
await Promise.race([ | |
// 60s loops | |
// run these offset from each other for better utilization | |
startGrid(), | |
new Promise(ok => setTimeout(ok, 30*1000)).then(startAncillary), | |
// 10+ minute loops, they can overlap, it's ok | |
startEea(), | |
startMetar(), | |
startOutages(), | |
startPrices(), | |
]); | |
} |
// deno run --allow-net --allow-env examples/emit-metrics.ts | |
import { Sha256 } from "./deps.ts"; | |
let lastHash = ''; | |
import { runMetricsLoop, MetricSubmission, headers, fetch } from "./_lib.ts"; | |
export async function start() { | |
await runMetricsLoop(grabUserMetrics, 30, 'poweroutages_us'); | |
} | |
if (import.meta.main) start(); | |
async function grabUserMetrics(): Promise<MetricSubmission[]> { | |
const bodyText = await fetch(`https://poweroutage.us/api/web/counties?key=18561563181588&countryid=us&statename=Texas`, headers('application/json')).then(resp => resp.text()); | |
const hash = new Sha256().update(bodyText).hex().slice(0, 12); | |
if (hash === lastHash) { | |
console.log(new Date, 'Outages', hash); | |
return []; | |
} | |
lastHash = hash; | |
const body = JSON.parse(bodyText) as { | |
WebCountyRecord: { | |
CountyName: string; | |
OutageCount: number; | |
CustomerCount: number; | |
}[]; | |
}; | |
console.log(new Date, 'Outages', hash, body.WebCountyRecord[0].CountyName, body.WebCountyRecord[0].OutageCount); | |
return body.WebCountyRecord.flatMap(x => [{ | |
metric_name: `poweroutageus.outages`, | |
tags: [ | |
`county_name:${x.CountyName}`, | |
`county_state:Texas`, | |
], | |
points: [{value: x.OutageCount}], | |
interval: 60, | |
metric_type: 'gauge', | |
}, { | |
metric_name: `poweroutageus.customers`, | |
tags: [ | |
`county_name:${x.CountyName}`, | |
`county_state:Texas`, | |
], | |
points: [{value: x.CustomerCount}], | |
interval: 60, | |
metric_type: 'gauge', | |
}]); | |
} |
// deno run --allow-net --allow-env examples/emit-metrics.ts | |
import { runMetricsLoop, MetricSubmission, headers, fetch } from "./_lib.ts"; | |
export async function start() { | |
await waitForNextPrices(); | |
await runMetricsLoop(grabUserMetrics, 15, 'ercot_pricing'); | |
} | |
if (import.meta.main) start(); | |
async function grabUserMetrics(): Promise<MetricSubmission[]> { | |
const body = await fetch(`http://127.0.0.1:5102/content/cdr/html/real_time_spp`, headers('text/html')).then(x => x.text()); | |
const sections = body.split('</table>')[0].split('<tr>').slice(1).map(x => x.split(/[<>]/).filter((_, idx) => idx % 4 == 2)); | |
const header = sections[0]?.slice(2, -1) ??[]; | |
const last = sections[sections.length-1]?.slice(2, -1) ??[]; | |
const timestamp = sections[sections.length-1][1]; | |
console.log(new Date, 'Prices', timestamp, header[0], last[0]); | |
return header.map((h, idx) => { | |
return { | |
metric_name: `ercot.pricing`, | |
tags: [`ercot_region:${h}`], | |
points: [{value: parseFloat(last[idx])}], | |
interval: 60*15, | |
metric_type: 'gauge', | |
}; | |
}); | |
} | |
// launches this script 2m30s after the 15-minute mark for most-timely data | |
async function waitForNextPrices() { | |
const startDate = new Date(); | |
while (startDate.getMinutes() % 15 !== 2) { | |
startDate.setMinutes(startDate.getMinutes()+1); | |
} | |
startDate.setSeconds(30); | |
startDate.setMilliseconds(0); | |
const waitMillis = startDate.valueOf() - Date.now(); | |
if (waitMillis > 0) { | |
console.log(`Waiting ${waitMillis/1000/60}min for next pricing cycle`); | |
await new Promise(ok => setTimeout(ok, waitMillis)); | |
} | |
} |
Was there a reason to use 127.0.0.1:5102 instead of ercot.com?
Yes, and it's dumb. The ercot web server includes a pointless folded http header (one header spanning multiple lines) and because that is deprecated in modern http, the runtime I run this with (Deno) considers the response invalid.
So the localhost server is a golang proxy that drops all set-cookies on the floor. Originally i ran curl
as a subprocess but had some stability issues after long uptimes.
This isn't a problem under Node, just Deno. Upstream considered it a bug (on Discord) but i didn't file an issue.
This appears to have stopped working as of 5/11/2022. Did ERCOT change something on the backend to prevent the data polling?
@texastoastt looks like the source of the data hasn't changed: http://www.ercot.com/content/cdr/html/real_time_system_conditions.html
OP's server might of stopped fetching it or something on the datadog side
It was DNS 🥲
ERCOT blocks European traffic so I can't run this from home, I have it on a US-based VPS. And it's not noticeable when something goes wrong there.
I fixed the server. Thanks for the heads up
@danopia Are you still maintaining this?
Only very passively. Something up?
https://p.datadoghq.com/sb/5c2fc00be-393be929c9c55c3b80b557d08c30787a seems to be mostly nonfunctional at the moment.
Was there a reason to use 127.0.0.1:5102 instead of ercot.com? Website Outages maybe?