ERCOT Frozen Grid 2021 - Metrics Reporters

_lib.ts

export * from "./deps.ts";
import {
  DatadogApi,
  MetricSubmission,
  fixedInterval,
} from "./deps.ts";

const datadog = DatadogApi.fromEnvironment(Deno.env);

export function headers(accept = 'text/html') {
  return {
    headers: {
      'Accept': accept,
      'User-Agent': `Deno/${Deno.version} (+https://p.datadoghq.com/sb/5c2fc00be-393be929c9c55c3b80b557d08c30787a)`,
    },
  };
}

export async function runMetricsLoop(
  gather: () => Promise<MetricSubmission[]>,
  intervalMinutes: number,
  loopName: string,
) {
  for await (const dutyCycle of fixedInterval(intervalMinutes * 60 * 1000)) {
    try {

      const data = await gather();

      // Our own loop-health metric
      data.push({
        metric_name: `ercot.app.duty_cycle`,
        points: [{value: dutyCycle*100}],
        tags: [`app:${loopName}`],
        interval: 60,
        metric_type: 'gauge',
      });

      // Submit all metrics
      try {
        await datadog.v1Metrics.submit(data);
      } catch (err) {
        console.log(new Date().toISOString(), 'eh', err.message);
        await datadog.v1Metrics.submit(data);
      }

    } catch (err) {
      console.log(new Date().toISOString(), '!!', err.message);
    }
  }
};

ancillary.ts

// deno run --allow-net --allow-env examples/emit-metrics.ts

import { runMetricsLoop, MetricSubmission, headers, fetch } from "./_lib.ts";
export async function start() {
  await runMetricsLoop(grabUserMetrics, 5, 'ercot_ancillary');
}
if (import.meta.main) start();

async function grabUserMetrics(): Promise<MetricSubmission[]> {
  const body = await fetch('http://127.0.0.1:5102/content/cdr/html/as_capacity_monitor.html', headers('text/html')).then(x => x.text());

  const sections = body.split('an="2">').slice(1);
  const metrics = new Array<MetricSubmission>();
  for (const section of sections) {
    const label = section.slice(0, section.indexOf('<'));
    const boxes = section.match(/    <td class="tdLeft">[^<]+<\/td>\r\n    <td class="labelClassCenter">[^<]+<\/td>/g) ?? [];
    for (const box of boxes) {
      const parts = box.split(/[<>]/);
      const field = parts[2]
        .replace(/Controllable Load Resource/g, 'CLR')
        .replace(/Load Resource/g, 'LR')
        .replace(/Generation Resource/g, 'GR')
        .replace(/Energy Offer Curve/g, 'EOC')
        .replace(/Output Schedule/g, 'OS')
        .replace(/Base Point/g, 'BP')
        .replace(/Resource Status/g, 'RS')
        .replace(/ \(energy consumption\)/g, '')
        .replace(/telemetered/g, 'TMd')
        .replace(/Fast Frequency Response/g, 'FFR')
        .replace(/available to decrease/g, 'to decr')
        .replace(/available to increase/g, 'to incr')
        .replace(/in the next 5 minutes/g, 'in 5min')
        .replace(/Physical Responsive Capability \(PRC\)/g, 'PRC')
        .replace(/^Real-Time /, '')
        .replace(/[ ()-]+/g, ' ').trim().replace(/ /g, '_');
      // console.log(label, field, parts[6]);
      metrics.push({
        metric_name: `ercot_ancillary.${field}`,
        points: [{value: parseFloat(parts[6].replace(/,/g, ''))}],
        interval: 60,
        metric_type: 'gauge',
      });
    }
  }

  console.log(new Date, 'ancillary', metrics
    .find(x => x.metric_name.endsWith('PRC'))
    ?.points[0]?.value);

  return metrics;
}

deps.ts

export { default as DatadogApi } from "https://deno.land/x/datadog_api@v0.1.3/mod.ts";
export type { MetricSubmission } from "https://deno.land/x/datadog_api@v0.1.3/v1/metrics.ts";

export { fixedInterval } from "https://crux.land/4MC9JG#fixed-interval@v1";

export { Sha256 } from "https://deno.land/std@0.95.0/hash/sha256.ts";

export { runMetricsServer } from "https://deno.land/x/observability@v0.1.0/sinks/openmetrics/server.ts";
export { replaceGlobalFetch, fetch } from "https://deno.land/x/observability@v0.1.0/sources/fetch.ts";

Dockerfile

FROM hayd/alpine-deno:1.10.1
WORKDIR /src/app

ADD deps.ts ./
RUN ["deno", "cache", "deps.ts"]

ADD *.ts ./
RUN ["deno", "cache", "mod.ts"]

ENTRYPOINT ["deno", "run", "--unstable", "--allow-net", "--allow-hrtime", "--allow-env", "--cached-only", "--no-check", "mod.ts"]

eea.ts

// deno run --allow-net --allow-env examples/emit-metrics.ts

import { runMetricsLoop, MetricSubmission, headers, fetch } from "./_lib.ts";
export async function start() {
  await runMetricsLoop(grabUserMetrics, 10, 'ercot_eea');
}
if (import.meta.main) start();

async function grabUserMetrics(): Promise<MetricSubmission[]> {
  const body = await fetch(`http://127.0.0.1:5102/content/alerts/conservation_state.js`, headers('application/javascript')).then(x => x.text());

  const line = body.split(/\r?\n/).find(x => x.startsWith('eeaLevel = '));
  if (!line) {
    console.log(new Date, 'EEA Unknown');
    return [];
  }

  const level = parseInt(line.split('=')[1].trim());
  console.log(new Date, 'EEA Level', level);

  return [{
    metric_name: `ercot.eea_level`,
    points: [{value: level}],
    interval: 60*10,
    metric_type: 'gauge',
  }];
}

grid.ts

// deno run --allow-net --allow-env examples/emit-metrics.ts

import { runMetricsLoop, MetricSubmission, headers, fetch } from "./_lib.ts";
export async function start() {
  await runMetricsLoop(grabUserMetrics, 1, 'ercot_realtime');
}
if (import.meta.main) start();

async function grabUserMetrics(): Promise<MetricSubmission[]> {
  const body = await fetch('http://127.0.0.1:5102/content/cdr/html/real_time_system_conditions.html', headers('text/html')).then(x => x.text());

  const sections = body.split('an="2">').slice(1);
  const metrics = new Array<MetricSubmission>();
  for (const section of sections) {
    const label = section.slice(0, section.indexOf('<'));
    const boxes = section.match(/    <td class="tdLeft">[^<]+<\/td>\r\n    <td class="labelClassCenter">[^<]+<\/td>/g) ?? [];
    for (const box of boxes) {
      const parts = box.split(/[<>]/);
      // console.log(label, parts[2], parts[6]);
      if (label === 'DC Tie Flows') {
        metrics.push({
          metric_name: `ercot.${label}`.replace(/[ -]+/g, '_'),
          tags: [`ercot_dc_tie:${parts[2].split('(')[0].trim()}`],
          points: [{value: parseFloat(parts[6])}],
          interval: 60,
          metric_type: 'gauge',
        });
      } else {
        metrics.push({
          metric_name: `ercot.${label}.${parts[2].split('(')[0].trim()}`.replace(/[ -]+/g, '_'),
          points: [{value: parseFloat(parts[6])}],
          interval: 60,
          metric_type: 'gauge',
        });
      }
    }
  }

  console.log(new Date, 'grid', metrics[0]?.points[0]?.value);

  return metrics;
}

LICENSE

Licensed under the MIT license:

    http://www.opensource.org/licenses/mit-license.php

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

metar.ts

// deno run --allow-net --allow-env examples/emit-metrics.ts

const knownTexts = new Map<string,string>();

// https://www.faa.gov/air_traffic/weather/asos/?state=TX
// https://en.wikipedia.org/wiki/List_of_power_stations_in_Texas#Wind_farms
const ids = [
  'KABI', // Abilene (near Roscoe Wind Farm)
  'KAUS',
  'KDFW',
  'KEFD', // Houston/Ellington Ar
  'KGLS', // Galveston/Scholes In
  'KHOU', // Houston/Hobby Arpt
  'KIAH',
  'KLBX', // Angleton/Texas Gulf
  'KLRD', // Laredo (nearish Javelina Wind Energy Center)
  'KLVJ', // Houston/Pearland Rgn
  'KMAF',
  'KSAT',
  'KSGR', // Houston/Sugar Land R
  'KTKI',
];

import { runMetricsLoop, MetricSubmission, headers, fetch } from "./_lib.ts";
export async function start() {
  await runMetricsLoop(grabUserMetrics, 30, 'metar');
}
if (import.meta.main) start();

async function grabUserMetrics(): Promise<MetricSubmission[]> {
  const body = await fetch(`https://www.aviationweather.gov/metar/data?ids=${ids.join('%2C')}&format=decoded`, headers('text/html')).then(resp => resp.text());

  const sections = body.split(/<!-- Data (?:starts|ends) here -->/)[1].split(`METAR for:</span></td><td>`).slice(1);

  const stations = new Array<MetricSubmission[]>();
  for (const sect of sections) {
    const title = sect.slice(0, sect.indexOf('<'));
    const code = title.split(' ')[0];
    const name = title.split(/[\(\)]/)[1].replace(/[ ,]+/g, '_');
    const metrics = new Array<MetricSubmission>();
    let text = '';
    for (const row of sect.match(/>[^<]+<\/span><\/td><td[^>]*>[^<]+<\/td>/g) ?? []) {
      const cells = row.split(/[<>]/);
      const metric = cells[1].split(/[:(]/)[0].toLowerCase().trim();
      const value = cells[7].trim();
      // console.log([code, name, cells[1].slice(0, -1), cells[7]]);

      if (metric === 'text') {
        text = value;
      }

      if ([
        'temperature',
        'dewpoint',
        'pressure',
      ].includes(metric)) {
        metrics.push({
          metric_name: `metar.${metric}`,
          tags: [
            `metar_code:${code}`,
            `metar_location:${name}`,
          ],
          points: [{value: parseFloat(value)}],
          interval: 60,
          metric_type: 'gauge',
        });
      }

      if (metric === 'winds' && value.includes('MPH')) {
        const [speed, gusts] = value.match(/([0-9.]+) MPH/g) ?? [];
        if (speed) metrics.push({
          metric_name: `metar.winds.speed`,
          tags: [
            `metar_code:${code}`,
            `metar_location:${name}`,
          ],
          points: [{value: parseFloat(speed)}],
          interval: 60,
          metric_type: 'gauge',
        });
        if (gusts) metrics.push({
          metric_name: `metar.winds.gusts`,
          tags: [
            `metar_code:${code}`,
            `metar_location:${name}`,
          ],
          points: [{value: parseFloat(gusts)}],
          interval: 60,
          metric_type: 'gauge',
        });
      }
    }

    // console.log(code, text, knownTexts.get(code), metrics.length);
    if (!text) continue;
    if (knownTexts.get(code) === text) continue;
    stations.push(metrics);
    knownTexts.set(code, text);
  }
  console.log(new Date, 'METAR', (stations[0] ?? [])[0]?.tags);
  return stations.flat();
}

mod.ts

import { start as startAncillary } from "./ancillary.ts";
import { start as startEea } from "./eea.ts";
import { start as startGrid } from "./grid.ts";
import { start as startMetar } from "./metar.ts";
import { start as startOutages } from "./outages.ts";
import { start as startPrices } from "./prices.ts";

import {
  runMetricsServer, replaceGlobalFetch,
} from './deps.ts';
if (Deno.args.includes('--serve-metrics')) {
  runMetricsServer({ port: 9090 });
  console.log("Now serving OpenMetrics @ :9090/metrics");
}

if (import.meta.main) {
  await Promise.race([
    // 60s loops
    // run these offset from each other for better utilization
    startGrid(),
    new Promise(ok => setTimeout(ok, 30*1000)).then(startAncillary),

    // 10+ minute loops, they can overlap, it's ok
    startEea(),
    startMetar(),
    startOutages(),
    startPrices(),
  ]);
}

outages.ts

// deno run --allow-net --allow-env examples/emit-metrics.ts

import { Sha256 } from "./deps.ts";
let lastHash = '';

import { runMetricsLoop, MetricSubmission, headers, fetch } from "./_lib.ts";
export async function start() {
  await runMetricsLoop(grabUserMetrics, 30, 'poweroutages_us');
}
if (import.meta.main) start();

async function grabUserMetrics(): Promise<MetricSubmission[]> {
  const bodyText = await fetch(`https://poweroutage.us/api/web/counties?key=18561563181588&countryid=us&statename=Texas`, headers('application/json')).then(resp => resp.text());

  const hash = new Sha256().update(bodyText).hex().slice(0, 12);
  if (hash === lastHash) {
    console.log(new Date, 'Outages', hash);
    return [];
  }
  lastHash = hash;

  const body = JSON.parse(bodyText) as {
    WebCountyRecord: {
      CountyName: string;
      OutageCount: number;
      CustomerCount: number;
    }[];
  };

  console.log(new Date, 'Outages', hash, body.WebCountyRecord[0].CountyName, body.WebCountyRecord[0].OutageCount);
  return body.WebCountyRecord.flatMap(x => [{
    metric_name: `poweroutageus.outages`,
    tags: [
      `county_name:${x.CountyName}`,
      `county_state:Texas`,
    ],
    points: [{value: x.OutageCount}],
    interval: 60,
    metric_type: 'gauge',
  }, {
    metric_name: `poweroutageus.customers`,
    tags: [
      `county_name:${x.CountyName}`,
      `county_state:Texas`,
    ],
    points: [{value: x.CustomerCount}],
    interval: 60,
    metric_type: 'gauge',
  }]);
}

prices.ts

// deno run --allow-net --allow-env examples/emit-metrics.ts

import { runMetricsLoop, MetricSubmission, headers, fetch } from "./_lib.ts";
export async function start() {
  await waitForNextPrices();
  await runMetricsLoop(grabUserMetrics, 15, 'ercot_pricing');
}
if (import.meta.main) start();

async function grabUserMetrics(): Promise<MetricSubmission[]> {
  const body = await fetch(`http://127.0.0.1:5102/content/cdr/html/real_time_spp`, headers('text/html')).then(x => x.text());

  const sections = body.split('</table>')[0].split('<tr>').slice(1).map(x => x.split(/[<>]/).filter((_, idx) => idx % 4 == 2));
  const header = sections[0]?.slice(2, -1) ??[];
  const last = sections[sections.length-1]?.slice(2, -1) ??[];

  const timestamp = sections[sections.length-1][1];
  console.log(new Date, 'Prices', timestamp, header[0], last[0]);

  return header.map((h, idx) => {
    return {
      metric_name: `ercot.pricing`,
      tags: [`ercot_region:${h}`],
      points: [{value: parseFloat(last[idx])}],
      interval: 60*15,
      metric_type: 'gauge',
    };
  });
}

// launches this script 2m30s after the 15-minute mark for most-timely data
async function waitForNextPrices() {
  const startDate = new Date();
  while (startDate.getMinutes() % 15 !== 2) {
    startDate.setMinutes(startDate.getMinutes()+1);
  }
  startDate.setSeconds(30);
  startDate.setMilliseconds(0);

  const waitMillis = startDate.valueOf() - Date.now();
  if (waitMillis > 0) {
    console.log(`Waiting ${waitMillis/1000/60}min for next pricing cycle`);
    await new Promise(ok => setTimeout(ok, waitMillis));
  }
}

Was there a reason to use 127.0.0.1:5102 instead of ercot.com? Website Outages maybe?

Was there a reason to use 127.0.0.1:5102 instead of ercot.com?

Yes, and it's dumb. The ercot web server includes a pointless folded http header (one header spanning multiple lines) and because that is deprecated in modern http, the runtime I run this with (Deno) considers the response invalid.

So the localhost server is a golang proxy that drops all set-cookies on the floor. Originally i ran curl as a subprocess but had some stability issues after long uptimes.

This isn't a problem under Node, just Deno. Upstream considered it a bug (on Discord) but i didn't file an issue.

This appears to have stopped working as of 5/11/2022. Did ERCOT change something on the backend to prevent the data polling?

@texastoastt looks like the source of the data hasn't changed: http://www.ercot.com/content/cdr/html/real_time_system_conditions.html

OP's server might of stopped fetching it or something on the datadog side

It was DNS 🥲

ERCOT blocks European traffic so I can't run this from home, I have it on a US-based VPS. And it's not noticeable when something goes wrong there.

I fixed the server. Thanks for the heads up