This is a hack Perl script to scrape data from the Ambient Weather ObserverIP "Live Data" web interface, process and merge data from the NOAA Aviation Weather ADDS Text Data Server and the Ambient Weather API, then dump the whole mess to JSON that can be used in conjunction with home automation APIs (e.g. SmartThings Device Handler) to provide I…

apwsd.pl

#!/usr/bin/perl
#
# PWS/METAR Polling Daemon II
# ==================================================
# Modified: Fri Nov 29 15:35:26 2019 -0600
# Authored: Mark Page [m.e.page@gmail.com]
#  Version: 19.11.29.15
#
# Run this script via cron
#
# This is a hack Perl script to scrape data from the Ambient Weather ObserverIP
# "Live Data" web interface, process and merge data from the NOAA Aviation Weather
# ADDS Text Data Server and the Ambient Weather API. Updates may also be pushed to
# a Weather Underground station page.
#
# This script also outputs JSON that can be used in conjunction with home automation
# APIs (e.g. SmartThings Device Handler) to provide Illuminance, Relative Humidity,
# Temperature, and Ultraviolet Index data via a web server (Apache, NGINX, etc.).
#
# It's really ugly but it gets the job done =)
#
# For more info:
#
#   https://www.ambientweather.com/amws1400ip.html
#   https://www.aviationweather.gov/dataserver
#   https://aviationweather.gov/metar/info
#
#
use strict;
use warnings;

use LWP::Simple qw(!head);
use LWP::UserAgent;
use HTTP::Request;
use Time::HiRes;
use JSON;
use Data::Dumper;
    $Data::Dumper::Indent = 1;
    $Data::Dumper::Purity = 1;
    $Data::Dumper::Terse  = 1;

my $start = Time::HiRes::gettimeofday;
my  $json = JSON->new->allow_nonref;
    $json = $json->pretty();

#############################################################################################################################################################
# You'll need to change the following...
#############################################################################################################################################################
#
my $pws_ip    = 'xxx.xxx.xxx.xxx';                                                  # Internal (LAN) IP address of ObserverIP device
my $api_mac   = 'XX:XX:XX:XX:XX:XX';                                                # ObserverIP MAC Address for Ambient Weather API
my $api_key   = 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx', # Ambient Weather API Key
my $api_akey  = 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx', # Ambient Weather API Application Key
my $wu_id     = 'XXXXXXXXX';                                                        # Weather Underground ID (blank this value to bypass)
my $wu_pass   = 'XXXXXXXX';                                                         # Weather Underground Password (blank this value to bypass)
my $metar_loc = 'XXXX';                                                             # Local METAR Station ID (see https://aviationweather.gov/docs/metar/stations.txt)
my $altitude  = '[+/-] xx.xxx';                                                     # Local Altitude (e.g., -1.833)
my $latitude  = '[+/-] xx.xxx';                                                     # Local Latitude (e.g., +34.504)
my $longitude = '[+/-] xx.xxx';                                                     # Local Longitude (e.g., -92.575)
my $timezone  = 'Country/City';                                                     # Country/City (e.g., America/Chicago)
my $ignore    = 'ip|outdoor2id|indoorid|currtime|outdoor1id';                       # ObserverIP fields to ignore (should be fine as is)
my $json_out  = '/var/www/path/to/public.json';                                     # Server file path to publicly accessible JSON (over HTTP/HTTPS)
#
#############################################################################################################################################################

my ($_data);
my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);

$_data->{'app'}->{'updated'}       = time;
$_data->{'app'}->{'updated_long'}  = localtime(time);
$_data->{'app'}->{'updated_short'} = sprintf("%02d",$hour).':'.sprintf("%02d",$min).':'.sprintf("%02d",$sec).' '.sprintf("%02d",($mon+1)).'/'.sprintf("%02d",$mday).'/'.($year+1900);
$_data->{'app'}->{'source'}        = 'AW_API';

$_data->{'pws'}   = get_awapi();
$_data->{'metar'} = get_metar();
$_data->{'local'} = get_local();

if (defined($_data->{'pws'}->{'error'}) and $_data->{'pws'}->{'error'} eq 'AWAPI_OK') {
  $_data->{'pws'}->{'monthlyrainin'}  = $_data->{'local'}->{'rainofmonthly'};
  $_data->{'pws'}->{'humidityin'}     = $_data->{'local'}->{'inhumi'};
  $_data->{'pws'}->{'uv'}             = $_data->{'local'}->{'uvi'};
  $_data->{'pws'}->{'tempf'}          = $_data->{'local'}->{'outtemp'};
  $_data->{'pws'}->{'tempinf'}        = $_data->{'local'}->{'intemp'};
  $_data->{'pws'}->{'maxdailygust'}   = $_data->{'local'}->{'avgwind'};
  $_data->{'pws'}->{'windgustmph'}    = $_data->{'local'}->{'gustspeed'};
  $_data->{'pws'}->{'dailyrainin'}    = $_data->{'local'}->{'rainofdaily'};
  $_data->{'pws'}->{'totalrainin'}    = $_data->{'local'}->{'rainofyearly'};
  $_data->{'pws'}->{'date'}           = $_data->{'app'}->{'updated_long'};
  $_data->{'pws'}->{'baromabsin'}     = $_data->{'local'}->{'abspress'};
  $_data->{'pws'}->{'solarradiation'} = $_data->{'local'}->{'solarrad'};
  $_data->{'pws'}->{'dateutc'}        = $_data->{'app'}->{'updated'} * 1000;
  $_data->{'pws'}->{'hourlyrainin'}   = $_data->{'local'}->{'rainofhourly'};
  $_data->{'pws'}->{'humidity'}       = $_data->{'local'}->{'outhumi'};
  $_data->{'pws'}->{'dewPoint'}       = $_data->{'local'}->{'dewpoint'};
  $_data->{'pws'}->{'weeklyrainin'}   = $_data->{'local'}->{'rainofweekly'};
  $_data->{'pws'}->{'baromrelin'}     = $_data->{'local'}->{'relpress'};
  $_data->{'pws'}->{'eventrainin'}    = $_data->{'local'}->{'rainofhourly'};
  $_data->{'pws'}->{'windspeedmph'}   = $_data->{'local'}->{'avgwind'};
  $_data->{'pws'}->{'lastRain'}       = 'unknown';
  $_data->{'pws'}->{'feelsLike'}      = $_data->{'local'}->{'apptemp'};
  $_data->{'pws'}->{'winddir'}        = $_data->{'local'}->{'windir'};
  $_data->{'pws'}->{'source'}         = 'LOCAL_PWS';
}

if ($wu_id ne "" and $wu_pass ne "") {
  my $wu_url  = 'https://weatherstation.wunderground.com/weatherstation/updateweatherstation.php?';
     $wu_url .= 'ID='.$wu_id.'&';
     $wu_url .= 'PASSWORD='.$wu_pass.'&';
     $wu_url .= 'dateutc=now&';
     $wu_url .= 'softwaretype=PerlWeather v0.12 (c)2016 Very3&';
     $wu_url .= 'action=updateraw&';
     $wu_url .= 'winddir='.$_data->{'pws'}->{'winddir'}.'&';
     $wu_url .= 'windspeedmph='.$_data->{'pws'}->{'windspeedmph'}.'&';
     $wu_url .= 'windgustmph='.$_data->{'pws'}->{'windgustmph'}.'&';
     $wu_url .= 'tempf='.$_data->{'pws'}->{'tempf'}.'&';
     $wu_url .= 'dewptf='.$_data->{'pws'}->{'dewPoint'}.'&';
     $wu_url .= 'humidity='.$_data->{'pws'}->{'humidity'}.'&';
     $wu_url .= 'baromin='.$_data->{'pws'}->{'baromabsin'}.'&';
     $wu_url .= 'rainin='.$_data->{'pws'}->{'hourlyrainin'}.'&';
     $wu_url .= 'dailyrainin='.$_data->{'pws'}->{'dailyrainin'}.'&';
     $wu_url .= 'clouds='.$_data->{'metar'}->{'sky_condition'}.'&';
     $wu_url .= 'solarradiation='.$_data->{'pws'}->{'solarradiation'}.'&';
     $wu_url .= 'visibility='.$_data->{'metar'}->{'visibility_statute_mi'}.'&';
     $wu_url .= 'weather='.$_data->{'metar'}->{'raw_text'};

  chomp($_data->{'app'}->{'wu_data'} = LWP::Simple::get($wu_url));
}

$_data->{'pws'}->{'sunrise'} = sun()->{'rise'};
$_data->{'pws'}->{'sunset'}  = sun()->{'set'};
$_data->{'app'}->{'runtime'} = (Time::HiRes::gettimeofday - $start);

# When is a number not a number? When it's a string =)
while (my ($k1,$v1) = each(%{$_data})) {
  while (my ($k2,$v2) = each(%{$_data->{$k1}})) {
    if ($_data->{$k1}->{$k2} !~ m/[A-Za-z:]/) {
      $_data->{$k1}->{$k2} = ($_data->{$k1}->{$k2} * 1);
    }
  }
}

open(my $json_file,'>',"$json_out");
  print $json_file $json->encode($_data);
close($json_file);

print localtime(time).": $json_out updated ($_data->{'app'}->{'runtime'}s)\n";

exit;


###################################################################################
# Subs/Functions
###################################################################################

# Scrape ObserverIP "Live Data" Page
sub get_local {
  use WWW::Mechanize;

  my ($_return);
  my $_pws_url = 'http://'.$pws_ip.'/livedata.htm';

  (my $_mech = WWW::Mechanize->new(cookie_jar => undef, autocheck => 0))->get($_pws_url);
  if ($_mech->success() and $_mech->{'content'} =~ m/LiveData/) {
    my @_inputs = $_mech->find_all_inputs(type => 'text',);
    for my $i (0 .. (scalar(@_inputs) - 1)) {
      if ($_inputs[$i]->{'name'} !~ m/$ignore/i) {
        if ($_inputs[$i]->{'value'} !~ m/\.|[0-9]/i) {
          $_inputs[$i]->{'value'} = 0;
        }
        $_return->{lc($_inputs[$i]->{'name'})} = $_inputs[$i]->{'value'};
      }
    }

    # Dewpoint Calc
    my $_dp = 243.04*(log($_return->{'outhumi'}/100)+((17.625*$_return->{'outtemp'})/(243.04+$_return->{'outtemp'})))/(17.625-log($_return->{'outhumi'}/100)-((17.625*$_return->{'outtemp'})/(243.04+$_return->{'outtemp'})));
    $_return->{'dewpoint'} = sprintf("%.2f",$_dp);

    # HeatIndex Calc
    $_return->{'heatindex'} = $_return->{'outtemp'};
    if ($_return->{'outtemp'} >= 60) {
      $_return->{'heatindex'} = sprintf("%.2f",-42.379+2.04901523*$_return->{'outtemp'}+10.14333127*$_return->{'outhumi'}-0.22475541*$_return->{'outtemp'}*$_return->{'outhumi'}-(6.83783*10**(-3))*($_return->{'outtemp'}**(2))-(5.481717 * 10**(-2))*($_return->{'outhumi'}**(2))+(1.22874*10**(-3))*($_return->{'outtemp'}**(2))*$_return->{'outhumi'}+(8.5282*10**(-4))*$_return->{'outtemp'}*($_return->{'outhumi'}**(2))-(1.99*10**(-6))*($_return->{'outtemp'}**(2))*($_return->{'outhumi'}**(2)));
    }

    # Apparent Temp Calc
    if ($_return->{'outhumi'} > 96) {
      $_return->{'apptemp'} = sprintf("%.2f",$_return->{'outtemp'});
    }
    else {
      my $_tc  = ($_return->{'outtemp'} - 32) * (5/9);                                  # Convert degrees F to C
      my $_hp  = ($_return->{'outhumi'}/100) * 6.105 * exp(17.27 * $_tc/(237.7+$_tc));  # Dry bulb humidity calc
      my $_ws  = ($_return->{'avgwind'}*0.447) * 0.27777;                               # Avg wind / wind gust calc
      my $_atc = $_tc + (0.33 * $_hp) - (0.7 * $_ws) - 4;                               # Apparent Temperature calc
      $_return->{'apptemp'} = sprintf("%.2f",$_atc*(9/5)+32);                           # Convert degrees C to F
    }

  }
  else {
    $_return->{'error'} = 'LOCAL_ERROR';
  }

  # Hack for humidity errors from METAR
  if ($_return->{'outtemp'} == 1 and $_return->{'outhumi'} == 1) {
    $_data->{'app'}->{'source'} = 'METAR';
    $_return->{'outtemp'} = (($_data->{'metar'}->{'temp_c'} * 1.8) + 32);
    $_return->{'outhumi'} = int(100*(exp((17.625*$_data->{'metar'}->{'dewpoint_c'})/(243.04+$_data->{'metar'}->{'dewpoint_c'}))/exp((17.625*$_data->{'metar'}->{'temp_c'})/(243.04+$_data->{'metar'}->{'temp_c'}))));
  }

  return $_return;
}


# Get Ambient Weather API Data (yes, we're grabbing our own WS data ;-))
sub get_awapi {
  use URI::Escape;

  my ($_return);
  $_return->{'error'} = 'AWAPI_OK';

  my $api_url  = 'https://api.ambientweather.net/v1/devices/'.uri_escape($api_mac).'?apiKey='.$api_key.'&applicationKey='.$api_akey;
  my $ua       = LWP::UserAgent->new(ssl_opts => { verify_hostname => 1 });
  my $header   = HTTP::Request->new(GET => $api_url);
  my $request  = HTTP::Request->new('GET', $api_url, $header);
  my $response = $ua->request($request);

  if ($response->is_success) {
    my $_pwsdata = $json->decode($response->content);

    while (my ($k,$v) = each(%{$_pwsdata->[0]})) {
      if ($k !~ m/$ignore/i) {
        $_return->{$k} = $v;
      }
    }

    $_return->{'dewPoint'}  = sprintf("%.2f",$_return->{'dewPoint'});
    $_return->{'feelsLike'} = sprintf("%.2f",$_return->{'feelsLike'});
  }
  else {
    $_return->{'error'} = 'AWAPI_ERROR';
  }

  return $_return;
}


# Get METAR Data
sub get_metar {
  my ($_return);

  my $metar_url = 'https://www.aviationweather.gov/adds/dataserver_current/httpparam?dataSource=metars&requestType=retrieve&format=csv&stationString='.$metar_loc.'&hoursBeforeNow=1&mostRecentForEachStation=constraint';
  my $sky_cond  = {
    'CLR' => 'Clear',
    'NSC' => 'Clear',
    'SKC' => 'Clear',
    'BKN' => 'Broken',
    'FEW' => 'Few',
    'SCT' => 'Scattered',
    'OVC' => 'Overcast',
    'CB'  => 'Cumulonimbus',
    'TCU' => 'Towering Cumulus',
    'OVX' => 'Obscured',
  };

  my $ua       = LWP::UserAgent->new(ssl_opts => { verify_hostname => 1 });
  my $header   = HTTP::Request->new(GET => $metar_url);
  my $request  = HTTP::Request->new('GET', $metar_url, $header);
  my $response = $ua->request($request);

  if ($response->is_success) {
    chomp(my @_metar_raw = split("\n",$response->content));

    if (!defined($_metar_raw[6])) {
      print localtime(time).": Malformed METAR return! (not uncommon, try a different source.)\n";

      $_return->{'sky_condition'}         = 'none';
      $_return->{'visibility_statute_mi'} = 'none';
      $_return->{'raw_text'}              = 'none';
      $_return->{'error'}                 = 'METAR_ERROR';

      return $_return;
    }

    my @_fields = split(/\,/,$_metar_raw[5]);
    my @_values = split(/\,/,$_metar_raw[6]);
    for my $_f (0 .. (scalar(@_fields) - 1)) {
      if ($_values[$_f]) {
        if ($_values[$_f] !~ m/[A-Z]|[a-z]/i) {
          $_return->{$_fields[$_f]} = ($_values[$_f] * 1);
        }
        else {
          $_return->{$_fields[$_f]} = $_values[$_f];
        }
      }
    }
    $_return->{'sky_condition'} = $sky_cond->{$_return->{'sky_cover'}};
    $_return->{'raw_text'} =~ s/$metar_loc/VRY3/;
  }
  else {
    $_return->{'error'} = 'METAR_ERROR';
    $_return->{'sky_condition'} = 'N/A';
    $_return->{'visibility_statute_mi'} = 0;
    $_return->{'raw_text'} = 'N/A';
  }

  return $_return;
}


# Sunrise / Sunset Calc
sub sun {
  use DateTime;
  use DateTime::Event::Sunrise;
  use Date::Parse;

  my ($_return);
  my ($args) = @_;

  my $now = time;

  my $date      = DateTime->now(time_zone => $timezone);
  my $riseset   = DateTime::Event::Sunrise->new('longitude' => $longitude, 'latitude' => $latitude, 'altitude' => $altitude,);
  my $span      = $riseset->sunrise_sunset_span($date);

  $_return->{'date_raw'} = $date;
  $_return->{'rise_raw'} = $span->start->datetime;
  $_return->{'set_raw'}  = $span->end->datetime;

  $_return->{'rise'} = str2time($span->start->datetime);
  $_return->{'set'}  = str2time($span->end->datetime);

  if ($now >= $_return->{'rise'} or $now <= $_return->{'set'}) {
    $_return->{'isnight'} = 'false';
  }

  if ($now <= $_return->{'rise'} or $now >= $_return->{'set'}) {
    $_return->{'isnight'} = 'true';
  }

  return $_return;
}


__END__

# vim: set ts=2 sw=2 tw=0 et :

Morning,

Can this script be changed to out in text format as pwsWDxx weather page is looking for the file format in aother way?

Thank you
Quaky Carl

Hey Carl, I'm not familiar with that format. Do you have a reference sheet or maybe a link to the format spec? Mark

…

On Fri, Jun 5, 2020, 02:38 Quark0ne ***@***.***> wrote: ***@***.**** commented on this gist. ------------------------------ Morning, Can this script be changed to out in text format as pwsWDxx weather page is looking for the file format in aother way? Thank you Quaky Carl — You are receiving this because you authored the thread. Reply to this email directly, view it on GitHub <https://gist.github.com/3a53c1c6df78766862e45601acbeca2e#gistcomment-3330628>, or unsubscribe <https://github.com/notifications/unsubscribe-auth/AACKW6ZN5QEYXA2HITQP4DTRVCOG5ANCNFSM4NTJWQFA> .