fskale/gen-moref-op.pl

## gen-moref-op.pl
#!/usr/bin/env perl
package Vmware::Moref::Operations;
use Mojo::Base -base, -signatures, -async_await;
use Mojo::Collection 'c';
use Mojo::DOM;
use Mojo::JSON 'j';
use Mojo::Promise;
use Mojo::URL;
use Mojo::UserAgent;

#
# gen-moref-op.pl - Map managed object references to operations
# / live (concurrent) non blocking using Mojolicious.
#
# Copyright (C) 2020 Franz Skale <i.bin@dah.am>
#
# WCIT
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.

our $VERSION = "1.0";

BEGIN {

  #Windows is untested (feel free to do so and report back)
  die(sprintf("Windows OS not supported !\n")) if $^O =~ /Win/;

  my $modules = {
    q{Future::AsyncAwait} => {minver => q{0.36}},
    Mojolicious           => {minver => q{8.33}}
  };

  my $ev = q{use EV 4.27};

  #EV tested on recent osx(openbsd), linux and freebsd.
  if ($^O =~ qr/(?:linux|darwin|freebsd|openbsd)/i) {
    eval $ev;
    die(sprintf(
      "Please install module EV version %s or higher !\nError:%s\n",
      q{4.27}, $@
    ))
      if $@;
  }

  #Mojolicious and other modules should work with most IX OS !
  foreach my $module (keys %{$modules}) {
    my $minver = $modules->{$module}->{minver};
    eval qq{use $module $minver};
    die(sprintf(
      "Please install module %s version %s or higher !\nError:%s\n",
      $module, $minver, $@
    ))
      if $@;
  }
}

has ua    => sub { Mojo::UserAgent->new->insecure(1); };
has base  => sub { Mojo::URL->new('https://pubs.vmware.com/vi3/sdk/ReferenceGuide'); };
has srv_url => sub { my $base = shift->base; [qq{$base/vim.ServiceInstanceContent.html}]; };
has collection => sub { c(); };
has store => sub { {} };
has concurrency => sub { 6 };

# Initial fetch of tabledata to grab links for further processing
async sub search_vmware_sdk_p ($self, $urls) {
  my ($collection, $error) = (c(), undef);
  for my $url (@{$urls}) {
    my $tx = await $self->ua->get_p($url)->catch(sub($err) { $error = $err; });
    ($tx->result->is_success || (  $tx->code >= 200 && $tx->code < 300 ))
      ? push(@{$collection}, $tx->result->dom->find('td'))
      : warn(sprintf(
      "Error getting sdk base contents: Code: %d Error: %s\n",
      $tx->code, $error ? $error : q{unknown error}
      ));
  }
  return $collection;
}

async sub process($self, $dom) {
  $dom->size or return;
  foreach my $c (keys @{$dom}) {
    $dom->[$c]->map('content')->each(
      sub ($w, $n) {
        my $dom        = Mojo::DOM->new($w);
        my $collection = $dom->find('a');
        if ($collection->size == 2) {
          $dom->find('a')->each(
            sub ($ww, $wn) {
              if ($ww !~ qr/(?:ManagedObjectReference|DynamicData)/i) {
                my $frag = Mojo::DOM->new($ww);
                if (my $url = $frag->at('a')->{href}) {
                  $self->store->{$frag->at('a')->content}->{obj_url} = sprintf((
                    "%s/%s", $self->base->to_string, $frag->at('a')->{href}));
                }
              }
            }
          );
        }
      }
    );
  }
}

async sub moref_to_ops($self) {
  my $obj = shift @{$self->collection};
  $obj or return;
  my $error;
  my $res
    = await $self->ua->get_p($self->store->{$obj}->{obj_url})
    ->catch(sub($err) { $error = $err; })
    ->finally(
    sub { printf(STDERR "non-blocking fetch of object %s done\n", $obj); });
  if ($res->result->is_success ||  ( $res->code >= 200 && $res->code < 300 )) {
    $res->result->dom->find('h1')->map('text')->each(
      sub ($w, $n) {
        $w = $w =~ s/(?:[\r\n])//gr;
        $w = $w =~ s/\s+//gr;
        if (my ($moref) = $w =~ /^(?:ManagedObject-)(.*)$/) {
          printf(STDERR "Debug:\nObject: \"%s\"\n", $moref);
        }
        else {
          printf(STDERR "\tOperation: \"%s\"\n", $w);
          push(@{$self->store->{$obj}->{operations}}, $w);
        }
      }
    );
  }
  else {
    warn(sprintf(
      "Error getting object Code: %s Error: %s\n",
      $res->code, $obj, $error ? $error : q{unknown error}
    ));
  }
  $self->moref_to_ops;
};

async sub get_table($self) {
  my $dom = await $self->search_vmware_sdk_p($self->srv_url);
  await $self->process($dom);
  push(@{$self->collection}, $_) for (keys %{$self->store});
}

sub start($self) {
$self->get_table()->wait;
my @promises = ();
push(@promises, $self->moref_to_ops) for 1 .. $self->concurrency;
Mojo::Promise->all(@promises)->finally(sub { printf("%s\n", j($self->store)) })->wait;
}

package main;
use Mojo::Base -strict;

Vmware::Moref::Operations->new->start;
	#!/usr/bin/env perl
	package Vmware::Moref::Operations;
	use Mojo::Base -base, -signatures, -async_await;
	use Mojo::Collection 'c';
	use Mojo::DOM;
	use Mojo::JSON 'j';
	use Mojo::Promise;
	use Mojo::URL;
	use Mojo::UserAgent;

	#
	# gen-moref-op.pl - Map managed object references to operations
	# / live (concurrent) non blocking using Mojolicious.
	#
	# Copyright (C) 2020 Franz Skale <i.bin@dah.am>
	#
	# WCIT
	#
	# This program is free software; you can redistribute it and/or
	# modify it under the terms of the GNU General Public License
	# as published by the Free Software Foundation; either version 2
	# of the License, or (at your option) any later version.
	#
	# This program is distributed in the hope that it will be useful,
	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	# GNU General Public License for more details.
	#
	# You should have received a copy of the GNU General Public License
	# along with this program; if not, write to the Free Software
	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.

	our $VERSION = "1.0";

	BEGIN {

	#Windows is untested (feel free to do so and report back)
	die(sprintf("Windows OS not supported !\n")) if $^O =~ /Win/;

	my $modules = {
	q{Future::AsyncAwait} => {minver => q{0.36}},
	Mojolicious => {minver => q{8.33}}
	};

	my $ev = q{use EV 4.27};

	#EV tested on recent osx(openbsd), linux and freebsd.
	if ($^O =~ qr/(?:linux\|darwin\|freebsd\|openbsd)/i) {
	eval $ev;
	die(sprintf(
	"Please install module EV version %s or higher !\nError:%s\n",
	q{4.27}, $@
	))
	if $@;
	}

	#Mojolicious and other modules should work with most IX OS !
	foreach my $module (keys %{$modules}) {
	my $minver = $modules->{$module}->{minver};
	eval qq{use $module $minver};
	die(sprintf(
	"Please install module %s version %s or higher !\nError:%s\n",
	$module, $minver, $@
	))
	if $@;
	}
	}

	has ua => sub { Mojo::UserAgent->new->insecure(1); };
	has base => sub { Mojo::URL->new('https://pubs.vmware.com/vi3/sdk/ReferenceGuide'); };
	has srv_url => sub { my $base = shift->base; [qq{$base/vim.ServiceInstanceContent.html}]; };
	has collection => sub { c(); };
	has store => sub { {} };
	has concurrency => sub { 6 };

	# Initial fetch of tabledata to grab links for further processing
	async sub search_vmware_sdk_p ($self, $urls) {
	my ($collection, $error) = (c(), undef);
	for my $url (@{$urls}) {
	my $tx = await $self->ua->get_p($url)->catch(sub($err) { $error = $err; });
	($tx->result->is_success \|\| ( $tx->code >= 200 && $tx->code < 300 ))
	? push(@{$collection}, $tx->result->dom->find('td'))
	: warn(sprintf(
	"Error getting sdk base contents: Code: %d Error: %s\n",
	$tx->code, $error ? $error : q{unknown error}
	));
	}
	return $collection;
	}

	async sub process($self, $dom) {
	$dom->size or return;
	foreach my $c (keys @{$dom}) {
	$dom->[$c]->map('content')->each(
	sub ($w, $n) {
	my $dom = Mojo::DOM->new($w);
	my $collection = $dom->find('a');
	if ($collection->size == 2) {
	$dom->find('a')->each(
	sub ($ww, $wn) {
	if ($ww !~ qr/(?:ManagedObjectReference\|DynamicData)/i) {
	my $frag = Mojo::DOM->new($ww);
	if (my $url = $frag->at('a')->{href}) {
	$self->store->{$frag->at('a')->content}->{obj_url} = sprintf((
	"%s/%s", $self->base->to_string, $frag->at('a')->{href}));
	}
	}
	}
	);
	}
	}
	);
	}
	}

	async sub moref_to_ops($self) {
	my $obj = shift @{$self->collection};
	$obj or return;
	my $error;
	my $res
	= await $self->ua->get_p($self->store->{$obj}->{obj_url})
	->catch(sub($err) { $error = $err; })
	->finally(
	sub { printf(STDERR "non-blocking fetch of object %s done\n", $obj); });
	if ($res->result->is_success \|\| ( $res->code >= 200 && $res->code < 300 )) {
	$res->result->dom->find('h1')->map('text')->each(
	sub ($w, $n) {
	$w = $w =~ s/(?:[\r\n])//gr;
	$w = $w =~ s/\s+//gr;
	if (my ($moref) = $w =~ /^(?:ManagedObject-)(.*)$/) {
	printf(STDERR "Debug:\nObject: \"%s\"\n", $moref);
	}
	else {
	printf(STDERR "\tOperation: \"%s\"\n", $w);
	push(@{$self->store->{$obj}->{operations}}, $w);
	}
	}
	);
	}
	else {
	warn(sprintf(
	"Error getting object Code: %s Error: %s\n",
	$res->code, $obj, $error ? $error : q{unknown error}
	));
	}
	$self->moref_to_ops;
	};

	async sub get_table($self) {
	my $dom = await $self->search_vmware_sdk_p($self->srv_url);
	await $self->process($dom);
	push(@{$self->collection}, $_) for (keys %{$self->store});
	}

	sub start($self) {
	$self->get_table()->wait;
	my @promises = ();
	push(@promises, $self->moref_to_ops) for 1 .. $self->concurrency;
	Mojo::Promise->all(@promises)->finally(sub { printf("%s\n", j($self->store)) })->wait;
	}

	package main;
	use Mojo::Base -strict;

	Vmware::Moref::Operations->new->start;