Skip to content

Instantly share code, notes, and snippets.

@fskale
Last active March 24, 2020 10:27
Show Gist options
  • Save fskale/30b8833f4d7a7e196689f776acf3e566 to your computer and use it in GitHub Desktop.
Save fskale/30b8833f4d7a7e196689f776acf3e566 to your computer and use it in GitHub Desktop.
async/await/non-blocking I/O demo (Vmware MOREF -> operations JSON mapper - concurrency via recursive queuing - live fetching)
#!/usr/bin/env perl
package Vmware::Moref::Operations;
use Mojo::Base -base, -signatures, -async_await;
use Mojo::Collection 'c';
use Mojo::DOM;
use Mojo::JSON 'j';
use Mojo::Promise;
use Mojo::URL;
use Mojo::UserAgent;
#
# gen-moref-op.pl - Map managed object references to operations
# / live (concurrent) non blocking using Mojolicious.
#
# Copyright (C) 2020 Franz Skale <i.bin@dah.am>
#
# WCIT
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
our $VERSION = "1.0";
BEGIN {
#Windows is untested (feel free to do so and report back)
die(sprintf("Windows OS not supported !\n")) if $^O =~ /Win/;
my $modules = {
q{Future::AsyncAwait} => {minver => q{0.36}},
Mojolicious => {minver => q{8.33}}
};
my $ev = q{use EV 4.27};
#EV tested on recent osx(openbsd), linux and freebsd.
if ($^O =~ qr/(?:linux|darwin|freebsd|openbsd)/i) {
eval $ev;
die(sprintf(
"Please install module EV version %s or higher !\nError:%s\n",
q{4.27}, $@
))
if $@;
}
#Mojolicious and other modules should work with most IX OS !
foreach my $module (keys %{$modules}) {
my $minver = $modules->{$module}->{minver};
eval qq{use $module $minver};
die(sprintf(
"Please install module %s version %s or higher !\nError:%s\n",
$module, $minver, $@
))
if $@;
}
}
has ua => sub { Mojo::UserAgent->new->insecure(1); };
has base => sub { Mojo::URL->new('https://pubs.vmware.com/vi3/sdk/ReferenceGuide'); };
has srv_url => sub { my $base = shift->base; [qq{$base/vim.ServiceInstanceContent.html}]; };
has collection => sub { c(); };
has store => sub { {} };
has concurrency => sub { 6 };
# Initial fetch of tabledata to grab links for further processing
async sub search_vmware_sdk_p ($self, $urls) {
my ($collection, $error) = (c(), undef);
for my $url (@{$urls}) {
my $tx = await $self->ua->get_p($url)->catch(sub($err) { $error = $err; });
($tx->result->is_success || ( $tx->code >= 200 && $tx->code < 300 ))
? push(@{$collection}, $tx->result->dom->find('td'))
: warn(sprintf(
"Error getting sdk base contents: Code: %d Error: %s\n",
$tx->code, $error ? $error : q{unknown error}
));
}
return $collection;
}
async sub process($self, $dom) {
$dom->size or return;
foreach my $c (keys @{$dom}) {
$dom->[$c]->map('content')->each(
sub ($w, $n) {
my $dom = Mojo::DOM->new($w);
my $collection = $dom->find('a');
if ($collection->size == 2) {
$dom->find('a')->each(
sub ($ww, $wn) {
if ($ww !~ qr/(?:ManagedObjectReference|DynamicData)/i) {
my $frag = Mojo::DOM->new($ww);
if (my $url = $frag->at('a')->{href}) {
$self->store->{$frag->at('a')->content}->{obj_url} = sprintf((
"%s/%s", $self->base->to_string, $frag->at('a')->{href}));
}
}
}
);
}
}
);
}
}
async sub moref_to_ops($self) {
my $obj = shift @{$self->collection};
$obj or return;
my $error;
my $res
= await $self->ua->get_p($self->store->{$obj}->{obj_url})
->catch(sub($err) { $error = $err; })
->finally(
sub { printf(STDERR "non-blocking fetch of object %s done\n", $obj); });
if ($res->result->is_success || ( $res->code >= 200 && $res->code < 300 )) {
$res->result->dom->find('h1')->map('text')->each(
sub ($w, $n) {
$w = $w =~ s/(?:[\r\n])//gr;
$w = $w =~ s/\s+//gr;
if (my ($moref) = $w =~ /^(?:ManagedObject-)(.*)$/) {
printf(STDERR "Debug:\nObject: \"%s\"\n", $moref);
}
else {
printf(STDERR "\tOperation: \"%s\"\n", $w);
push(@{$self->store->{$obj}->{operations}}, $w);
}
}
);
}
else {
warn(sprintf(
"Error getting object Code: %s Error: %s\n",
$res->code, $obj, $error ? $error : q{unknown error}
));
}
$self->moref_to_ops;
};
async sub get_table($self) {
my $dom = await $self->search_vmware_sdk_p($self->srv_url);
await $self->process($dom);
push(@{$self->collection}, $_) for (keys %{$self->store});
}
sub start($self) {
$self->get_table()->wait;
my @promises = ();
push(@promises, $self->moref_to_ops) for 1 .. $self->concurrency;
Mojo::Promise->all(@promises)->finally(sub { printf("%s\n", j($self->store)) })->wait;
}
package main;
use Mojo::Base -strict;
Vmware::Moref::Operations->new->start;
@fskale
Copy link
Author

fskale commented Mar 24, 2020

Output (JSON)

{
   "SearchIndex" : {
      "obj_url" : "https://pubs.vmware.com/vi3/sdk/ReferenceGuide/vim.SearchIndex.html",
      "operations" : [
         "FindByDatastorePath",
         "FindByDnsName",
         "FindByInventoryPath",
         "FindByIp",
         "FindByUuid",
         "FindChild"
      ]
   },
   "LicenseManager" : {
      "obj_url" : "https://pubs.vmware.com/vi3/sdk/ReferenceGuide/vim.LicenseManager.html",
      "operations" : [
         "CheckLicenseFeature",
         "ConfigureLicenseSource",
         "DisableFeature",
         "EnableFeature",
         "QueryLicenseSourceAvailability",
         "QueryLicenseUsage",
         "SetLicenseEdition"
      ]
   },
   "PerformanceManager" : {
      "obj_url" : "https://pubs.vmware.com/vi3/sdk/ReferenceGuide/vim.PerformanceManager.html",
      "operations" : [
         "CreatePerfInterval",
         "QueryAvailablePerfMetric",
         "QueryPerf",
         "QueryPerfComposite",
         "QueryPerfCounter",
         "QueryPerfProviderSummary",
         "RemovePerfInterval",
         "UpdatePerfInterval"
      ]
   },
   "PropertyCollector" : {
      "operations" : [
         "CancelWaitForUpdates",
         "CheckForUpdates",
         "CreateFilter",
         "RetrieveProperties",
         "WaitForUpdates"
      ],
      "obj_url" : "https://pubs.vmware.com/vi3/sdk/ReferenceGuide/vmodl.query.PropertyCollector.html"
   },
   "CustomizationSpecManager" : {
      "obj_url" : "https://pubs.vmware.com/vi3/sdk/ReferenceGuide/vim.CustomizationSpecManager.html",
      "operations" : [
         "CheckCustomizationResources",
         "CreateCustomizationSpec",
         "CustomizationSpecItemToXml",
         "DeleteCustomizationSpec",
         "DoesCustomizationSpecExist",
         "DuplicateCustomizationSpec",
         "GetCustomizationSpec",
         "OverwriteCustomizationSpec",
         "RenameCustomizationSpec",
         "XmlToCustomizationSpecItem"
      ]
   },
   "SessionManager" : {
      "operations" : [
         "AcquireLocalTicket",
         "Login",
         "Logout",
         "SetLocale",
         "TerminateSession",
         "UpdateServiceMessage"
      ],
      "obj_url" : "https://pubs.vmware.com/vi3/sdk/ReferenceGuide/vim.SessionManager.html"
   },
   "DiagnosticManager" : {
      "obj_url" : "https://pubs.vmware.com/vi3/sdk/ReferenceGuide/vim.DiagnosticManager.html",
      "operations" : [
         "BrowseDiagnosticLog",
         "GenerateLogBundles_Task",
         "QueryDescriptions"
      ]
   },
   "TaskManager" : {
      "operations" : [
         "CreateCollectorForTasks"
      ],
      "obj_url" : "https://pubs.vmware.com/vi3/sdk/ReferenceGuide/vim.TaskManager.html"
   },
   "Folder" : {
      "operations" : [
         "AddStandaloneHost_Task",
         "CreateCluster",
         "CreateDatacenter",
         "CreateFolder",
         "CreateVM_Task",
         "MoveIntoFolder_Task",
         "RegisterVM_Task",
         "UnregisterAndDestroy_Task"
      ],
      "obj_url" : "https://pubs.vmware.com/vi3/sdk/ReferenceGuide/vim.Folder.html"
   },
   "AlarmManager" : {
      "obj_url" : "https://pubs.vmware.com/vi3/sdk/ReferenceGuide/vim.alarm.AlarmManager.html",
      "operations" : [
         "CreateAlarm",
         "GetAlarm",
         "GetAlarmState"
      ]
   },
   "HostLocalAccountManager" : {
      "operations" : [
         "AssignUserToGroup",
         "CreateGroup",
         "CreateUser",
         "RemoveGroup",
         "RemoveUser",
         "UnassignUserFromGroup",
         "UpdateUser"
      ],
      "obj_url" : "https://pubs.vmware.com/vi3/sdk/ReferenceGuide/vim.host.LocalAccountManager.html"
   },
   "CustomFieldsManager" : {
      "operations" : [
         "AddCustomFieldDef",
         "RemoveCustomFieldDef",
         "RenameCustomFieldDef",
         "SetField"
      ],
      "obj_url" : "https://pubs.vmware.com/vi3/sdk/ReferenceGuide/vim.CustomFieldsManager.html"
   },
   "AuthorizationManager" : {
      "operations" : [
         "AddAuthorizationRole",
         "MergePermissions",
         "RemoveAuthorizationRole",
         "RemoveEntityPermission",
         "ResetEntityPermissions",
         "RetrieveAllPermissions",
         "RetrieveEntityPermissions",
         "RetrieveRolePermissions",
         "SetEntityPermissions",
         "UpdateAuthorizationRole"
      ],
      "obj_url" : "https://pubs.vmware.com/vi3/sdk/ReferenceGuide/vim.AuthorizationManager.html"
   },
   "ScheduledTaskManager" : {
      "operations" : [
         "CreateScheduledTask",
         "RetrieveEntityScheduledTask"
      ],
      "obj_url" : "https://pubs.vmware.com/vi3/sdk/ReferenceGuide/vim.scheduler.ScheduledTaskManager.html"
   },
   "UserDirectory" : {
      "obj_url" : "https://pubs.vmware.com/vi3/sdk/ReferenceGuide/vim.UserDirectory.html",
      "operations" : [
         "RetrieveUserGroups"
      ]
   },
   "OptionManager" : {
      "obj_url" : "https://pubs.vmware.com/vi3/sdk/ReferenceGuide/vim.option.OptionManager.html",
      "operations" : [
         "QueryOptions",
         "UpdateOptions"
      ]
   },
   "EventManager" : {
      "operations" : [
         "CreateCollectorForEvents",
         "LogUserEvent",
         "QueryEvents"
      ],
      "obj_url" : "https://pubs.vmware.com/vi3/sdk/ReferenceGuide/vim.event.EventManager.html"
   }
}

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment