add HV Monitor, a generic means for monitoring hypvervisors (#14218)

* add HV::Monitor support

* document hv monitor

* add HV Monitor tests

* style cleanup

* fix a few missed style items

* test fix

* more test cleanup

* more minor test tweaks

* more test cleanup

* more test tweaking

* test ordering fix all done... hopefully

* more test cleanup

* minor formatting change

LibreNMS/Util/StringHelpers.php

@@ -55,6 +55,7 @@ class StringHelpers
             'fbsd-nfs-server' => 'FreeBSD NFS Server',
             'freeradius' => 'FreeRADIUS',
             'gpsd' => 'GPSD',
+            'hv-monitor' => 'HV Monitor',
             'mailcow-postfix' => 'mailcow-dockerized postfix',
             'mysql' => 'MySQL',
             'nfs-server' => 'NFS Server',

doc/Extensions/Applications.md

@@ -816,6 +816,56 @@ You may need to configure `$server` or `$port`.
 
 Verify it is working by running `/usr/lib/check_mk_agent/local/gpsd`
 
+## HV Monitor
+
+HV Monitor provides a generic way to monitor hypervisors. Currently
+CBSD+bhyve on FreeBSD and Libvirt+QEMU on Linux are support.
+
+For more information see
+HV::Monitor on
+[Github](https://github.com/VVelox/HV-Monitor)
+or [MetaCPAN](https://metacpan.org/dist/HV-Monitor).
+
+### SNMP Extend
+
+1. Install the SNMP Extend.
+
+For Debian based systems this is as below.
+
+```
+apt-get install zlib1g-dev cpanminus libjson-perl
+cpanm HV::Monitor
+```
+
+And on FreeBSD as below.
+
+```
+pkg install p5-App-cpanminus p5-JSON p5-MIME-Base64 p5-Gzip-Faster
+cpanm HV::Monitor
+```
+
+2. Set it up to be be ran by cron by root. Yes, you can directly call
+   this script from SNMPD, but be aware, especially with Libvirt,
+   there is a very real possibility of the snmpget timing out,
+   especially if a VM is spinning up/down as virsh domstats can block
+   for a few seconds or so then.
+
+```
+*/5 * * * * /usr/local/bin/hv_monitor > /var/cache/hv_monitor.json -c 2> /dev/null
+```
+
+3. Setup snmpd.conf as below.
+
+```
+extend hv-monitor /bin/cat
+/var/cache/hv_monitor.json
+
+```
+
+4. Restart SNMPD.
+
+5. Either wait for it to be re-discovered or manually enable it.
+
 ## Icecast
 
 Shell script that reports load average/memory/open-files stats of Icecast

includes/discovery/applications.inc.php

@@ -42,6 +42,7 @@ if ($results) {
     $applications['dhcpstats'] = 'dhcp-stats';
     $applications['fbsdnfsclient'] = 'fbsd-nfs-client';
     $applications['fbsdnfsserver'] = 'fbsd-nfs-server';
+    $applications['hv-monitor'] = 'hv-monitor';
     $applications['mailq'] = 'postfix';
     $applications['osupdate'] = 'os-updates';
     $applications['phpfpmsp'] = 'php-fpm';

includes/html/graphs/application/hv-monitor_cow.inc.php

@@ -0,0 +1,28 @@
+<?php
+
+$name = 'hv-monitor';
+$unit_text = 'COWs / sec';
+$colours = 'psychedelic';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+if (isset($vars['vm'])) {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'vm', $vars['vm']]);
+} else {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
+}
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'COW',
+        'ds'       => 'cow',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/hv-monitor_csw.inc.php

@@ -0,0 +1,33 @@
+<?php
+
+$name = 'hv-monitor';
+$unit_text = 'Context Switches';
+$colours = 'psychedelic';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+if (isset($vars['vm'])) {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'vm', $vars['vm']]);
+} else {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
+}
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Involuntary',
+        'ds'       => 'nivcsw',
+    ];
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Voluntary',
+        'ds'       => 'nvcsw',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/hv-monitor_disk-freqs.inc.php

@@ -0,0 +1,30 @@
+<?php
+
+$name = 'hv-monitor';
+$unit_text = 'Reqs/Sec';
+$colours = 'psychedelic';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+if (isset($vars['vmdisk']) && isset($vars['vm'])) {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'vmdisk', $vars['vm'], '__-__', $vars['vmdisk']]);
+} elseif (isset($vars['vm'])) {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'vm', $vars['vm']]);
+} else {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
+}
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Flushes',
+        'ds'       => 'freqs',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/hv-monitor_disk-ftime.inc.php

@@ -0,0 +1,30 @@
+<?php
+
+$name = 'hv-monitor';
+$unit_text = 'msecs/sec';
+$colours = 'psychedelic';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+if (isset($vars['vmdisk']) && isset($vars['vm'])) {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'vmdisk', $vars['vm'], '__-__', $vars['vmdisk']]);
+} elseif (isset($vars['vm'])) {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'vm', $vars['vm']]);
+} else {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
+}
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Flush Time',
+        'ds'       => 'ftime',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/hv-monitor_disk-rw-blocks.inc.php

@@ -0,0 +1,34 @@
+<?php
+
+$name = 'hv-monitor';
+$unit_text = 'Blocks/Second';
+$colours = 'psychedelic';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+if (isset($vars['vm'])) {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'vm', $vars['vm']]);
+} else {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
+}
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Read',
+        'ds'       => 'inblk',
+    ];
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Write',
+        'ds'       => 'oublk',
+        'invert'   => true,
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/hv-monitor_disk-rw-bytes.inc.php

@@ -0,0 +1,36 @@
+<?php
+
+$name = 'hv-monitor';
+$unit_text = 'Bytes/Second';
+$colours = 'psychedelic';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+if (isset($vars['vmdisk']) && isset($vars['vm'])) {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'vmdisk', $vars['vm'], '__-__', $vars['vmdisk']]);
+} elseif (isset($vars['vm'])) {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'vm', $vars['vm']]);
+} else {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
+}
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Read',
+        'ds'       => 'rbytes',
+    ];
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Write',
+        'ds'       => 'wbytes',
+        'invert'   => true,
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/hv-monitor_disk-rw-reqs.inc.php

@@ -0,0 +1,36 @@
+<?php
+
+$name = 'hv-monitor';
+$unit_text = 'Reqs/Sec';
+$colours = 'psychedelic';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+if (isset($vars['vmdisk']) && isset($vars['vm'])) {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'vmdisk', $vars['vm'], '__-__', $vars['vmdisk']]);
+} elseif (isset($vars['vm'])) {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'vm', $vars['vm']]);
+} else {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
+}
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Read',
+        'ds'       => 'rreqs',
+    ];
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Write',
+        'ds'       => 'wreqs',
+        'invert'   => true,
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/hv-monitor_disk-rw-time.inc.php

@@ -0,0 +1,34 @@
+<?php
+
+$name = 'hv-monitor';
+$unit_text = 'msecs/sec';
+$colours = 'psychedelic';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+if (isset($vars['vm'])) {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'vm', $vars['vm']]);
+} else {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
+}
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Read',
+        'ds'       => 'rtime',
+    ];
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Write',
+        'ds'       => 'wtime',
+        'invert'   => true,
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/hv-monitor_disk-size.inc.php

@@ -0,0 +1,34 @@
+<?php
+
+$name = 'hv-monitor';
+$unit_text = 'Bytes';
+$colours = 'psychedelic';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'vmdisk', $vars['vm'], '__-__', $vars['vmdisk']]);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Allocated',
+        'ds'       => 'alloc',
+    ];
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'In Use',
+        'ds'       => 'in_use',
+    ];
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'On Disk',
+        'ds'       => 'on_disk',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/hv-monitor_disk-size2.inc.php

@@ -0,0 +1,38 @@
+<?php
+
+$name = 'hv-monitor';
+$unit_text = 'Bytes';
+$colours = 'psychedelic';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+if (isset($vars['vm'])) {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'vm', $vars['vm']]);
+} else {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
+}
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Allocated',
+        'ds'       => 'disk_alloc',
+    ];
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'In Use',
+        'ds'       => 'disk_in_use',
+    ];
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'On Disk',
+        'ds'       => 'disk_on_disk',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/hv-monitor_etimes.inc.php

@@ -0,0 +1,28 @@
+<?php
+
+$name = 'hv-monitor';
+$unit_text = 'Seconds';
+$colours = 'psychedelic';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+if (isset($vars['vm'])) {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'vm', $vars['vm']]);
+} else {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
+}
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Etimes',
+        'ds'       => 'etimes',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/hv-monitor_flt.inc.php

@@ -0,0 +1,33 @@
+<?php
+
+$name = 'hv-monitor';
+$unit_text = 'Faults Per Second';
+$colours = 'psychedelic';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+if (isset($vars['vm'])) {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'vm', $vars['vm']]);
+} else {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
+}
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Major',
+        'ds'       => 'majflt',
+    ];
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Minor',
+        'ds'       => 'minflt',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/hv-monitor_memory.inc.php

@@ -0,0 +1,33 @@
+<?php
+
+$name = 'hv-monitor';
+$unit_text = 'Kilobytes';
+$colours = 'psychedelic';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+if (isset($vars['vm'])) {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'vm', $vars['vm']]);
+} else {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
+}
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Allocated',
+        'ds'       => 'mem_alloc',
+    ];
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'RSS',
+        'ds'       => 'rss',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/hv-monitor_net-bytes.inc.php

@@ -0,0 +1,37 @@
+<?php
+
+$name = 'hv-monitor';
+$unit_text = 'Bytes/Second';
+$colours = 'psychedelic';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+if (isset($vars['vmif'])) {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'vmif', $vars['vm'], '__-__', $vars['vmif']]);
+} elseif (isset($vars['vm'])) {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'vm', $vars['vm']]);
+} else {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
+}
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'In',
+        'ds'       => 'ibytes',
+    ];
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Out',
+        'ds'       => 'obytes',
+        'invert'   => true,
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';
+//  require 'includes/html/graphs/generic_multi_line_exact_numbers.inc.php';

includes/html/graphs/application/hv-monitor_net-coll.inc.php

@@ -0,0 +1,30 @@
+<?php
+
+$name = 'hv-monitor';
+$unit_text = 'Per Second';
+$colours = 'psychedelic';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+if (isset($vars['vmif'])) {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'vmif', $vars['vm'], '__-__', $vars['vmif']]);
+} elseif (isset($vars['vm'])) {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'vm', $vars['vm']]);
+} else {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
+}
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Collisions',
+        'ds'       => 'coll',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/hv-monitor_net-drops.inc.php

@@ -0,0 +1,36 @@
+<?php
+
+$name = 'hv-monitor';
+$unit_text = 'Drops/Second';
+$colours = 'psychedelic';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+if (isset($vars['vmif'])) {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'vmif', $vars['vm'], '__-__', $vars['vmif']]);
+} elseif (isset($vars['vm'])) {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'vm', $vars['vm']]);
+} else {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
+}
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'In',
+        'ds'       => 'idrop',
+    ];
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Out',
+        'ds'       => 'odrop',
+        'invert'   => true,
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/hv-monitor_net-errs.inc.php

@@ -0,0 +1,36 @@
+<?php
+
+$name = 'hv-monitor';
+$unit_text = 'Errors/Second';
+$colours = 'psychedelic';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+if (isset($vars['vmif'])) {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'vmif', $vars['vm'], '__-__', $vars['vmif']]);
+} elseif (isset($vars['vm'])) {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'vm', $vars['vm']]);
+} else {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
+}
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'In',
+        'ds'       => 'ierrs',
+    ];
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Out',
+        'ds'       => 'oerrs',
+        'invert'   => true,
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/hv-monitor_net-pkts.inc.php

@@ -0,0 +1,36 @@
+<?php
+
+$name = 'hv-monitor';
+$unit_text = 'Packets/Second';
+$colours = 'psychedelic';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+if (isset($vars['vmif'])) {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'vmif', $vars['vm'], '__-__', $vars['vmif']]);
+} elseif (isset($vars['vm'])) {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'vm', $vars['vm']]);
+} else {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
+}
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'In',
+        'ds'       => 'ipkts',
+    ];
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Out',
+        'ds'       => 'opkts',
+        'invert'   => true,
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/hv-monitor_pcpu.inc.php

@@ -0,0 +1,28 @@
+<?php
+
+$name = 'hv-monitor';
+$unit_text = 'Memory Usage %';
+$colours = 'psychedelic';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+if (isset($vars['vm'])) {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'vm', $vars['vm']]);
+} else {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
+}
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'CPU%',
+        'ds'       => 'pcpu',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/hv-monitor_pmem.inc.php

@@ -0,0 +1,28 @@
+<?php
+
+$name = 'hv-monitor';
+$unit_text = 'Memory Usage %';
+$colours = 'psychedelic';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+if (isset($vars['vm'])) {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'vm', $vars['vm']]);
+} else {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
+}
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Mem%',
+        'ds'       => 'pmem',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/hv-monitor_snaps.inc.php

@@ -0,0 +1,28 @@
+<?php
+
+$name = 'hv-monitor';
+$unit_text = 'Snapshots';
+$colours = 'psychedelic';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+if (isset($vars['vm'])) {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'vm', $vars['vm']]);
+} else {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
+}
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Snaps',
+        'ds'       => 'snaps',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/hv-monitor_snaps_size.inc.php

@@ -0,0 +1,28 @@
+<?php
+
+$name = 'hv-monitor';
+$unit_text = 'Bytes';
+$colours = 'psychedelic';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+if (isset($vars['vm'])) {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'vm', $vars['vm']]);
+} else {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
+}
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Snap Size',
+        'ds'       => 'snaps_size',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/hv-monitor_status-int.inc.php

@@ -0,0 +1,28 @@
+<?php
+
+$name = 'hv-monitor';
+$unit_text = 'VM Statuses';
+$colours = 'psychedelic';
+$dostack = 1;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+if (isset($vars['vm'])) {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'vm', $vars['vm']]);
+} else {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
+}
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Status',
+        'ds'       => 'status_int',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/hv-monitor_status.inc.php

@@ -0,0 +1,73 @@
+<?php
+
+$name = 'hv-monitor';
+$unit_text = 'VM Statuses';
+$colours = 'psychedelic';
+$dostack = 0;
+$printtotal = 0;
+$addarea = 0;
+$transparency = 15;
+
+if (isset($vars['vm'])) {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'vm', $vars['vm']]);
+} else {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
+}
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'On',
+        'ds'       => 'on',
+    ];
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Off',
+        'ds'       => 'off',
+    ];
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Off, Hard',
+        'ds'       => 'off_hard',
+    ];
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Off, Soft',
+        'ds'       => 'off_soft',
+    ];
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Unknown',
+        'ds'       => 'unknown',
+    ];
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Paused',
+        'ds'       => 'paused',
+    ];
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Crashed',
+        'ds'       => 'crashed',
+    ];
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Blocked',
+        'ds'       => 'blocked',
+    ];
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'No State',
+        'ds'       => 'nostate',
+    ];
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'PM Suspended',
+        'ds'       => 'pmsuspended',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/hv-monitor_time.inc.php

@@ -0,0 +1,33 @@
+<?php
+
+$name = 'hv-monitor';
+$unit_text = 'Seconds';
+$colours = 'psychedelic';
+$dostack = 1;
+$printtotal = 1;
+$addarea = 1;
+$transparency = 15;
+
+if (isset($vars['vm'])) {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'vm', $vars['vm']]);
+} else {
+    $rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
+}
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'User',
+        'ds'       => 'usertime',
+    ];
+    $rrd_list[] = [
+        'filename' => $rrd_filename,
+        'descr'    => 'Sys',
+        'ds'       => 'systime',
+    ];
+} else {
+    d_echo('RRD "' . $rrd_filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/pages/apps.inc.php

@@ -417,6 +417,19 @@ $graphs['sagan'] = [
     'uptime',
     'alert',
 ];
+$graphs['hv-monitor'] = [
+    'status',
+    'memory',
+    'pmem',
+    'time',
+    'pcpu',
+    'flt',
+    'csw',
+    'cow',
+    'etimes',
+    'snaps',
+    'snaps_size',
+];
 $graphs['pwrstatd'] = [
     'wattage',
     'voltage',

includes/html/pages/device/apps/hv-monitor.inc.php

@@ -0,0 +1,267 @@
+<?php
+
+use App\Models\Port;
+
+$link_array = [
+    'page'   => 'device',
+    'device' => $device['device_id'],
+    'tab'    => 'apps',
+    'app'    => 'hv-monitor',
+];
+
+print_optionbar_start();
+
+if (! isset($vars['vm'])) {
+    echo generate_link('<span class="pagemenu-selected"><b>Totals</b></span>', $link_array);
+} else {
+    echo generate_link('<b>Totals</b>', $link_array);
+}
+echo '<b> | VMs: </b>';
+$vm_links = [];
+foreach ($app->data['VMs'] as $vm) {
+    $label = $vm;
+
+    if ($vars['vm'] == $vm) {
+        $label = '<span class="pagemenu-selected">' . $vm . '</span>';
+    }
+
+    $vm_links[] = generate_link($label, $link_array, ['vm'=>$vm]);
+}
+echo implode(', ', $vm_links);
+
+if (! isset($vars['vmif']) && ! isset($vars['vmdisk'])) {
+    if (! isset($vars['vmpage'])) {
+        $vars['vmpage'] = 'general';
+    }
+}
+
+echo '<br><b>Pages: </b>';
+if ($vars['vmpage'] == 'general') {
+    $page_links[] = '<span class="pagemenu-selected">General</span>';
+} else {
+    $page_links[] = generate_link('General', $link_array, ['vm' => $vars['vm'], 'vmpage'=>'general']);
+}
+if ($vars['vmpage'] == 'disk') {
+    $page_links[] = '<span class="pagemenu-selected">Disk</span>';
+} else {
+    $page_links[] = generate_link('Disk', $link_array, ['vm' => $vars['vm'], 'vmpage'=>'disk']);
+}
+if ($vars['vmpage'] == 'network') {
+    $page_links[] = '<span class="pagemenu-selected">Network</span>';
+} else {
+    $page_links[] = generate_link('Network', $link_array, ['vm' => $vars['vm'], 'vmpage'=>'network']);
+}
+if ($vars['vmpage'] == 'Snapshots') {
+    $page_links[] = '<span class="pagemenu-selected">Network</span>';
+} else {
+    $page_links[] = generate_link('Snapshots', $link_array, ['vm' => $vars['vm'], 'vmpage'=>'snapshots']);
+}
+echo implode(', ', $page_links);
+
+if (isset($vars['vm'])) {
+    echo '<br><b>Disks:</b> ';
+    $disk_links = [];
+    foreach ($app->data['VMdisks'][$vars['vm']] as $index => $disk) {
+        $label = $disk;
+
+        if ($vars['vmdisk'] == $disk) {
+            $label = '<span class="pagemenu-selected">' . $disk . '</span>';
+        }
+        if ($vars['vmdisk'] == $disk) {
+            $disk_links[] = $label;
+        } else {
+            $disk_links[] = generate_link($label, $link_array, ['vm' => $vars['vm'], 'vmdisk'=>$disk]);
+        }
+    }
+    echo implode(', ', $disk_links);
+
+    echo '<br><b>Interfaces:</b> ';
+    $if_links = [];
+    foreach ($app->data['VMifs'][$vars['vm']] as $vmif => $if_info) {
+        $label = $vmif;
+
+        if ($vars['vmif'] == $vmif) {
+            $if_links[] = '<span class="pagemenu-selected">' . $vmif . '</span>';
+        } else {
+            $if_links[] = generate_link($label, $link_array, ['vm' => $vars['vm'], 'vmif'=>$vmif]);
+        }
+    }
+    echo implode(', ', $if_links);
+}
+
+if (isset($vars['vmif']) and isset($vars['vm'])) {
+    $mac = $app->data['VMifs'][$vars['vm']][$vars['vmif']]['mac'];
+    $port = Port::with('device')->firstWhere(['ifPhysAddress' => str_replace(':', '', $mac)]);
+
+    echo "\n<br>\n" .
+        '<b>MAC:</b> ' . $mac;
+    if (isset($port) && isset($mac) && $mac != '') {
+        echo ' (' .
+               generate_device_link([device_id=>$port->device_id]) .
+               ', ' .
+               generate_port_link([
+                   'label' => $port->label,
+                   'port_id' => $port->port_id,
+                   'ifName' => $port->ifName,
+                   'device_id' => $port->device_id,
+               ]) .
+            ')';
+    }
+    echo "<br>\n";
+
+    // This is likely to be unknown if the device is not up and running
+    //
+    // Likely not relevant for libvirt as it can't do interface re-use. And vnet interfaces are likely set to be ignored as
+    // it bringing them up and down spam the LibreNMS logs. Also massiving spams the RRD dir as well... even more so if
+    // it is a CAPE box.
+    // $config['bad_if'][] = 'vnet';  <--- a must for Libvirt boxes. :(
+    //
+    // Mainly for CBSD
+    $port = Port::with('device')->firstWhere(['device_id' => $app->device_id, 'ifName' => $app->data['VMifs'][$vars['vm']][$vars['vmif']]['if']]);
+    if (! isset($port)) {
+        echo '<b>HV if:</b> ' . $app->data['VMifs'][$vars['vm']][$vars['vmif']]['if'] . "\n";
+    } else {
+        echo '<b>HV if:</b> ' .
+            generate_port_link([
+                'label' => $port->label,
+                'port_id' => $port->port_id,
+                'ifName' => $port->ifName,
+                'device_id' => $port->device_id,
+            ]);
+    }
+
+    // Not likely to be known on Libvirt systems thanks to Libvirt sucking at reporting some info... and IF stuff in general
+    if ($app->data['VMifs'][$vars['vm']][$vars['vmif']]['parent'] != '') {
+        $port = Port::with('device')->firstWhere(['device_id' => $app->device_id, 'ifName' => $app->data['VMifs'][$vars['vm']][$vars['vmif']]['parent']]);
+        if (! isset($port)) {
+            echo '<br><b>HV parent if:</b> ' . $app->data['VMifs'][$vars['vm']][$vars['vmif']]['parent'];
+        } else {
+            echo '<br><b>HV parent if:</b> ' .
+                generate_port_link([
+                    'label' => $port->label,
+                    'port_id' => $port->port_id,
+                    'ifName' => $port->ifName,
+                    'device_id' => $port->device_id,
+                ]);
+        }
+    }
+}
+
+print_optionbar_end();
+
+$link_array = [
+    'page'   => 'device',
+    'device' => $device['device_id'],
+    'tab'    => 'apps',
+    'app'    => 'hv-monitor',
+];
+
+$graphs = [];
+if (! isset($vars['vm']) && isset($vars['vmpage']) && $vars['vmpage'] == 'general') {
+    $graphs['hv-monitor_status'] = 'VM Statuses Count';
+}
+
+if (! isset($vars['vmdisk']) and ! isset($vars['vmif'])) {
+    if (isset($vars['vmpage']) && $vars['vmpage'] == 'general') {
+        if (isset($vars['vm']) && $app->data['hv'] == 'libvirt') {
+            $graphs['hv-monitor_status-int'] = 'VM Status: 0=no state, 1=running, 2=blocked, 3=paused, 4=being shut down, 5=shut off, 6=crashed, 7=PM suspended';
+        }
+        if (isset($vars['vm']) && $app->data['hv'] == 'CBSD') {
+            $graphs['hv-monitor_status-int'] = 'VM Status: 1=Running, 8=Shut Off, 9=Maintenace';
+        }
+        $graphs['hv-monitor_memory'] = 'Memmory Usage';
+        $graphs['hv-monitor_pmem'] = 'Memory Percent';
+        $graphs['hv-monitor_time'] = 'CPU Time';
+        $graphs['hv-monitor_pcpu'] = 'CPU Percent';
+        $graphs['hv-monitor_flt'] = 'Faults';
+        $graphs['hv-monitor_csw'] = 'Context Switches';
+        $graphs['hv-monitor_etimes'] = 'Etimes';
+    }
+
+    if (isset($vars['vmpage']) && $vars['vmpage'] == 'disk') {
+        $graphs['hv-monitor_disk-size2'] = 'Disk Size';
+        // Linux does not support fetching block IO for procs
+        if ($app->data['hv'] != 'libvirt') {
+            $graphs['hv-monitor_disk-rw-blocks'] = 'Disk RW, Blocks';
+        }
+        $graphs['hv-monitor_disk-rw-bytes'] = 'Disk RW, Bytes';
+        $graphs['hv-monitor_disk-rw-reqs'] = 'Disk RW, Requests';
+        $graphs['hv-monitor_cow'] = 'COWs';
+        if ($app->data['hv'] == 'CBSD') {
+            $graphs['hv-monitor_disk-rw-time'] = 'Disk RW, Time';
+        }
+        // does not appear to be a tracked stat on FreeBSD
+        if ($app->data['hv'] == 'libvirt') {
+            $graphs['hv-monitor_disk-ftime'] = 'Disk Flush, Time';
+            $graphs['hv-monitor_disk-freqs'] = 'Disk Flush, Requests';
+        }
+    }
+
+    if (isset($vars['vmpage']) && $vars['vmpage'] == 'snapshots') {
+        $graphs['hv-monitor_snaps'] = 'Snapshots';
+        // curious not supported by libvirt
+        // CBSD and other future bhyve based ones this is easy to get if using ZFS
+        if ($app->data['hv'] == 'CBSD') {
+            $graphs['hv-monitor_snaps_size'] = 'Snapshots Size';
+        }
+    }
+
+    if (isset($vars['vmpage']) && $vars['vmpage'] == 'network') {
+        $graphs['hv-monitor_net-pkts'] = 'Network Packets';
+        $graphs['hv-monitor_net-bytes'] = 'Network Bytes';
+        $graphs['hv-monitor_net-errs'] = 'Network Errors';
+        $graphs['hv-monitor_net-drops'] = 'Network Drops';
+        $graphs['hv-monitor_net-coll'] = 'Network Collisions';
+    }
+} elseif (isset($vars['vmdisk'])) {
+    $graphs['hv-monitor_disk-size'] = 'Size';
+    $graphs['hv-monitor_disk-rw-bytes'] = 'Disk RW, Bytes';
+    $graphs['hv-monitor_disk-rw-reqs'] = 'Disk RW, Requests';
+    if ($app->data['hv'] != 'CBSD') {
+        $graphs['hv-monitor_disk-rw-time'] = 'Disk RW, Time';
+    }
+    if ($app->data['hv'] == 'libvirt') {
+        $graphs['hv-monitor_disk-ftime'] = 'Disk Flush, Time';
+        $graphs['hv-monitor_disk-freqs'] = 'Disk Flush, Requests';
+    }
+} elseif (isset($vars['vmif'])) {
+    $graphs['hv-monitor_net-pkts'] = 'Packets';
+    $graphs['hv-monitor_net-bytes'] = 'Bytes';
+    $graphs['hv-monitor_net-errs'] = 'Errors';
+    $graphs['hv-monitor_net-drops'] = 'Drops';
+    $graphs['hv-monitor_net-coll'] = 'Collisions';
+}
+
+foreach ($graphs as $key => $text) {
+    $graph_type = $key;
+    $graph_array['height'] = '100';
+    $graph_array['width'] = '215';
+    $graph_array['to'] = \LibreNMS\Config::get('time.now');
+    $graph_array['id'] = $app['app_id'];
+    $graph_array['type'] = 'application_' . $key;
+
+    if (isset($vars['vm'])) {
+        $graph_array['vm'] = $vars['vm'];
+    }
+
+    if (isset($vars['vmdisk'])) {
+        $graph_array['vmdisk'] = $vars['vmdisk'];
+    }
+
+    if (isset($vars['vmif'])) {
+        $graph_array['vmif'] = $vars['vmif'];
+    }
+
+    $graph_array['hv'] = $app->data['hv'];
+
+    echo '<div class="panel panel-default">
+    <div class="panel-heading">
+        <h3 class="panel-title">' . $text . '</h3>
+    </div>
+    <div class="panel-body">
+    <div class="row">';
+    include 'includes/html/print-graphrow.inc.php';
+    echo '</div>';
+    echo '</div>';
+    echo '</div>';
+}

includes/polling/applications/hv-monitor.inc.php

@@ -0,0 +1,339 @@
+<?php
+
+use LibreNMS\Exceptions\JsonAppException;
+use LibreNMS\RRD\RrdDefinition;
+
+$name = 'hv-monitor';
+$app_id = $app['app_id'];
+
+try {
+    $return_data = json_app_get($device, 'hv-monitor')['data'];
+} catch (JsonAppException $e) {
+    echo PHP_EOL . $name . ':' . $e->getCode() . ':' . $e->getMessage() . PHP_EOL;
+    update_application($app, $e->getCode() . ':' . $e->getMessage(), []); // Set empty metrics and error message
+
+    return;
+}
+
+$app_data['hv'] = $return_data['hv'];
+
+if (! is_array($app_data['VMs'])) {
+    $app_data['VMs'] = [];
+}
+
+if (! is_array($app_data['VMdisks'])) {
+    $app_data['VMdisks'] = [];
+}
+
+if (! is_array($app_data['VMifs'])) {
+    $app_data['VMdifs'] = [];
+}
+
+if (! is_array($app_data['VMstatus'])) {
+    $app_data['VMdstatus'] = [];
+}
+
+//
+// totals graph stuff
+//
+$rrd_def = RrdDefinition::make()
+    ->addDataset('usertime', 'DDERIVE', 0)
+    ->addDataset('pmem', 'GAUGE', 0)
+    ->addDataset('oublk', 'DERIVE', 0)
+    ->addDataset('minflt', 'DERIVE', 0)
+    ->addDataset('pcpu', 'GAUGE', 0)
+    ->addDataset('mem_alloc', 'GAUGE', 0)
+    ->addDataset('nvcsw', 'DERIVE', 0)
+    ->addDataset('snaps', 'GAUGE', 0)
+    ->addDataset('rss', 'GAUGE', 0)
+    ->addDataset('snaps_size', 'GAUGE', 0)
+    ->addDataset('cpus', 'GAUGE', 0)
+    ->addDataset('cow', 'DERIVE', 0)
+    ->addDataset('nivcsw', 'DERIVE', 0)
+    ->addDataset('systime', 'DDERIVE', 0)
+    ->addDataset('vsz', 'GAUGE', 0)
+    ->addDataset('etimes', 'GAUGE', 0)
+    ->addDataset('majflt', 'GAUGE', 0)
+    ->addDataset('inblk', 'DERIVE', 0)
+    ->addDataset('nswap', 'GAUGE', 0)
+    ->addDataset('on', 'GAUGE', 0)
+    ->addDataset('off', 'GAUGE', 0)
+    ->addDataset('off_hard', 'GAUGE', 0)
+    ->addDataset('off_soft', 'GAUGE', 0)
+    ->addDataset('unknown', 'GAUGE', 0)
+    ->addDataset('paused', 'GAUGE', 0)
+    ->addDataset('crashed', 'GAUGE', 0)
+    ->addDataset('blocked', 'GAUGE', 0)
+    ->addDataset('nostate', 'GAUGE', 0)
+    ->addDataset('pmsuspended', 'GAUGE', 0)
+    ->addDataset('rbytes', 'DERIVE', 0)
+    ->addDataset('rtime', 'DDERIVE', 0)
+    ->addDataset('rreqs', 'DERIVE', 0)
+    ->addDataset('wbytes', 'DERIVE', 0)
+    ->addDataset('wtime', 'DDERIVE', 0)
+    ->addDataset('wreqs', 'DERIVE', 0)
+    ->addDataset('disk_alloc', 'GAUGE', 0)
+    ->addDataset('disk_in_use', 'GAUGE', 0)
+    ->addDataset('disk_on_disk', 'GAUGE', 0)
+    ->addDataset('ftime', 'DDERIVE', 0)
+    ->addDataset('freqs', 'DERIVE', 0)
+    ->addDataset('ipkts', 'DERIVE', 0)
+    ->addDataset('ierrs', 'DERIVE', 0)
+    ->addDataset('ibytes', 'DERIVE', 0)
+    ->addDataset('idrop', 'DERIVE', 0)
+    ->addDataset('opkts', 'DERIVE', 0)
+    ->addDataset('oerrs', 'DERIVE', 0)
+    ->addDataset('obytes', 'DERIVE', 0)
+    ->addDataset('odrop', 'DERIVE', 0)
+    ->addDataset('coll', 'DERIVE', 0);
+
+$totals_fields = [
+    'usertime' => $return_data['totals']['usertime'],
+    'pmem' => $return_data['totals']['pmem'],
+    'oublk' => $return_data['totals']['oublk'],
+    'minflt' => $return_data['totals']['minflt'],
+    'pcpu' => $return_data['totals']['pcpu'],
+    'mem_alloc' => $return_data['totals']['mem_alloc'],
+    'nvcsw' => $return_data['totals']['nvcsw'],
+    'snaps' => $return_data['totals']['snaps'],
+    'rss' => $return_data['totals']['rss'],
+    'snaps_size' => $return_data['totals']['snaps_size'],
+    'cpus' => $return_data['totals']['cpus'],
+    'cow' => $return_data['totals']['cow'],
+    'nivcsw' => $return_data['totals']['nivcsw'],
+    'systime' => $return_data['totals']['systime'],
+    'vsz' => $return_data['totals']['vsz'],
+    'etimes' => $return_data['totals']['etimes'],
+    'majflt' => $return_data['totals']['majflt'],
+    'inblk' => $return_data['totals']['inblk'],
+    'nswap' => $return_data['totals']['nswap'],
+    'on' => $return_data['totals']['on'],
+    'off' => $return_data['totals']['off'],
+    'off_hard' => $return_data['totals']['off_hard'],
+    'off_soft' => $return_data['totals']['off_soft'],
+    'unknown' => $return_data['totals']['unknown'],
+    'paused' => $return_data['totals']['paused'],
+    'crashed' => $return_data['totals']['crashed'],
+    'blocked' => $return_data['totals']['blocked'],
+    'nostate' => $return_data['totals']['nostate'],
+    'pmsuspended' => $return_data['totals']['pmsuspended'],
+    'rbytes' => $return_data['totals']['rbytes'],
+    'rtime' => $return_data['totals']['rtime'],
+    'rreqs' => $return_data['totals']['rreqs'],
+    'wbytes' => $return_data['totals']['wbytes'],
+    'wtime' => $return_data['totals']['wtime'],
+    'wreqs' => $return_data['totals']['wreqs'],
+    'disk_alloc' => $return_data['totals']['disk_alloc'],
+    'disk_in_use' => $return_data['totals']['disk_in_use'],
+    'disk_on_disk' => $return_data['totals']['disk_on_disk'],
+    'ftime' => $return_data['totals']['ftime'],
+    'freqs' => $return_data['totals']['freqs'],
+    'ipkts' => $return_data['totals']['ipkts'],
+    'ierrs' => $return_data['totals']['ierrs'],
+    'ibytes' => $return_data['totals']['ibytes'],
+    'idrop' => $return_data['totals']['idrop'],
+    'opkts' => $return_data['totals']['opkts'],
+    'oerrs' => $return_data['totals']['oerrs'],
+    'obytes' => $return_data['totals']['obytes'],
+    'odrop' => $return_data['totals']['odrop'],
+    'coll' => $return_data['totals']['coll'],
+];
+
+$rrd_name = ['app', $name, $app_id];
+$tags = ['name' => $name, 'app_id' => $app_id, 'rrd_def' => $rrd_def, 'rrd_name' => $rrd_name];
+data_update($device, 'app', $tags, $totals_fields);
+
+//
+// handle each VM
+//
+$vm_rrd_def = RrdDefinition::make()
+    ->addDataset('usertime', 'DDERIVE', 0)
+    ->addDataset('pmem', 'GAUGE', 0)
+    ->addDataset('oublk', 'DERIVE', 0)
+    ->addDataset('minflt', 'DERIVE', 0)
+    ->addDataset('pcpu', 'GAUGE', 0)
+    ->addDataset('mem_alloc', 'GAUGE', 0)
+    ->addDataset('nvcsw', 'DERIVE', 0)
+    ->addDataset('snaps', 'GAUGE', 0)
+    ->addDataset('rss', 'GAUGE', 0)
+    ->addDataset('snaps_size', 'GAUGE', 0)
+    ->addDataset('cpus', 'GAUGE', 0)
+    ->addDataset('cow', 'DERIVE', 0)
+    ->addDataset('nivcsw', 'DERIVE', 0)
+    ->addDataset('systime', 'DDERIVE', 0)
+    ->addDataset('vsz', 'GAUGE', 0)
+    ->addDataset('etimes', 'GAUGE', 0)
+    ->addDataset('majflt', 'GAUGE', 0)
+    ->addDataset('inblk', 'DERIVE', 0)
+    ->addDataset('nswap', 'GAUGE', 0)
+    ->addDataset('status_int', 'GAUGE', 0)
+    ->addDataset('rbytes', 'DERIVE', 0)
+    ->addDataset('rtime', 'DDERIVE', 0)
+    ->addDataset('rreqs', 'DERIVE', 0)
+    ->addDataset('wbytes', 'DERIVE', 0)
+    ->addDataset('wtime', 'DDERIVE', 0)
+    ->addDataset('wreqs', 'DERIVE', 0)
+    ->addDataset('disk_alloc', 'GAUGE', 0)
+    ->addDataset('disk_in_use', 'GAUGE', 0)
+    ->addDataset('disk_on_disk', 'GAUGE', 0)
+    ->addDataset('ftime', 'DDERIVE', 0)
+    ->addDataset('freqs', 'DERIVE', 0)
+    ->addDataset('ipkts', 'DERIVE', 0)
+    ->addDataset('ierrs', 'DERIVE', 0)
+    ->addDataset('ibytes', 'DERIVE', 0)
+    ->addDataset('idrop', 'DERIVE', 0)
+    ->addDataset('opkts', 'DERIVE', 0)
+    ->addDataset('oerrs', 'DERIVE', 0)
+    ->addDataset('obytes', 'DERIVE', 0)
+    ->addDataset('odrop', 'DERIVE', 0)
+    ->addDataset('coll', 'DERIVE', 0);
+
+$VMs = [];
+foreach ($return_data['VMs'] as $vm => $vm_info) {
+    $VMs[] = $vm;
+
+    $vm_fields = [
+        'usertime' => $vm_info['usertime'],
+        'pmem' => $vm_info['pmem'],
+        'oublk' => $vm_info['oublk'],
+        'minflt' => $vm_info['minflt'],
+        'pcpu' => $vm_info['pcpu'],
+        'mem_alloc' => $vm_info['mem_alloc'],
+        'nvcsw' => $vm_info['nvcsw'],
+        'snaps' => $vm_info['snaps'],
+        'rss' => $vm_info['rss'],
+        'snaps_size' => $vm_info['snaps_size'],
+        'cpus' => $vm_info['cpus'],
+        'cow' => $vm_info['cow'],
+        'nivcsw' => $vm_info['nivcsw'],
+        'systime' => $vm_info['systime'],
+        'vsz' => $vm_info['vsz'],
+        'etimes' => $vm_info['etimes'],
+        'majflt' => $vm_info['majflt'],
+        'inblk' => $vm_info['inblk'],
+        'nswap' => $vm_info['nswap'],
+        'status_int' => $vm_info['status_int'],
+        'rbytes' => $vm_info['rbytes'],
+        'rtime' => $vm_info['rtime'],
+        'rreqs' => $vm_info['rreqs'],
+        'wbytes' => $vm_info['wbytes'],
+        'wtime' => $vm_info['wtime'],
+        'wreqs' => $vm_info['wreqs'],
+        'disk_alloc' => $vm_info['disk_alloc'],
+        'disk_in_use' => $vm_info['disk_in_use'],
+        'disk_on_disk' => $vm_info['disk_on_disk'],
+        'ftime' => $vm_info['ftime'],
+        'freqs' => $vm_info['freqs'],
+        'ipkts' => $vm_info['ipkts'],
+        'ierrs' => $vm_info['ierrs'],
+        'ibytes' => $vm_info['ibytes'],
+        'idrop' => $vm_info['idrop'],
+        'opkts' => $vm_info['opkts'],
+        'oerrs' => $vm_info['oerrs'],
+        'obytes' => $vm_info['obytes'],
+        'odrop' => $vm_info['odrop'],
+        'coll' => $vm_info['coll'],
+    ];
+
+    $rrd_name = ['app', $name, $app_id, 'vm', $vm];
+    $tags = ['name' => $name, 'app_id' => $app_id, 'rrd_def' => $vm_rrd_def, 'rrd_name' => $rrd_name];
+    data_update($device, 'app', $tags, $vm_fields);
+}
+sort($VMs);
+$app_data['VMs'] = $VMs;
+
+//
+// process each disk
+//
+$disk_rrd_def = RrdDefinition::make()
+    ->addDataset('in_use', 'GAUGE', 0)
+    ->addDataset('on_disk', 'GAUGE', 0)
+    ->addDataset('alloc', 'GAUGE', 0)
+    ->addDataset('rbytes', 'DERIVE', 0)
+    ->addDataset('rtime', 'DDERIVE', 0)
+    ->addDataset('rreqs', 'DERIVE', 0)
+    ->addDataset('wbytes', 'DERIVE', 0)
+    ->addDataset('wtime', 'DDERIVE', 0)
+    ->addDataset('wreqs', 'DERIVE', 0)
+    ->addDataset('ftime', 'DDERIVE', 0)
+    ->addDataset('freqs', 'DERIVE', 0);
+
+foreach ($VMs as $vm) {
+    $vm_disks = [];
+
+    foreach ($return_data['VMs'][$vm]['disks'] as $disk => $disk_info) {
+        $vm_disks[] = $disk;
+
+        $disk_fields = [
+            'in_use' => $disk_info['in_use'],
+            'on_disk' => $disk_info['on_disk'],
+            'alloc' => $disk_info['alloc'],
+            'rbytes' => $disk_info['rbytes'],
+            'rtime' => $disk_info['rtime'],
+            'rreqs' => $disk_info['rreqs'],
+            'wbytes' => $disk_info['wbytes'],
+            'wtime' => $disk_info['wtime'],
+            'wreqs' => $disk_info['wreqs'],
+            'ftime' => $disk_info['ftime'],
+            'freqs' => $disk_info['freqs'],
+        ];
+
+        $rrd_name = ['app', $name, $app_id, 'vmdisk', $vm, '__-__', $disk];
+        $tags = ['name' => $name, 'app_id' => $app_id, 'rrd_def' => $disk_rrd_def, 'rrd_name' => $rrd_name];
+        data_update($device, 'app', $tags, $disk_fields);
+    }
+    sort($vm_disks);
+
+    $app_data['VMdisks'][$vm] = $vm_disks;
+}
+
+//
+// process each if
+//
+$if_rrd_def = RrdDefinition::make()
+    ->addDataset('ipkts', 'DERIVE', 0)
+    ->addDataset('ierrs', 'DERIVE', 0)
+    ->addDataset('ibytes', 'DERIVE', 0)
+    ->addDataset('idrop', 'DERIVE', 0)
+    ->addDataset('opkts', 'DERIVE', 0)
+    ->addDataset('oerrs', 'DERIVE', 0)
+    ->addDataset('obytes', 'DERIVE', 0)
+    ->addDataset('odrop', 'DERIVE', 0)
+    ->addDataset('coll', 'DERIVE', 0);
+
+foreach ($VMs as $vm) {
+    $vm_ifs = [];
+
+    foreach ($return_data['VMs'][$vm]['ifs'] as $vm_if => $if_info) {
+        $vm_ifs[$vm_if] = [
+            'mac' => $if_info['mac'],
+            'parent' => $if_info['parent'],
+            'if' => $if_info['if'],
+        ];
+
+        $if_fields = [
+            'ipkts' => $if_info['ipkts'],
+            'ierrs' => $if_info['ierrs'],
+            'ibytes' => $if_info['ibytes'],
+            'idrop' => $if_info['idrop'],
+            'opkts' => $if_info['opkts'],
+            'oerrs' => $if_info['oerrs'],
+            'obytes' => $if_info['obytes'],
+            'odrop' => $if_info['odrop'],
+            'coll' => $if_info['coll'],
+        ];
+
+        $rrd_name = ['app', $name, $app_id, 'vmif', $vm, '__-__', $vm_if];
+        $tags = ['name' => $name, 'app_id' => $app_id, 'rrd_def' => $if_rrd_def, 'rrd_name' => $rrd_name];
+        data_update($device, 'app', $tags, $if_fields);
+    }
+
+    $app_data['VMifs'][$vm] = $vm_ifs;
+}
+
+//
+// all done so update the app metrics and app_data
+//
+$app->data = $app_data;
+unset($return_data['hv']);
+update_application($app, 'OK', data_flatten($return_data));