add the Mojo CAPE Submit app (#15140)

* add poller for mojo cape submit

* mojo_cape_submit poller now works

* add app page for mojo_cape_submit

* add basic generic stat graphs for mojo cape submit

* general size stats graph for mojo_cape_submit

* make sure we update slugs that did not see anything submitted for them

* sort the slugs by sub count

* add app protos graph and sub top 12 graphs

* add mojo cape submit docs

* add tests for mojo_cape_submit

* add mojo_cape_submit to the apps page

* add string helper for mojo_cape_submit

* poke lots of stuff with php-cs-fixer

* remove a unneeded line

* add a missing metric and app data to test
This commit is contained in:
Zane C. Bowers-Hadley 2023-07-19 22:41:51 -05:00 committed by GitHub
parent 510d5dc94b
commit 54a38dd4cd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 629 additions and 0 deletions

View File

@ -57,6 +57,7 @@ class StringHelpers
'freeradius' => 'FreeRADIUS',
'gpsd' => 'GPSD',
'hv-monitor' => 'HV Monitor',
'mojo_cape_submit' => 'Mojo CAPE Submit',
'mailcow-postfix' => 'mailcow-dockerized postfix',
'mysql' => 'MySQL',
'nfs-server' => 'NFS Server',

View File

@ -1226,6 +1226,20 @@ The application should be auto-discovered as described at the top of
the page. If it is not, please follow the steps set out under `SNMP
Extend` heading top of page.
## Mojo CAPE Submit
### SNMP
This assumes you've already configured mojo_cape_submit from CAPE::Utils.
1. Add the following to `snmpd.conf` and restarted SNMPD
```
extend mojo_cape_submit /usr/local/bin/mojo_cape_submit_extend
```
Then just wait for the machine in question to be rediscovered or
enabled it in the device settings app page.
## Munin
### Agent

View File

@ -0,0 +1,19 @@
<?php
$name = 'mojo_cape_submit';
$unit_text = 'App Protos';
$colours = 'psychedelic';
$descr = 'App Protos';
$ds = 'app_protos';
if (isset($vars['slug'])) {
$filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'slugs___-___' . $vars['slug']]);
} else {
$filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
}
if (! Rrd::checkRrdExists($filename)) {
d_echo('RRD "' . $filename . '" not found');
}
require 'includes/html/graphs/generic_stats.inc.php';

View File

@ -0,0 +1,19 @@
<?php
$name = 'mojo_cape_submit';
$unit_text = 'Changed Hashes';
$colours = 'psychedelic';
$descr = 'Change Count';
$ds = 'hash_changed';
if (isset($vars['slug'])) {
$filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'slugs___-___' . $vars['slug']]);
} else {
$filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
}
if (! Rrd::checkRrdExists($filename)) {
d_echo('RRD "' . $filename . '" not found');
}
require 'includes/html/graphs/generic_stats.inc.php';

View File

@ -0,0 +1,19 @@
<?php
$name = 'mojo_cape_submit';
$unit_text = 'Bytes';
$colours = 'psychedelic';
$descr = 'Size Sum';
$ds = 'Size Sum';
if (isset($vars['slug'])) {
$filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'slugs___-___' . $vars['slug']]);
} else {
$filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
}
if (! Rrd::checkRrdExists($filename)) {
d_echo('RRD "' . $filename . '" not found');
}
require 'includes/html/graphs/generic_stats.inc.php';

View File

@ -0,0 +1,19 @@
<?php
$name = 'mojo_cape_submit';
$unit_text = 'Bytes';
$colours = 'psychedelic';
$descr = 'Size max';
$ds = 'size_max';
if (isset($vars['slug'])) {
$filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'slugs___-___' . $vars['slug']]);
} else {
$filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
}
if (! Rrd::checkRrdExists($filename)) {
d_echo('RRD "' . $filename . '" not found');
}
require 'includes/html/graphs/generic_stats.inc.php';

View File

@ -0,0 +1,19 @@
<?php
$name = 'mojo_cape_submit';
$unit_text = 'Bytes';
$colours = 'psychedelic';
$descr = 'Size Mean';
$ds = 'size_mean';
if (isset($vars['slug'])) {
$filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'slugs___-___' . $vars['slug']]);
} else {
$filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
}
if (! Rrd::checkRrdExists($filename)) {
d_echo('RRD "' . $filename . '" not found');
}
require 'includes/html/graphs/generic_stats.inc.php';

View File

@ -0,0 +1,19 @@
<?php
$name = 'mojo_cape_submit';
$unit_text = 'Bytes';
$colours = 'psychedelic';
$descr = 'Size Median';
$ds = 'size_median';
if (isset($vars['slug'])) {
$filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'slugs___-___' . $vars['slug']]);
} else {
$filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
}
if (! Rrd::checkRrdExists($filename)) {
d_echo('RRD "' . $filename . '" not found');
}
require 'includes/html/graphs/generic_stats.inc.php';

View File

@ -0,0 +1,19 @@
<?php
$name = 'mojo_cape_submit';
$unit_text = 'Bytes';
$colours = 'psychedelic';
$descr = 'Size Min';
$ds = 'size_min';
if (isset($vars['slug'])) {
$filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'slugs___-___' . $vars['slug']]);
} else {
$filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
}
if (! Rrd::checkRrdExists($filename)) {
d_echo('RRD "' . $filename . '" not found');
}
require 'includes/html/graphs/generic_stats.inc.php';

View File

@ -0,0 +1,19 @@
<?php
$name = 'mojo_cape_submit';
$unit_text = 'Bytes';
$colours = 'psychedelic';
$descr = 'Size Mode';
$ds = 'size_mode';
if (isset($vars['slug'])) {
$filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'slugs___-___' . $vars['slug']]);
} else {
$filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
}
if (! Rrd::checkRrdExists($filename)) {
d_echo('RRD "' . $filename . '" not found');
}
require 'includes/html/graphs/generic_stats.inc.php';

View File

@ -0,0 +1,48 @@
<?php
$name = 'mojo_cape_submit';
$unit_text = 'Bytes';
$colours = 'psychedelic';
$dostack = 0;
$printtotal = 0;
$addarea = 1;
$transparency = 15;
if (isset($vars['slug'])) {
$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'slugs___-___' . $vars['slug']]);
} else {
$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
}
$rrd_list = [];
if (Rrd::checkRrdExists($rrd_filename)) {
$rrd_list[] = [
'filename' => $rrd_filename,
'descr' => 'Max',
'ds' => 'size_max',
];
$rrd_list[] = [
'filename' => $rrd_filename,
'descr' => 'Mean',
'ds' => 'size_mean',
];
$rrd_list[] = [
'filename' => $rrd_filename,
'descr' => 'Median',
'ds' => 'size_median',
];
$rrd_list[] = [
'filename' => $rrd_filename,
'descr' => 'Mode',
'ds' => 'size_mode',
];
$rrd_list[] = [
'filename' => $rrd_filename,
'descr' => 'Min',
'ds' => 'size_min',
];
} else {
d_echo('RRD "' . $rrd_filename . '" not found');
}
require 'includes/html/graphs/generic_multi_line.inc.php';

View File

@ -0,0 +1,19 @@
<?php
$name = 'mojo_cape_submit';
$unit_text = 'Bytes';
$colours = 'psychedelic';
$descr = 'Size StdDev';
$ds = 'size_stddev';
if (isset($vars['slug'])) {
$filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'slugs___-___' . $vars['slug']]);
} else {
$filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
}
if (! Rrd::checkRrdExists($filename)) {
d_echo('RRD "' . $filename . '" not found');
}
require 'includes/html/graphs/generic_stats.inc.php';

View File

@ -0,0 +1,19 @@
<?php
$name = 'mojo_cape_submit';
$unit_text = 'Bytes';
$colours = 'psychedelic';
$descr = 'Size Sum';
$ds = 'size_sum';
if (isset($vars['slug'])) {
$filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'slugs___-___' . $vars['slug']]);
} else {
$filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
}
if (! Rrd::checkRrdExists($filename)) {
d_echo('RRD "' . $filename . '" not found');
}
require 'includes/html/graphs/generic_stats.inc.php';

View File

@ -0,0 +1,19 @@
<?php
$name = 'mojo_cape_submit';
$unit_text = 'Submissions';
$colours = 'psychedelic';
$descr = 'Sub Count';
$ds = 'sub_count';
if (isset($vars['slug'])) {
$filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'slugs___-___' . $vars['slug']]);
} else {
$filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
}
if (! Rrd::checkRrdExists($filename)) {
d_echo('RRD "' . $filename . '" not found');
}
require 'includes/html/graphs/generic_stats.inc.php';

View File

@ -0,0 +1,31 @@
<?php
$name = 'mojo_cape_submit';
$app_id = $app['app_id'];
$unit_text = 'Bytes';
$colours = 'rainbow';
$dostack = 0;
$printtotal = 1;
$addarea = 0;
$transparency = 0;
$float_precision = 3;
$slugs_all = $app->data['slugs'] ?? [];
$slugs = array_slice(array_keys($slugs_all), 0, 12);
$rrd_list = [];
foreach ($slugs as $index => $slug) {
$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'slugs___-___' . $slug]);
$rrd_list[] = [
'filename' => $rrd_filename,
'descr' => $slug,
'ds' => 'sub_count',
];
}
if (count($rrd_list)) {
d_echo('No relevant log file RRDs found');
}
require 'includes/html/graphs/generic_multi_line.inc.php';

View File

@ -486,6 +486,20 @@ $graphs['suricata_extract'] = [
'zero_sized',
'sub_size',
];
$graphs['mojo_cape_submit'] = [
'subs',
'subs_top12',
'hash_changed',
'app_protos',
'size_sum',
'size_stats',
'size_max',
'size_mean',
'size_median',
'size_mode',
'size_min',
'size_stddev',
];
$graphs['linux_softnet_stat'] = [
'packets',
'time_squeeze',

View File

@ -0,0 +1,84 @@
<?php
$link_array = [
'page' => 'device',
'device' => $device['device_id'],
'tab' => 'apps',
'app' => 'mojo_cape_submit',
];
print_optionbar_start();
echo generate_link('General', $link_array);
echo ' | Slugs: ';
$slugs = $app->data['slugs'];
foreach (array_keys($slugs) as $index => $slug) {
$label = $vars['slug'] == $slug
? '<span class="pagemenu-selected">' . $slug . '</span>'
: $slug;
echo generate_link($label, $link_array, ['slug' => $slug]);
if ($index < (count($slugs) - 1)) {
echo ', ';
}
}
print_optionbar_end();
if (isset($vars['slug'])) {
$graphs = [
'mojo_cape_submit-subs' => 'Submissions',
'mojo_cape_submit-hash_changed' => 'Submissions where the hashes changed',
'mojo_cape_submit-app_protos' => 'App protocols seen',
'mojo_cape_submit-size_sum' => 'Size in bytes of submissions',
'mojo_cape_submit-size_stats' => 'Size stats of submissions',
'mojo_cape_submit-size_max' => 'Size max of any submissions',
'mojo_cape_submit-size_mean' => 'Size mean of submissions',
'mojo_cape_submit-size_mode' => 'Size mode of submissions',
'mojo_cape_submit-size_median' => 'Size median of submissions',
'mojo_cape_submit-size_min' => 'Size median of submissions',
'mojo_cape_submit-size_stddev' => 'Size standard deviation of submissions',
];
} else {
$graphs = [
'mojo_cape_submit-subs' => 'Submissions',
'mojo_cape_submit-subs_top12' => 'Submissions, top 12 slugs',
'mojo_cape_submit-hash_changed' => 'Submissions where the hashes changed',
'mojo_cape_submit-app_protos' => 'App protocols seen',
'mojo_cape_submit-size_sum' => 'Size in bytes of submissions',
'mojo_cape_submit-size_stats' => 'Size stats of submissions',
'mojo_cape_submit-size_max' => 'Size max of any submissions',
'mojo_cape_submit-size_mean' => 'Size mean of submissions',
'mojo_cape_submit-size_mode' => 'Size mode of submissions',
'mojo_cape_submit-size_median' => 'Size median of submissions',
'mojo_cape_submit-size_min' => 'Size median of submissions',
'mojo_cape_submit-size_stddev' => 'Size standard deviation of submissions',
];
}
foreach ($graphs as $key => $text) {
$graph_type = $key;
$graph_array['height'] = '100';
$graph_array['width'] = '215';
$graph_array['to'] = \LibreNMS\Config::get('time.now');
$graph_array['id'] = $app['app_id'];
$graph_array['type'] = 'application_' . $key;
if (isset($vars['slug'])) {
$graph_array['slug'] = $vars['slug'];
}
echo '<div class="panel panel-default">
<div class="panel-heading">
<h3 class="panel-title">' . $text . '</h3>
</div>
<div class="panel-body">
<div class="row">';
include 'includes/html/print-graphrow.inc.php';
echo '</div>';
echo '</div>';
echo '</div>';
}

View File

@ -0,0 +1,120 @@
<?php
use LibreNMS\Exceptions\JsonAppException;
use LibreNMS\RRD\RrdDefinition;
$name = 'mojo_cape_submit';
try {
$data = json_app_get($device, $name)['data'];
} catch (JsonAppException $e) {
echo PHP_EOL . $name . ':' . $e->getCode() . ':' . $e->getMessage() . PHP_EOL;
update_application($app, $e->getCode() . ':' . $e->getMessage(), []); // Set empty metrics and error message
return;
}
$app_data = $app->data;
if (! is_array($app_data['slugs'])) {
$app_data['slugs'] = [];
}
$rrd_name = ['app', $name, $app->app_id];
$rrd_def = RrdDefinition::make()
->addDataset('app_protos', 'GAUGE', 0)
->addDataset('hash_changed', 'GAUGE', 0)
->addDataset('size_max', 'GAUGE', 0)
->addDataset('size_mean', 'GAUGE', 0)
->addDataset('size_median', 'GAUGE', 0)
->addDataset('size_min', 'GAUGE', 0)
->addDataset('size_mode', 'GAUGE', 0)
->addDataset('size_stddev', 'GAUGE', 0)
->addDataset('size_sum', 'GAUGE', 0)
->addDataset('sub_count', 'GAUGE', 0);
$fields = [
'app_protos' => $data['totals']['app_protos'],
'hash_changed' => $data['totals']['hash_changed'],
'size_max' => $data['totals']['size_max'],
'size_mean' => $data['totals']['size_mean'],
'size_median' => $data['totals']['size_median'],
'size_min' => $data['totals']['size_min'],
'size_mode' => $data['totals']['size_mode'],
'size_stddev' => $data['totals']['size_stddev'],
'size_sum' => $data['totals']['size_sum'],
'sub_count' => $data['totals']['sub_count'],
];
$tags = ['name' => $name, 'app_id' => $app->app_id, 'rrd_def' => $rrd_def, 'rrd_name' => $rrd_name];
data_update($device, 'app', $tags, $fields);
$new_slugs = [];
$seen_slugs = [];
foreach ($data['slugs'] as $slug => $slug_data) {
$fields = [
'app_protos' => $slug_data['app_protos'],
'hash_changed' => $slug_data['hash_changed'],
'size_max' => $slug_data['size_max'],
'size_mean' => $slug_data['size_mean'],
'size_median' => $slug_data['size_median'],
'size_min' => $slug_data['size_min'],
'size_mode' => $slug_data['size_mode'],
'size_stddev' => $slug_data['size_stddev'],
'size_sum' => $slug_data['size_sum'],
'sub_count' => $slug_data['sub_count'],
];
$rrd_name = ['app', $name, $app->app_id, 'slugs___-___' . $slug];
$tags = ['name' => $name, 'app_id' => $app->app_id, 'rrd_def' => $rrd_def, 'rrd_name' => $rrd_name];
data_update($device, 'app', $tags, $fields);
if (! isset($app_data['slugs'][$slug])) {
array_push($new_slugs, $slug);
}
$app_data['slugs'][$slug] = $slug_data['sub_count'];
$seen_slugs[$slug] = 1;
}
// make sure we update the RRDs for slugs that have not been seen
// if this is not done slugs that do not generate data regularly
// will only display nan
foreach ($app_data['slugs'] as $slug => $slug_data) {
if (! isset($seen_slugs[$slug])) {
$fields = [
'app_protos' => 0,
'hash_changed' => 0,
'size_max' => 0,
'size_mean' => 0,
'size_median' => 0,
'size_min' => 0,
'size_mode' => 0,
'size_stddev' => 0,
'size_sum' => 0,
'sub_count' => 0,
];
$rrd_name = ['app', $name, $app->app_id, 'slugs___-___' . $slug];
$tags = ['name' => $name, 'app_id' => $app->app_id, 'rrd_def' => $rrd_def, 'rrd_name' => $rrd_name];
data_update($device, 'app', $tags, $fields);
if (! isset($app_data['slugs'][$slug])) {
array_push($new_slugs, $slug);
}
$app_data['slugs'][$slug] = 0;
}
}
if ($data['totals']['hash_changed'] >= 1) {
log_event('Mojo Cape Submit has recieved submissions with changed hashes: ' . json_encode($data['changed_hashes']), $device, 'application', 5);
}
if (isset($new_slugs[0])) {
log_event('Mojo Cape Submit has seen one or more new slugs: ' . json_encode($new_slugs), $device, 'application', 1);
}
uasort($app_data['slugs'], function ($a, $b) {
if ($a == $b) {
return 0;
}
return ($a > $b) ? -1 : 1;
});
$app->data = $app_data;
update_application($app, 'OK', $data['totals']);

View File

@ -0,0 +1,98 @@
{
"applications": {
"discovery": {
"applications": [
{
"app_type": "mojo_cape_submit",
"app_state": "UNKNOWN",
"discovered": 1,
"app_state_prev": null,
"app_status": "",
"app_instance": "",
"data": null
}
]
},
"poller": {
"applications": [
{
"app_type": "mojo_cape_submit",
"app_state": "OK",
"discovered": 1,
"app_state_prev": "UNKNOWN",
"app_status": "",
"app_instance": "",
"data": "{\"slugs\":[]}"
}
],
"application_metrics": [
{
"metric": "app_proto",
"value": 0,
"value_prev": null,
"app_type": "mojo_cape_submit"
},
{
"metric": "app_protos",
"value": 0,
"value_prev": null,
"app_type": "mojo_cape_submit"
},
{
"metric": "hash_changed",
"value": 0,
"value_prev": null,
"app_type": "mojo_cape_submit"
},
{
"metric": "size_max",
"value": 0,
"value_prev": null,
"app_type": "mojo_cape_submit"
},
{
"metric": "size_mean",
"value": 0,
"value_prev": null,
"app_type": "mojo_cape_submit"
},
{
"metric": "size_median",
"value": 0,
"value_prev": null,
"app_type": "mojo_cape_submit"
},
{
"metric": "size_min",
"value": 0,
"value_prev": null,
"app_type": "mojo_cape_submit"
},
{
"metric": "size_mode",
"value": 0,
"value_prev": null,
"app_type": "mojo_cape_submit"
},
{
"metric": "size_stddev",
"value": 0,
"value_prev": null,
"app_type": "mojo_cape_submit"
},
{
"metric": "size_sum",
"value": 0,
"value_prev": null,
"app_type": "mojo_cape_submit"
},
{
"metric": "sub_count",
"value": 0,
"value_prev": null,
"app_type": "mojo_cape_submit"
}
]
}
}
}

View File

@ -0,0 +1,10 @@
1.3.6.1.2.1.1.1.0|4|Linux server 3.10.0-693.5.2.el7.x86_64 #1 SMP Fri Oct 20 20:32:50 UTC 2017 x86_64
1.3.6.1.2.1.1.2.0|6|1.3.6.1.4.1.8072.3.2.10
1.3.6.1.2.1.1.3.0|67|77550514
1.3.6.1.2.1.1.4.0|4|<private>
1.3.6.1.2.1.1.5.0|4|<private>
1.3.6.1.2.1.1.6.0|4|<private>
1.3.6.1.2.1.25.1.1.0|67|77552962
1.3.6.1.4.1.8072.1.3.2.2.1.21.6.100.105.115.116.114.111|2|1
1.3.6.1.4.1.8072.1.3.2.2.1.21.16.109.111.106.111.95.99.97.112.101.95.115.117.98.109.105.116|2|1
1.3.6.1.4.1.8072.1.3.2.3.1.2.16.109.111.106.111.95.99.97.112.101.95.115.117.98.109.105.116|4x|7b2264617461223a7b226368616e6765645f686173686573223a5b5d2c22746f74616c73223a7b2273697a655f6d696e223a302c226170705f70726f746f73223a302c2273697a655f6d6f6465223a302c226170705f70726f746f223a7b7d2c2273697a655f6d656469616e223a302c22686173685f6368616e676564223a302c2273697a655f6d6178223a302c2273697a655f73756d223a302c2273697a655f6d65616e223a302c2273697a655f737464646576223a302c227375625f636f756e74223a307d2c22736c756773223a7b7d7d2c2276657273696f6e223a312c226572726f72537472696e67223a22222c226572726f72223a307d0a