From 8d5bc3fc4151551eedbfb1a7299e0978d7308976 Mon Sep 17 00:00:00 2001 From: Tony Murray Date: Wed, 6 Oct 2021 17:09:54 -0500 Subject: [PATCH] Use Measurements for all statistic collection (#13333) * Use Measurements for all statistic collection * fix space * Use colors cleanup * fix style * manually fix license notice * add return times and new line * add return times and new line * fix mistake in copyright template * fix style --- LibreNMS/Data/Source/SnmpQuery.php | 14 +- LibreNMS/Data/Store/BaseDatastore.php | 4 +- LibreNMS/Data/Store/Datastore.php | 16 +- LibreNMS/Data/Store/Graphite.php | 2 +- LibreNMS/Data/Store/InfluxDB.php | 2 +- LibreNMS/Data/Store/OpenTSDB.php | 2 +- LibreNMS/Data/Store/Prometheus.php | 2 +- LibreNMS/Data/Store/Rrd.php | 2 +- .../Polling}/Measure/Measurement.php | 36 +++-- .../Measure/MeasurementCollection.php | 31 ++-- app/Polling/Measure/MeasurementManager.php | 142 ++++++++++++++++++ .../Polling}/Measure/MeasurementSummary.php | 25 +-- app/Providers/AppServiceProvider.php | 4 +- discovery.php | 3 +- html/graph.php | 2 +- includes/dbFacile.php | 83 ---------- includes/discovery/functions.inc.php | 6 +- includes/functions.php | 128 ---------------- includes/polling/functions.inc.php | 7 +- includes/snmp.inc.php | 29 ++-- poller.php | 3 +- 21 files changed, 251 insertions(+), 292 deletions(-) rename {LibreNMS/Data => app/Polling}/Measure/Measurement.php (67%) rename {LibreNMS/Data => app/Polling}/Measure/MeasurementCollection.php (68%) create mode 100644 app/Polling/Measure/MeasurementManager.php rename {LibreNMS/Data => app/Polling}/Measure/MeasurementSummary.php (78%) diff --git a/LibreNMS/Data/Source/SnmpQuery.php b/LibreNMS/Data/Source/SnmpQuery.php index 17a3e0fb23..ef251ea5d3 100644 --- a/LibreNMS/Data/Source/SnmpQuery.php +++ b/LibreNMS/Data/Source/SnmpQuery.php @@ -26,6 +26,7 @@ namespace LibreNMS\Data\Source; use App\Models\Device; +use App\Polling\Measure\Measurement; use DeviceCache; use Illuminate\Support\Arr; use Illuminate\Support\Str; @@ -215,15 +216,6 @@ class SnmpQuery return $this->exec('snmptranslate', $this->parseOid($oid)); } - private function recordStatistic(string $type, float $start_time): void - { - global $snmp_stats; - - $runtime = microtime(true) - $start_time; - $snmp_stats['ops'][$type] = isset($snmp_stats['ops'][$type]) ? $snmp_stats['ops'][$type] + 1 : 0; - $snmp_stats['time'][$type] = isset($snmp_stats['time'][$type]) ? $snmp_stats['time'][$type] + $runtime : $runtime; - } - private function buildCli(string $command, array $oids): array { $cmd = $this->initCommand($command); @@ -285,7 +277,7 @@ class SnmpQuery private function exec(string $command, array $oids): SnmpResponse { - $time_start = microtime(true); + $measure = Measurement::start($command); $proc = new Process($this->buildCli($command, $oids)); $proc->setTimeout(Config::get('snmp.exec_timeout', 1200)); @@ -301,7 +293,7 @@ class SnmpQuery $this->checkExitCode($exitCode, $stderr); $this->logOutput($output, $stderr); - $this->recordStatistic($command, $time_start); + $measure->manager()->recordSnmp($measure->end()); return new SnmpResponse($output, $stderr, $exitCode); } diff --git a/LibreNMS/Data/Store/BaseDatastore.php b/LibreNMS/Data/Store/BaseDatastore.php index 30bece8475..3441f33bb0 100644 --- a/LibreNMS/Data/Store/BaseDatastore.php +++ b/LibreNMS/Data/Store/BaseDatastore.php @@ -25,8 +25,8 @@ namespace LibreNMS\Data\Store; -use LibreNMS\Data\Measure\Measurement; -use LibreNMS\Data\Measure\MeasurementCollection; +use App\Polling\Measure\Measurement; +use App\Polling\Measure\MeasurementCollection; use LibreNMS\Interfaces\Data\Datastore as DatastoreContract; abstract class BaseDatastore implements DatastoreContract diff --git a/LibreNMS/Data/Store/Datastore.php b/LibreNMS/Data/Store/Datastore.php index 6faac2f3d0..87fba8c058 100644 --- a/LibreNMS/Data/Store/Datastore.php +++ b/LibreNMS/Data/Store/Datastore.php @@ -25,6 +25,7 @@ namespace LibreNMS\Data\Store; +use Illuminate\Support\Collection; use LibreNMS\Config; use LibreNMS\Interfaces\Data\Datastore as DatastoreContract; @@ -151,12 +152,15 @@ class Datastore return $this->stores; } - public function getStats() + /** + * Get the measurements for all datastores, keyed by datastore name + * + * @return \Illuminate\Support\Collection<\App\Polling\Measure\MeasurementCollection> + */ + public function getStats(): Collection { - return array_reduce($this->stores, function ($result, DatastoreContract $store) { - $result[$store->getName()] = $store->getStats(); - - return $result; - }, []); + return collect($this->stores)->mapWithKeys(function (DatastoreContract $store) { + return [$store->getName() => $store->getStats()]; + }); } } diff --git a/LibreNMS/Data/Store/Graphite.php b/LibreNMS/Data/Store/Graphite.php index baa4544155..7630a74b5b 100644 --- a/LibreNMS/Data/Store/Graphite.php +++ b/LibreNMS/Data/Store/Graphite.php @@ -26,9 +26,9 @@ namespace LibreNMS\Data\Store; +use App\Polling\Measure\Measurement; use Carbon\Carbon; use LibreNMS\Config; -use LibreNMS\Data\Measure\Measurement; use Log; class Graphite extends BaseDatastore diff --git a/LibreNMS/Data/Store/InfluxDB.php b/LibreNMS/Data/Store/InfluxDB.php index b1f697c860..22ee76e9b6 100644 --- a/LibreNMS/Data/Store/InfluxDB.php +++ b/LibreNMS/Data/Store/InfluxDB.php @@ -26,10 +26,10 @@ namespace LibreNMS\Data\Store; +use App\Polling\Measure\Measurement; use InfluxDB\Client; use InfluxDB\Driver\UDP; use LibreNMS\Config; -use LibreNMS\Data\Measure\Measurement; use Log; class InfluxDB extends BaseDatastore diff --git a/LibreNMS/Data/Store/OpenTSDB.php b/LibreNMS/Data/Store/OpenTSDB.php index a2d06dcb15..87ca3e97be 100644 --- a/LibreNMS/Data/Store/OpenTSDB.php +++ b/LibreNMS/Data/Store/OpenTSDB.php @@ -26,9 +26,9 @@ namespace LibreNMS\Data\Store; +use App\Polling\Measure\Measurement; use Carbon\Carbon; use LibreNMS\Config; -use LibreNMS\Data\Measure\Measurement; use Log; class OpenTSDB extends BaseDatastore diff --git a/LibreNMS/Data/Store/Prometheus.php b/LibreNMS/Data/Store/Prometheus.php index c28c4d9099..153d189e4b 100644 --- a/LibreNMS/Data/Store/Prometheus.php +++ b/LibreNMS/Data/Store/Prometheus.php @@ -26,10 +26,10 @@ namespace LibreNMS\Data\Store; +use App\Polling\Measure\Measurement; use GuzzleHttp\Exception\GuzzleException; use Illuminate\Support\Str; use LibreNMS\Config; -use LibreNMS\Data\Measure\Measurement; use Log; class Prometheus extends BaseDatastore diff --git a/LibreNMS/Data/Store/Rrd.php b/LibreNMS/Data/Store/Rrd.php index 8996a0aedc..601661e331 100644 --- a/LibreNMS/Data/Store/Rrd.php +++ b/LibreNMS/Data/Store/Rrd.php @@ -25,9 +25,9 @@ namespace LibreNMS\Data\Store; +use App\Polling\Measure\Measurement; use Illuminate\Support\Str; use LibreNMS\Config; -use LibreNMS\Data\Measure\Measurement; use LibreNMS\Exceptions\FileExistsException; use LibreNMS\Exceptions\RrdGraphException; use LibreNMS\Proc; diff --git a/LibreNMS/Data/Measure/Measurement.php b/app/Polling/Measure/Measurement.php similarity index 67% rename from LibreNMS/Data/Measure/Measurement.php rename to app/Polling/Measure/Measurement.php index 8269dc3bbc..7ef0f2ff3b 100644 --- a/LibreNMS/Data/Measure/Measurement.php +++ b/app/Polling/Measure/Measurement.php @@ -19,11 +19,11 @@ * * @link https://www.librenms.org * - * @copyright 2020 Tony Murray + * @copyright 2021 Tony Murray * @author Tony Murray */ -namespace LibreNMS\Data\Measure; +namespace App\Polling\Measure; class Measurement { @@ -31,10 +31,21 @@ class Measurement private $type; private $duration; - private function __construct(string $type) + private function __construct(string $type, float $duration = null) { $this->type = $type; $this->start = microtime(true); + if ($duration !== null) { + $this->duration = $duration; + } + } + + /** + * Create a measurement with an existing duration + */ + public static function make(string $type, float $duration): Measurement + { + return new static($type, $duration); } /** @@ -43,17 +54,15 @@ class Measurement * @param string $type * @return static */ - public static function start(string $type) + public static function start(string $type): Measurement { return new static($type); } /** * End the timer for this operation - * - * @return $this */ - public function end() + public function end(): Measurement { $this->duration = microtime(true) - $this->start; @@ -62,21 +71,22 @@ class Measurement /** * Get the duration of the operation - * - * @return float */ - public function getDuration() + public function getDuration(): float { return $this->duration; } /** * Get the type of the operation - * - * @return string */ - public function getType() + public function getType(): string { return $this->type; } + + public function manager(): MeasurementManager + { + return app(MeasurementManager::class); + } } diff --git a/LibreNMS/Data/Measure/MeasurementCollection.php b/app/Polling/Measure/MeasurementCollection.php similarity index 68% rename from LibreNMS/Data/Measure/MeasurementCollection.php rename to app/Polling/Measure/MeasurementCollection.php index 138e62dfd0..9d973b3dd2 100644 --- a/LibreNMS/Data/Measure/MeasurementCollection.php +++ b/app/Polling/Measure/MeasurementCollection.php @@ -19,42 +19,42 @@ * * @link https://www.librenms.org * - * @copyright 2020 Tony Murray + * @copyright 2021 Tony Murray * @author Tony Murray */ -namespace LibreNMS\Data\Measure; +namespace App\Polling\Measure; use Illuminate\Support\Collection; class MeasurementCollection extends Collection { - public function getTotalCount() + public function getTotalCount(): int { return $this->sumStat('getCount'); } - public function getTotalDuration() + public function getTotalDuration(): float { return $this->sumStat('getDuration'); } - public function getCountDiff() + public function getCountDiff(): int { return $this->sumStat('getCountDiff'); } - public function getDurationDiff() + public function getDurationDiff(): float { return $this->sumStat('getDurationDiff'); } - public function checkpoint() + public function checkpoint(): void { $this->each->checkpoint(); } - public function record(Measurement $measurement) + public function record(Measurement $measurement): void { $type = $measurement->getType(); @@ -65,10 +65,19 @@ class MeasurementCollection extends Collection $this->get($type)->add($measurement); } - private function sumStat($function) + public function getSummary(string $type): MeasurementSummary { - return $this->reduce(function ($sum, $measurement) use ($function) { - $sum += $measurement->$function(); + return $this->get($type, new MeasurementSummary($type)); + } + + /** + * @param string $method method on measurement class to call + * @return int|float + */ + private function sumStat(string $method) + { + return $this->reduce(function ($sum, $measurement) use ($method) { + $sum += $measurement->$method(); return $sum; }, 0); diff --git a/app/Polling/Measure/MeasurementManager.php b/app/Polling/Measure/MeasurementManager.php new file mode 100644 index 0000000000..4576c4d7a6 --- /dev/null +++ b/app/Polling/Measure/MeasurementManager.php @@ -0,0 +1,142 @@ +. + * + * @link https://www.librenms.org + * + * @copyright 2021 Tony Murray + * @author Tony Murray + */ + +namespace App\Polling\Measure; + +use DB; +use Illuminate\Database\Events\QueryExecuted; + +class MeasurementManager +{ + const SNMP_COLOR = "\e[0;36m"; + const DB_COLOR = "\e[1;33m"; + const DATASTORE_COLOR = "\e[0;32m"; + const NO_COLOR = "\e[0m"; + + /** + * @var MeasurementCollection + */ + private static $snmp; + + /** + * @var MeasurementCollection + */ + private static $db; + + public function __construct() + { + if (self::$snmp === null) { + self::$snmp = new MeasurementCollection(); + self::$db = new MeasurementCollection(); + } + } + + /** + * Register DB listener to record sql query stats + */ + public function listenDb(): void + { + DB::listen(function (QueryExecuted $event) { + $type = strtolower(substr($event->sql, 0, strpos($event->sql, ' '))); + $this->recordDb(Measurement::make($type, $event->time ? $event->time / 100 : 0)); + }); + } + + /** + * Update statistics for db operations + */ + public function recordDb(Measurement $measurement): void + { + self::$db->record($measurement); + } + + /** + * Print out the stats totals since the last checkpoint + */ + public function printChangedStats(): void + { + printf( + '>> %sSNMP%s: [%d/%.2fs] %sMySQL%s: [%d/%.2fs]', + self::SNMP_COLOR, + self::NO_COLOR, + self::$snmp->getCountDiff(), + self::$snmp->getDurationDiff(), + self::DB_COLOR, + self::NO_COLOR, + self::$db->getCountDiff(), + self::$db->getDurationDiff() + ); + + app('Datastore')->getStats()->each(function (MeasurementCollection $stats, $datastore) { + printf(' %s%s%s: [%d/%.2fs]', self::DATASTORE_COLOR, $datastore, self::NO_COLOR, $stats->getCountDiff(), $stats->getDurationDiff()); + }); + + $this->checkpoint(); + + echo PHP_EOL; + } + + /** + * Make a new checkpoint so to compare against + */ + public function checkpoint(): void + { + self::$snmp->checkpoint(); + self::$db->checkpoint(); + app('Datastore')->getStats()->each->checkpoint(); + } + + /** + * Record a measurement for snmp + */ + public function recordSnmp(Measurement $measurement): void + { + self::$snmp->record($measurement); + } + + /** + * Print global stat arrays + */ + public function printStats(): void + { + $this->printSummary('SNMP', self::$snmp, self::SNMP_COLOR); + $this->printSummary('SQL', self::$db, self::DB_COLOR); + + app('Datastore')->getStats()->each(function (MeasurementCollection $stats, string $datastore) { + $this->printSummary($datastore, $stats, self::DATASTORE_COLOR); + }); + } + + private function printSummary(string $name, MeasurementCollection $collection, string $color = ''): void + { + printf('%s%s%s [%d/%.2fs]:', $color, $name, $color ? self::NO_COLOR : '', $collection->getTotalCount(), $collection->getTotalDuration()); + + $collection->each(function (MeasurementSummary $stat) { + printf(' %s[%d/%.2fs]', ucfirst($stat->getType()), $stat->getCount(), $stat->getDuration()); + }); + + echo PHP_EOL; + } +} diff --git a/LibreNMS/Data/Measure/MeasurementSummary.php b/app/Polling/Measure/MeasurementSummary.php similarity index 78% rename from LibreNMS/Data/Measure/MeasurementSummary.php rename to app/Polling/Measure/MeasurementSummary.php index 0595a7eff6..2f2733a5e6 100644 --- a/LibreNMS/Data/Measure/MeasurementSummary.php +++ b/app/Polling/Measure/MeasurementSummary.php @@ -19,11 +19,11 @@ * * @link https://www.librenms.org * - * @copyright 2020 Tony Murray + * @copyright 2021 Tony Murray * @author Tony Murray */ -namespace LibreNMS\Data\Measure; +namespace App\Polling\Measure; class MeasurementSummary { @@ -39,7 +39,7 @@ class MeasurementSummary $this->type = $type; } - public function add(Measurement $measurement) + public function add(Measurement $measurement): void { $this->count++; $this->duration += $measurement->getDuration(); @@ -48,10 +48,8 @@ class MeasurementSummary /** * Get the measurement summary * ['count' => #, 'duration' => s] - * - * @return array */ - public function get() + public function get(): array { return [ 'count' => $this->count, @@ -59,33 +57,36 @@ class MeasurementSummary ]; } - public function getCount() + public function getCount(): int { return $this->count; } - public function getType() + public function getType(): string { return $this->type; } - public function getDuration() + public function getDuration(): float { return $this->duration; } - public function checkpoint() + /** + * Set a new checkpoint to compare against with diff methods + */ + public function checkpoint(): void { $this->checkpointCount = $this->count; $this->checkpointDuration = $this->duration; } - public function getCountDiff() + public function getCountDiff(): int { return $this->count - $this->checkpointCount; } - public function getDurationDiff() + public function getDurationDiff(): float { return $this->duration - $this->checkpointDuration; } diff --git a/app/Providers/AppServiceProvider.php b/app/Providers/AppServiceProvider.php index 256243a31b..69ca759d66 100644 --- a/app/Providers/AppServiceProvider.php +++ b/app/Providers/AppServiceProvider.php @@ -3,6 +3,7 @@ namespace App\Providers; use App\Models\Sensor; +use App\Polling\Measure\MeasurementManager; use Illuminate\Database\Eloquent\Relations\Relation; use Illuminate\Support\Facades\Blade; use Illuminate\Support\Facades\Log; @@ -38,8 +39,9 @@ class AppServiceProvider extends ServiceProvider * * @return void */ - public function boot() + public function boot(MeasurementManager $measure) { + $measure->listenDb(); \Illuminate\Pagination\Paginator::useBootstrap(); $this->app->booted('\LibreNMS\DB\Eloquent::initLegacyListeners'); diff --git a/discovery.php b/discovery.php index 28c90d0ba4..a46edf4a80 100755 --- a/discovery.php +++ b/discovery.php @@ -136,7 +136,8 @@ $string = $argv[0] . " $doing " . date(\LibreNMS\Config::get('dateformat.compact d_echo("$string\n"); if (! isset($options['q'])) { - printStats(); + echo PHP_EOL; + app(\App\Polling\Measure\MeasurementManager::class)->printStats(); } logfile($string); diff --git a/html/graph.php b/html/graph.php index b4277147d6..cd0e9c7911 100644 --- a/html/graph.php +++ b/html/graph.php @@ -34,5 +34,5 @@ if (Debug::isEnabled()) { echo '
'; printf('Runtime %.3fs', microtime(true) - $start); echo '
'; - printStats(); + app(\App\Polling\Measure\MeasurementManager::class)->printStats(); } diff --git a/includes/dbFacile.php b/includes/dbFacile.php index 87572ca6b6..03437c4345 100644 --- a/includes/dbFacile.php +++ b/includes/dbFacile.php @@ -18,7 +18,6 @@ */ use Illuminate\Database\QueryException; -use LibreNMS\Config; use LibreNMS\DB\Eloquent; use LibreNMS\Exceptions\DatabaseConnectException; use LibreNMS\Util\Laravel; @@ -110,8 +109,6 @@ function dbQuery($sql, $parameters = []) */ function dbInsert($data, $table) { - $time_start = microtime(true); - $sql = 'INSERT IGNORE INTO `' . $table . '` (`' . implode('`,`', array_keys($data)) . '`) VALUES (' . implode(',', dbPlaceHolders($data)) . ')'; try { @@ -120,7 +117,6 @@ function dbInsert($data, $table) dbHandleException(new QueryException($sql, $data, $pdoe)); } - recordDbStatistic('insert', $time_start); if ($result) { return Eloquent::DB()->getPdo()->lastInsertId(); } else { @@ -139,8 +135,6 @@ function dbInsert($data, $table) */ function dbBulkInsert($data, $table) { - $time_start = microtime(true); - // check that data isn't an empty array if (empty($data)) { return false; @@ -160,8 +154,6 @@ function dbBulkInsert($data, $table) try { $result = Eloquent::DB()->table($table)->insert((array) $data_chunk); - recordDbStatistic('insert', $time_start); - return $result; } catch (PDOException $pdoe) { // FIXME query? @@ -184,8 +176,6 @@ function dbBulkInsert($data, $table) */ function dbUpdate($data, $table, $where = null, $parameters = []) { - $time_start = microtime(true); - // need field name and placeholder value // but how merge these field placeholders with actual $parameters array for the WHERE clause $sql = 'UPDATE `' . $table . '` set '; @@ -213,8 +203,6 @@ function dbUpdate($data, $table, $where = null, $parameters = []) try { $result = Eloquent::DB()->update($sql, (array) $data); - recordDbStatistic('update', $time_start); - return $result; } catch (PDOException $pdoe) { dbHandleException(new QueryException($sql, $data, $pdoe)); @@ -225,8 +213,6 @@ function dbUpdate($data, $table, $where = null, $parameters = []) function dbDelete($table, $where = null, $parameters = []) { - $time_start = microtime(true); - $sql = 'DELETE FROM `' . $table . '`'; if ($where) { $sql .= ' WHERE ' . $where; @@ -238,8 +224,6 @@ function dbDelete($table, $where = null, $parameters = []) dbHandleException(new QueryException($sql, $parameters, $pdoe)); } - recordDbStatistic('delete', $time_start); - return $result; }//end dbDelete() @@ -253,8 +237,6 @@ function dbDelete($table, $where = null, $parameters = []) */ function dbDeleteOrphans($target_table, $parents) { - $time_start = microtime(true); - if (empty($parents)) { // don't delete all entries if parents is missing return false; @@ -288,8 +270,6 @@ function dbDeleteOrphans($target_table, $parents) dbHandleException(new QueryException($query, [], $pdoe)); } - recordDbStatistic('delete', $time_start); - return $result; } @@ -301,14 +281,11 @@ function dbDeleteOrphans($target_table, $parents) function dbFetchRows($sql, $parameters = []) { global $PDO_FETCH_ASSOC; - $time_start = microtime(true); try { $PDO_FETCH_ASSOC = true; $rows = Eloquent::DB()->select($sql, (array) $parameters); - recordDbStatistic('fetchrows', $time_start); - return $rows; } catch (PDOException $pdoe) { dbHandleException(new QueryException($sql, $parameters, $pdoe)); @@ -347,14 +324,11 @@ function dbFetch($sql, $parameters = []) function dbFetchRow($sql = null, $parameters = []) { global $PDO_FETCH_ASSOC; - $time_start = microtime(true); try { $PDO_FETCH_ASSOC = true; $row = Eloquent::DB()->selectOne($sql, (array) $parameters); - recordDbStatistic('fetchrow', $time_start); - return $row; } catch (PDOException $pdoe) { dbHandleException(new QueryException($sql, $parameters, $pdoe)); @@ -372,12 +346,10 @@ function dbFetchRow($sql = null, $parameters = []) function dbFetchCell($sql, $parameters = []) { global $PDO_FETCH_ASSOC; - $time_start = microtime(true); try { $PDO_FETCH_ASSOC = true; $row = Eloquent::DB()->selectOne($sql, (array) $parameters); - recordDbStatistic('fetchcell', $time_start); if ($row) { return reset($row); // shift first field off first row @@ -399,7 +371,6 @@ function dbFetchCell($sql, $parameters = []) function dbFetchColumn($sql, $parameters = []) { global $PDO_FETCH_ASSOC; - $time_start = microtime(true); $cells = []; @@ -410,8 +381,6 @@ function dbFetchColumn($sql, $parameters = []) } $PDO_FETCH_ASSOC = false; - recordDbStatistic('fetchcolumn', $time_start); - return $cells; } catch (PDOException $pdoe) { dbHandleException(new QueryException($sql, $parameters, $pdoe)); @@ -544,58 +513,6 @@ function dbGenPlaceholders($count) return '(' . implode(',', array_fill(0, $count, '?')) . ')'; } -/** - * Update statistics for db operations - * - * @param string $stat fetchcell, fetchrow, fetchrows, fetchcolumn, update, insert, delete - * @param float $start_time The time the operation started with 'microtime(true)' - * @return float The calculated run time - */ -function recordDbStatistic($stat, $start_time) -{ - global $db_stats, $db_stats_last; - - if (! isset($db_stats)) { - $db_stats = [ - 'ops' => [ - 'insert' => 0, - 'update' => 0, - 'delete' => 0, - 'fetchcell' => 0, - 'fetchcolumn' => 0, - 'fetchrow' => 0, - 'fetchrows' => 0, - ], - 'time' => [ - 'insert' => 0.0, - 'update' => 0.0, - 'delete' => 0.0, - 'fetchcell' => 0.0, - 'fetchcolumn' => 0.0, - 'fetchrow' => 0.0, - 'fetchrows' => 0.0, - ], - ]; - $db_stats_last = $db_stats; - } - - $runtime = microtime(true) - $start_time; - $db_stats['ops'][$stat]++; - $db_stats['time'][$stat] += $runtime; - - //double accounting corrections - if ($stat == 'fetchcolumn') { - $db_stats['ops']['fetchrows']--; - $db_stats['time']['fetchrows'] -= $runtime; - } - if ($stat == 'fetchcell') { - $db_stats['ops']['fetchrow']--; - $db_stats['time']['fetchrow'] -= $runtime; - } - - return $runtime; -} - /** * Synchronize a relationship to a list of related ids * diff --git a/includes/discovery/functions.inc.php b/includes/discovery/functions.inc.php index befb0e35dd..042b87c131 100644 --- a/includes/discovery/functions.inc.php +++ b/includes/discovery/functions.inc.php @@ -146,6 +146,10 @@ function discover_device(&$device, $force_module = false) $discovery_devices = Config::get('discovery_modules', []); $discovery_devices = ['core' => true] + $discovery_devices; + /** @var \App\Polling\Measure\MeasurementManager $measurements */ + $measurements = app(\App\Polling\Measure\MeasurementManager::class); + $measurements->checkpoint(); // don't count previous stats + foreach ($discovery_devices as $module => $module_status) { $os_module_status = Config::getOsSetting($device['os'], "discovery_modules.$module"); d_echo('Modules status: Global' . (isset($module_status) ? ($module_status ? '+ ' : '- ') : ' ')); @@ -173,7 +177,7 @@ function discover_device(&$device, $force_module = false) $module_time = substr($module_time, 0, 5); $module_mem = (memory_get_usage() - $start_memory); printf("\n>> Runtime for discovery module '%s': %.4f seconds with %s bytes\n", $module, $module_time, $module_mem); - printChangedStats(); + $measurements->printChangedStats(); echo "#### Unload disco module $module ####\n\n"; } elseif (isset($attribs['discover_' . $module]) && $attribs['discover_' . $module] == '0') { echo "Module [ $module ] disabled on host.\n\n"; diff --git a/includes/functions.php b/includes/functions.php index d69145936d..ab8ec44dfd 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -1449,134 +1449,6 @@ function q_bridge_bits2indices($hex_data) return $indices; } -/** - * Intialize global stat arrays - */ -function initStats() -{ - global $snmp_stats, $snmp_stats_last; - - if (! isset($snmp_stats)) { - $snmp_stats = [ - 'ops' => [ - 'snmpget' => 0, - 'snmpgetnext' => 0, - 'snmpwalk' => 0, - ], - 'time' => [ - 'snmpget' => 0.0, - 'snmpgetnext' => 0.0, - 'snmpwalk' => 0.0, - ], - ]; - $snmp_stats_last = $snmp_stats; - } -} - -/** - * Print out the stats totals since the last time this function was called - * - * @param bool $update_only Only update the stats checkpoint, don't print them - */ -function printChangedStats($update_only = false) -{ - global $snmp_stats, $db_stats; - global $snmp_stats_last, $db_stats_last; - $output = sprintf( - '>> SNMP: [%d/%.2fs] MySQL: [%d/%.2fs]', - array_sum($snmp_stats['ops'] ?? []) - array_sum($snmp_stats_last['ops'] ?? []), - array_sum($snmp_stats['time'] ?? []) - array_sum($snmp_stats_last['time'] ?? []), - array_sum($db_stats['ops'] ?? []) - array_sum($db_stats_last['ops'] ?? []), - array_sum($db_stats['time'] ?? []) - array_sum($db_stats_last['time'] ?? []) - ); - - foreach (app('Datastore')->getStats() as $datastore => $stats) { - /** @var \LibreNMS\Data\Measure\MeasurementCollection $stats */ - $output .= sprintf(' %s: [%d/%.2fs]', $datastore, $stats->getCountDiff(), $stats->getDurationDiff()); - $stats->checkpoint(); - } - - if (! $update_only) { - echo $output . PHP_EOL; - } - - // make a new checkpoint - $snmp_stats_last = $snmp_stats; - $db_stats_last = $db_stats; -} - -/** - * Print global stat arrays - */ -function printStats() -{ - global $snmp_stats, $db_stats; - - if ($snmp_stats) { - printf( - "SNMP [%d/%.2fs]: Get[%d/%.2fs] Getnext[%d/%.2fs] Walk[%d/%.2fs]\n", - array_sum($snmp_stats['ops']), - array_sum($snmp_stats['time']), - $snmp_stats['ops']['snmpget'], - $snmp_stats['time']['snmpget'], - $snmp_stats['ops']['snmpgetnext'], - $snmp_stats['time']['snmpgetnext'], - $snmp_stats['ops']['snmpwalk'], - $snmp_stats['time']['snmpwalk'] - ); - } - - if ($db_stats) { - printf( - "MySQL [%d/%.2fs]: Cell[%d/%.2fs] Row[%d/%.2fs] Rows[%d/%.2fs] Column[%d/%.2fs] Update[%d/%.2fs] Insert[%d/%.2fs] Delete[%d/%.2fs]\n", - array_sum($db_stats['ops']), - array_sum($db_stats['time']), - $db_stats['ops']['fetchcell'], - $db_stats['time']['fetchcell'], - $db_stats['ops']['fetchrow'], - $db_stats['time']['fetchrow'], - $db_stats['ops']['fetchrows'], - $db_stats['time']['fetchrows'], - $db_stats['ops']['fetchcolumn'], - $db_stats['time']['fetchcolumn'], - $db_stats['ops']['update'], - $db_stats['time']['update'], - $db_stats['ops']['insert'], - $db_stats['time']['insert'], - $db_stats['ops']['delete'], - $db_stats['time']['delete'] - ); - } - - foreach (app('Datastore')->getStats() as $datastore => $stats) { - /** @var \LibreNMS\Data\Measure\MeasurementCollection $stats */ - printf('%s [%d/%.2fs]:', $datastore, $stats->getTotalCount(), $stats->getTotalDuration()); - - foreach ($stats as $stat) { - /** @var \LibreNMS\Data\Measure\MeasurementSummary $stat */ - printf(' %s[%d/%.2fs]', ucfirst($stat->getType()), $stat->getCount(), $stat->getDuration()); - } - echo PHP_EOL; - } -} - -/** - * @param string $stat snmpget, snmpwalk - * @param float $start_time The time the operation started with 'microtime(true)' - * @return float The calculated run time - */ -function recordSnmpStatistic($stat, $start_time) -{ - global $snmp_stats; - initStats(); - - $runtime = microtime(true) - $start_time; - $snmp_stats['ops'][$stat]++; - $snmp_stats['time'][$stat] += $runtime; - - return $runtime; -} - function update_device_logo(&$device) { $icon = getImageName($device, false); diff --git a/includes/polling/functions.inc.php b/includes/polling/functions.inc.php index 5aa0c49831..75302fd432 100644 --- a/includes/polling/functions.inc.php +++ b/includes/polling/functions.inc.php @@ -310,7 +310,10 @@ function poll_device($device, $force_module = false) $device['status'] = $deviceModel->status; $device['status_reason'] = $deviceModel->status_reason; - printChangedStats(true); // don't count previous stats + /** @var \App\Polling\Measure\MeasurementManager $measurements */ + $measurements = app(\App\Polling\Measure\MeasurementManager::class); + $measurements->checkpoint(); // don't count previous stats + foreach (Config::get('poller_modules') as $module => $module_status) { $os_module_status = Config::get("os.{$device['os']}.poller_modules.$module"); d_echo('Modules status: Global' . (isset($module_status) ? ($module_status ? '+ ' : '- ') : ' ')); @@ -336,7 +339,7 @@ function poll_device($device, $force_module = false) $module_time = microtime(true) - $module_start; $module_mem = (memory_get_usage() - $start_memory); printf("\n>> Runtime for poller module '%s': %.4f seconds with %s bytes\n", $module, $module_time, $module_mem); - printChangedStats(); + $measurements->printChangedStats(); echo "#### Unload poller module $module ####\n\n"; // save per-module poller stats diff --git a/includes/snmp.inc.php b/includes/snmp.inc.php index 20bc761387..20972fd33e 100644 --- a/includes/snmp.inc.php +++ b/includes/snmp.inc.php @@ -16,6 +16,7 @@ */ use App\Models\Device; +use App\Polling\Measure\Measurement; use Illuminate\Support\Str; use LibreNMS\Config; use LibreNMS\Util\Debug; @@ -185,7 +186,7 @@ function gen_snmp_cmd($cmd, $device, $oids, $options = null, $mib = null, $mibdi function snmp_get_multi($device, $oids, $options = '-OQUs', $mib = null, $mibdir = null, $array = []) { - $time_start = microtime(true); + $measure = Measurement::start('snmpget'); if (! is_array($oids)) { $oids = explode(' ', $oids); @@ -212,14 +213,14 @@ function snmp_get_multi($device, $oids, $options = '-OQUs', $mib = null, $mibdir } } - recordSnmpStatistic('snmpget', $time_start); + $measure->manager()->recordSnmp($measure->end()); return $array; }//end snmp_get_multi() function snmp_get_multi_oid($device, $oids, $options = '-OUQn', $mib = null, $mibdir = null) { - $time_start = microtime(true); + $measure = Measurement::start('snmpget'); $oid_limit = get_device_oid_limit($device); if (! is_array($oids)) { @@ -255,7 +256,7 @@ function snmp_get_multi_oid($device, $oids, $options = '-OUQn', $mib = null, $mi } } - recordSnmpStatistic('snmpget', $time_start); + $measure->manager()->recordSnmp($measure->end()); return $array; }//end snmp_get_multi_oid() @@ -272,7 +273,7 @@ function snmp_get_multi_oid($device, $oids, $options = '-OUQn', $mib = null, $mi */ function snmp_get($device, $oid, $options = null, $mib = null, $mibdir = null) { - $time_start = microtime(true); + $measure = Measurement::start('snmpget'); if (strstr($oid, ' ')) { throw new Exception("snmp_get called for multiple OIDs: $oid"); @@ -282,7 +283,7 @@ function snmp_get($device, $oid, $options = null, $mib = null, $mibdir = null) $output = str_replace('Wrong Type (should be OBJECT IDENTIFIER): ', '', $output); $data = trim($output, "\\\" \n\r"); - recordSnmpStatistic('snmpget', $time_start); + $measure->manager()->recordSnmp($measure->end()); if (preg_match('/(No Such Instance|No Such Object|No more variables left|Authentication failure)/i', $data)) { return false; } elseif ($data || $data === '0') { @@ -305,13 +306,13 @@ function snmp_get($device, $oid, $options = null, $mib = null, $mibdir = null) */ function snmp_getnext($device, $oid, $options = null, $mib = null, $mibdir = null) { - $time_start = microtime(true); + $measure = Measurement::start('snmpgetnext'); $snmpcmd = [Config::get('snmpgetnext', 'snmpgetnext')]; $cmd = gen_snmp_cmd($snmpcmd, $device, $oid, $options, $mib, $mibdir); $data = trim(external_exec($cmd), "\" \n\r"); - recordSnmpStatistic('snmpgetnext', $time_start); + $measure->manager()->recordSnmp($measure->end()); if (preg_match('/(No Such Instance|No Such Object|No more variables left|Authentication failure)/i', $data)) { return false; } elseif ($data || $data === '0') { @@ -334,7 +335,7 @@ function snmp_getnext($device, $oid, $options = null, $mib = null, $mibdir = nul */ function snmp_getnext_multi($device, $oids, $options = '-OQUs', $mib = null, $mibdir = null, $array = []) { - $time_start = microtime(true); + $measure = Measurement::start('snmpgetnext'); if (! is_array($oids)) { $oids = explode(' ', $oids); } @@ -355,7 +356,7 @@ function snmp_getnext_multi($device, $oids, $options = '-OQUs', $mib = null, $mi } } } - recordSnmpStatistic('snmpgetnext', $time_start); + $measure->manager()->recordSnmp($measure->end()); return $array; }//end snmp_getnext_multi() @@ -366,7 +367,7 @@ function snmp_getnext_multi($device, $oids, $options = '-OQUs', $mib = null, $mi */ function snmp_check($device) { - $time_start = microtime(true); + $measure = Measurement::start('snmpget'); try { $oid = '.1.3.6.1.2.1.1.2.0'; @@ -379,7 +380,7 @@ function snmp_check($device) Log::debug("Device didn't respond to snmpget before {$e->getExceededTimeout()}s timeout"); } - recordSnmpStatistic('snmpget', $time_start); + $measure->manager()->recordSnmp($measure->end()); if ($code === 0) { return true; @@ -390,7 +391,7 @@ function snmp_check($device) function snmp_walk($device, $oid, $options = null, $mib = null, $mibdir = null) { - $time_start = microtime(true); + $measure = Measurement::start('snmpwalk'); $cmd = gen_snmpwalk_cmd($device, $oid, $options, $mib, $mibdir); $data = trim(external_exec($cmd)); @@ -410,7 +411,7 @@ function snmp_walk($device, $oid, $options = null, $mib = null, $mibdir = null) } } - recordSnmpStatistic('snmpwalk', $time_start); + $measure->manager()->recordSnmp($measure->end()); return $data; }//end snmp_walk() diff --git a/poller.php b/poller.php index de9cf12d36..b6896e0bbe 100755 --- a/poller.php +++ b/poller.php @@ -159,7 +159,8 @@ $string = $argv[0] . " $doing " . date(Config::get('dateformat.compact')) . " - d_echo("$string\n"); if (! isset($options['q'])) { - printStats(); + echo PHP_EOL; + app(\App\Polling\Measure\MeasurementManager::class)->printStats(); } logfile($string);