clones/librenms
1
0
Fork 0
mirror of https://github.com/librenms/librenms.git synced 2024-09-21 10:28:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: Collection and output of db and snmp stats (#5049)

Browse Source
This commit is contained in:
Tony Murray 2016-11-23 00:57:19 -06:00 committed by Neil Lathwood
parent 6708c7cb81
commit db240cb4eb
10 changed files with 162 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
discovery.php
View File

@ -15,7 +15,6 @@ $init_modules = array('discovery');
require __DIR__ . '/includes/init.php';
$start = microtime(true);
$runtime_stats = array();
$sqlparams = array();
$options = getopt('h:m:i:n:d::v::a::q', array('os:','type:'));
@ -133,8 +132,7 @@ $string = $argv[0]." $doing ".date($config['dateformat']['compact'])." - $discov
d_echo("$string\n");
if (!isset($options['q'])) {
echo ('MySQL: Cell['.($db_stats['fetchcell'] + 0).'/'.round(($db_stats['fetchcell_sec'] + 0), 2).'s]'.' Row['.($db_stats['fetchrow'] + 0).'/'.round(($db_stats['fetchrow_sec'] + 0), 2).'s]'.' Rows['.($db_stats['fetchrows'] + 0).'/'.round(($db_stats['fetchrows_sec'] + 0), 2).'s]'.' Column['.($db_stats['fetchcol'] + 0).'/'.round(($db_stats['fetchcol_sec'] + 0), 2).'s]'.' Update['.($db_stats['update'] + 0).'/'.round(($db_stats['update_sec'] + 0), 2).'s]'.' Insert['.($db_stats['insert'] + 0).'/'.round(($db_stats['insert_sec'] + 0), 2).'s]'.' Delete['.($db_stats['delete'] + 0).'/'.round(($db_stats['delete_sec'] + 0), 2).'s]');
echo "\n";
printStats();
}
logfile($string);

12
html/graph.php
View File

@ -36,9 +36,9 @@ require $config['install_dir'] . '/html/includes/graphs/graph.inc.php';
rrdtool_close();
$end = microtime(true);
$run = ($end - $start);
d_echo('<br />Runtime '.$run.' secs');
d_echo('<br />MySQL: Cell '.($db_stats['fetchcell'] + 0).'/'.round(($db_stats['fetchcell_sec'] + 0), 3).'s'.' Row '.($db_stats['fetchrow'] + 0).'/'.round(($db_stats['fetchrow_sec'] + 0), 3).'s'.' Rows '.($db_stats['fetchrows'] + 0).'/'.round(($db_stats['fetchrows_sec'] + 0), 3).'s'.' Column '.($db_stats['fetchcol'] + 0).'/'.round(($db_stats['fetchcol_sec'] + 0), 3).'s');
if ($debug) {
echo '<br />';
printf("Runtime %.3fs", microtime(true) - $start);
echo '<br />';
printStats();
}

6
html/index.php
View File

@ -260,10 +260,8 @@ $gentime = substr($runtime, 0, 5);
# FIXME - move this
if ($config['page_gen']) {
echo(' <br />MySQL: Cell '.($db_stats['fetchcell']+0).'/'.round($db_stats['fetchcell_sec']+0, 3).'s'.
' Row '.($db_stats['fetchrow']+0). '/'.round($db_stats['fetchrow_sec']+0, 3).'s'.
' Rows '.($db_stats['fetchrows']+0).'/'.round($db_stats['fetchrows_sec']+0, 3).'s'.
' Column '.($db_stats['fetchcol']+0). '/'.round($db_stats['fetchcol_sec']+0, 3).'s');
echo '<br />';
printStats();
$fullsize = memory_get_usage();
unset($cache);

101
includes/dbFacile.php
View File

@ -70,8 +70,8 @@ function dbQuery($sql, $parameters = array())
function dbInsert($data, $table)
{
global $database_link, $db_stats;
dbInitStats();
global $database_link;
$time_start = microtime(true);
// the following block swaps the parameters if they were given in the wrong order.
// it allows the method to work for those that would rather it (or expect it to)
@ -86,7 +86,6 @@ function dbInsert($data, $table)
$sql = 'INSERT INTO `'.$table.'` (`'.implode('`,`', array_keys($data)).'`) VALUES ('.implode(',', dbPlaceHolders($data)).')';
$time_start = microtime(true);
dbBeginTransaction();
$result = dbQuery($sql, $data);
if ($result) {
@ -99,14 +98,10 @@ function dbInsert($data, $table)
}
dbRollbackTransaction();
// $id = false;
$id = null;
}
// logfile($fullSql);
$time_end = microtime(true);
$db_stats['insert_sec'] += (float)number_format(($time_end - $time_start), 8);
$db_stats['insert']++;
recordDbStatistic('insert', $time_start);
return $id;
}//end dbInsert()
@ -120,9 +115,7 @@ function dbInsert($data, $table)
function dbBulkInsert($data, $table)
{
global $db_stats;
dbInitStats();
$time_start = microtime(true);
// the following block swaps the parameters if they were given in the wrong order.
// it allows the method to work for those that would rather it (or expect it to)
// follow closer with SQL convention:
@ -156,14 +149,9 @@ function dbBulkInsert($data, $table)
$values .= "(".$rowvalues.")";
}
$time_start = microtime(true);
$result = dbQuery($sql.$values);
// logfile($fullSql);
$time_end = microtime(true);
$db_stats['insert_sec'] += (float)number_format(($time_end - $time_start), 8);
$db_stats['insert']++;
recordDbStatistic('insert', $time_start);
return $result;
}//end dbBulkInsert()
@ -176,8 +164,8 @@ function dbBulkInsert($data, $table)
function dbUpdate($data, $table, $where = null, $parameters = array())
{
global $fullSql, $database_link, $db_stats;
dbInitStats();
global $fullSql, $database_link;
$time_start = microtime(true);
// the following block swaps the parameters if they were given in the wrong order.
// it allows the method to work for those that would rather it (or expect it to)
@ -204,7 +192,6 @@ function dbUpdate($data, $table, $where = null, $parameters = array())
$data = array_merge($data, $parameters);
}
$time_start = microtime(true);
if (dbQuery($sql, $data)) {
$return = mysqli_affected_rows($database_link);
} else {
@ -213,10 +200,7 @@ function dbUpdate($data, $table, $where = null, $parameters = array())
$return = false;
}
$time_end = microtime(true);
$db_stats['update_sec'] += (float)number_format(($time_end - $time_start), 8);
$db_stats['update']++;
recordDbStatistic('update', $time_start);
return $return;
}//end dbUpdate()
@ -224,12 +208,17 @@ function dbUpdate($data, $table, $where = null, $parameters = array())
function dbDelete($table, $where = null, $parameters = array())
{
global $database_link;
$time_start = microtime(true);
$sql = 'DELETE FROM `'.$table.'`';
if ($where) {
$sql .= ' WHERE '.$where;
}
if (dbQuery($sql, $parameters)) {
$result = dbQuery($sql, $parameters);
recordDbStatistic('delete', $time_start);
if ($result) {
return mysqli_affected_rows($database_link);
} else {
return false;
@ -245,8 +234,7 @@ function dbDelete($table, $where = null, $parameters = array())
function dbFetchRows($sql, $parameters = array(), $nocache = false)
{
global $db_stats, $config;
dbInitStats();
global $config;
if ($config['memcached']['enable'] && $nocache === false) {
$result = $config['memcached']['resource']->get(hash('sha512', $sql.'|'.serialize($parameters)));
@ -268,17 +256,15 @@ function dbFetchRows($sql, $parameters = array(), $nocache = false)
if ($config['memcached']['enable'] && $nocache === false) {
$config['memcached']['resource']->set(hash('sha512', $sql.'|'.serialize($parameters)), $rows, $config['memcached']['ttl']);
}
recordDbStatistic('fetchrows', $time_start);
return $rows;
}
mysqli_free_result($result);
$time_end = microtime(true);
$db_stats['fetchrows_sec'] += (float)number_format(($time_end - $time_start), 8);
$db_stats['fetchrows']++;
// no records, thus return empty array
// which should evaluate to false, and will prevent foreach notices/warnings
recordDbStatistic('fetchrows', $time_start);
return array();
}//end dbFetchRows()
@ -313,8 +299,7 @@ function dbFetch($sql, $parameters = array(), $nocache = false)
function dbFetchRow($sql = null, $parameters = array(), $nocache = false)
{
global $db_stats, $config;
dbInitStats();
global $config;
if ($config['memcached']['enable'] && $nocache === false) {
$result = $config['memcached']['resource']->get(hash('sha512', $sql.'|'.serialize($parameters)));
@ -328,10 +313,8 @@ function dbFetchRow($sql = null, $parameters = array(), $nocache = false)
if ($result) {
$row = mysqli_fetch_assoc($result);
mysqli_free_result($result);
$time_end = microtime(true);
$db_stats['fetchrow_sec'] += (float)number_format(($time_end - $time_start), 8);
$db_stats['fetchrow']++;
recordDbStatistic('fetchrow', $time_start);
if ($config['memcached']['enable'] && $nocache === false) {
$config['memcached']['resource']->set(hash('sha512', $sql.'|'.serialize($parameters)), $row, $config['memcached']['ttl']);
@ -350,21 +333,14 @@ function dbFetchRow($sql = null, $parameters = array(), $nocache = false)
function dbFetchCell($sql, $parameters = array(), $nocache = false)
{
global $db_stats, $config;
dbInitStats();
$time_start = microtime(true);
$row = dbFetchRow($sql, $parameters, $nocache);
$row = dbFetchRow($sql, $parameters, $nocache);
recordDbStatistic('fetchcell', $time_start);
if ($row) {
return array_shift($row);
// shift first field off first row
}
$time_end = microtime(true);
$db_stats['fetchcell_sec'] += (float)number_format(($time_end - $time_start), 8);
$db_stats['fetchcell']++;
return null;
}//end dbFetchCell()
@ -377,20 +353,13 @@ function dbFetchCell($sql, $parameters = array(), $nocache = false)
function dbFetchColumn($sql, $parameters = array(), $nocache = false)
{
global $db_stats;
dbInitStats();
$time_start = microtime(true);
$cells = array();
foreach (dbFetch($sql, $parameters, $nocache) as $row) {
$cells[] = array_shift($row);
}
$time_end = microtime(true);
$db_stats['fetchcol_sec'] += (float)number_format(($time_end - $time_start), 8);
$db_stats['fetchcol']++;
recordDbStatistic('fetchcolumn', $time_start);
return $cells;
}//end dbFetchColumn()
@ -503,28 +472,6 @@ function dbPrepareData($data)
return $values;
}//end dbPrepareData()
function dbInitStats()
{
global $db_stats;
if (!isset($db_stats)) {
$db_stats = array(
'insert' => 0,
'insert_sec' => 0.0,
'update' => 0,
'update_sec' => 0.0,
'fetchcell' => 0,
'fetchcell_sec' => 0.0,
'fetchcol' => 0,
'fetchcol_sec' => 0.0,
'fetchrow' => 0,
'fetchrow_sec' => 0.0,
'fetchrows' => 0,
'fetchrows_sec' => 0.0,
);
}
}
/**
* Given a data array, this returns an array of placeholders
* These may be question marks, or ":email" type

2
includes/discovery/functions.inc.php
View File

@ -148,7 +148,7 @@ function discover_device($device, $options = null)
include "includes/discovery/$module.inc.php";
$module_time = microtime(true) - $module_start;
$module_time = substr($module_time, 0, 5);
echo "\n>> Runtime for discovery module '$module': $module_time seconds\n";
printf("\n>> Runtime for discovery module '%s': %.4f seconds\n", $module, $module_time);
echo "#### Unload disco module $module ####\n\n";
} elseif (isset($attribs['discover_' . $module]) && $attribs['discover_' . $module] == '0') {
echo "Module [ $module ] disabled on host.\n\n";

113
includes/functions.php
View File

@ -1740,3 +1740,116 @@ function can_skip_discovery($needles, $haystack, $name)
}
return false;
}
/**
* Intialize global stat arrays
*/
function initStats()
{
global $snmp_stats, $db_stats;
if (!isset($snmp_stats)) {
$snmp_stats = array(
'snmpget' => 0,
'snmpget_sec' => 0.0,
'snmpwalk' => 0,
'snmpwalk_sec' => 0.0,
);
}
if (!isset($db_stats)) {
$db_stats = array(
'insert' => 0,
'insert_sec' => 0.0,
'update' => 0,
'update_sec' => 0.0,
'fetchcell' => 0,
'fetchcell_sec' => 0.0,
'fetchcol' => 0,
'fetchcol_sec' => 0.0,
'fetchrow' => 0,
'fetchrow_sec' => 0.0,
'fetchrows' => 0,
'fetchrows_sec' => 0.0,
);
}
}
/**
* Print global stat arrays
*/
function printStats()
{
global $snmp_stats, $db_stats;
printf(
"SNMP: Get[%d/%.2fs] Walk [%d/%.2fs]\n",
$snmp_stats['snmpget'],
$snmp_stats['snmpget_sec'],
$snmp_stats['snmpwalk'],
$snmp_stats['snmpwalk_sec']
);
printf(
"MySQL: Cell[%d/%.2fs] Row[%d/%.2fs] Rows[%d/%.2fs] Column[%d/%.2fs] Update[%d/%.2fs] Insert[%d/%.2fs] Delete[%d/%.2fs]\n",
$db_stats['fetchcell'],
$db_stats['fetchcell_sec'],
$db_stats['fetchrow'],
$db_stats['fetchrow_sec'],
$db_stats['fetchrows'],
$db_stats['fetchrows_sec'],
$db_stats['fetchcolumn'],
$db_stats['fetchcolumn_sec'],
$db_stats['update'],
$db_stats['update_sec'],
$db_stats['insert'],
$db_stats['insert_sec'],
$db_stats['delete'],
$db_stats['delete_sec']
);
}
/**
* Update statistics for db operations
*
* @param string $stat fetchcell, fetchrow, fetchrows, fetchcolumn, update, insert, delete
* @param float $start_time The time the operation started with 'microtime(true)'
* @return float The calculated run time
*/
function recordDbStatistic($stat, $start_time)
{
global $db_stats;
initStats();
$runtime = microtime(true) - $start_time;
$db_stats[$stat]++;
$db_stats["${stat}_sec"] += $runtime;
//double accounting corrections
if ($stat == 'fetchcolumn') {
$db_stats['fetchrows']--;
$db_stats['fetchrows_sec'] -= $runtime;
}
if ($stat == 'fetchcell') {
$db_stats['fetchrow']--;
$db_stats['fetchrow_sec'] -= $runtime;
}
return $runtime;
}
/**
* @param string $stat snmpget, snmpwalk
* @param float $start_time The time the operation started with 'microtime(true)'
* @return float The calculated run time
*/
function recordSnmpStatistic($stat, $start_time)
{
global $snmp_stats;
initStats();
$runtime = microtime(true) - $start_time;
$snmp_stats[$stat]++;
$snmp_stats["${stat}_sec"] += $runtime;
return $runtime;
}

4
includes/polling/functions.inc.php
View File

@ -147,7 +147,7 @@ function poll_sensor($device, $class, $unit)
function poll_device($device, $options)
{
global $config, $device, $polled_devices, $db_stats, $memcache;
global $config, $device, $polled_devices, $memcache;
$attribs = get_dev_attribs($device['device_id']);
$device['snmp_max_repeaters'] = $attribs['snmp_max_repeaters'];
@ -259,7 +259,7 @@ function poll_device($device, $options)
echo "\n#### Load poller module $module ####\n";
include "includes/polling/$module.inc.php";
$module_time = microtime(true) - $module_start;
echo "\n>> Runtime for poller module '$module': $module_time seconds\n";
printf("\n>> Runtime for poller module '%s': %.4f seconds\n", $module, $module_time);
echo "#### Unload poller module $module ####\n\n";
// save per-module poller stats

19
includes/snmp.inc.php
View File

@ -192,7 +192,7 @@ function gen_snmp_cmd($cmd, $device, $oids, $options = null, $mib = null, $mibdi
function snmp_get_multi($device, $oids, $options = '-OQUs', $mib = null, $mibdir = null)
{
global $runtime_stats;
$time_start = microtime(true);
if (is_array($oids)) {
$oids = implode(' ', $oids);
@ -201,7 +201,7 @@ function snmp_get_multi($device, $oids, $options = '-OQUs', $mib = null, $mibdir
$cmd = gen_snmpget_cmd($device, $oids, $options, $mib, $mibdir);
$data = trim(external_exec($cmd));
$runtime_stats['snmpget']++;
$array = array();
foreach (explode("\n", $data) as $entry) {
list($oid,$value) = explode('=', $entry, 2);
@ -213,12 +213,13 @@ function snmp_get_multi($device, $oids, $options = '-OQUs', $mib = null, $mibdir
}
}
recordSnmpStatistic('snmpget', $time_start);
return $array;
}//end snmp_get_multi()
function snmp_get_multi_oid($device, $oids, $options = '-OUQn', $mib = null, $mibdir = null)
{
global $runtime_stats;
$time_start = microtime(true);
if (is_array($oids)) {
$oids = implode(' ', $oids);
@ -227,7 +228,6 @@ function snmp_get_multi_oid($device, $oids, $options = '-OUQn', $mib = null, $mi
$cmd = gen_snmpget_cmd($device, $oids, $options, $mib, $mibdir);
$data = trim(external_exec($cmd));
$runtime_stats['snmpget']++;
$array = array();
foreach (explode("\n", $data) as $entry) {
list($oid,$value) = explode('=', $entry, 2);
@ -238,12 +238,13 @@ function snmp_get_multi_oid($device, $oids, $options = '-OUQn', $mib = null, $mi
}
}
recordSnmpStatistic('snmpget', $time_start);
return $array;
}//end snmp_get_multi_oid()
function snmp_get($device, $oid, $options = null, $mib = null, $mibdir = null)
{
global $runtime_stats;
$time_start = microtime(true);
if (strstr($oid, ' ')) {
echo report_this_text("snmp_get called for multiple OIDs: $oid");
@ -252,8 +253,7 @@ function snmp_get($device, $oid, $options = null, $mib = null, $mibdir = null)
$cmd = gen_snmpget_cmd($device, $oid, $options, $mib, $mibdir);
$data = trim(external_exec($cmd));
$runtime_stats['snmpget']++;
recordSnmpStatistic('snmpget', $time_start);
if (is_string($data) && (preg_match('/(No Such Instance|No Such Object|No more variables left|Authentication failure)/i', $data))) {
return false;
} elseif ($data || $data === '0') {
@ -266,7 +266,7 @@ function snmp_get($device, $oid, $options = null, $mib = null, $mibdir = null)
function snmp_walk($device, $oid, $options = null, $mib = null, $mibdir = null)
{
global $runtime_stats;
$time_start = microtime(true);
$cmd = gen_snmpwalk_cmd($device, $oid, $options, $mib, $mibdir);
$data = trim(external_exec($cmd));
@ -285,8 +285,7 @@ function snmp_walk($device, $oid, $options = null, $mib = null, $mibdir = null)
}
}
$runtime_stats['snmpwalk']++;
recordSnmpStatistic('snmpwalk', $time_start);
return $data;
}//end snmp_walk()

8
poller.php
View File

@ -27,7 +27,7 @@ echo "MySQL: ".$versions['mysql_ver']."\n";
echo "RRDTool: ".$versions['rrdtool_ver']."\n";
echo "SNMP: ".$versions['netsnmp_ver']."\n";
$options = getopt('h:m:i:n:r::d::v::a::f::');
$options = getopt('h:m:i:n:r::d::v::a::f::q');
if ($options['h'] == 'odd') {
$options['n'] = '1';
@ -149,9 +149,9 @@ if ($polled_devices) {
$string = $argv[0]." $doing ".date($config['dateformat']['compact'])." - $polled_devices devices polled in $poller_time secs";
d_echo("$string\n");
echo ("\n".'MySQL: Cell['.($db_stats['fetchcell'] + 0).'/'.round(($db_stats['fetchcell_sec'] + 0), 2).'s]'.' Row['.($db_stats['fetchrow'] + 0).'/'.round(($db_stats['fetchrow_sec'] + 0), 2).'s]'.' Rows['.($db_stats['fetchrows'] + 0).'/'.round(($db_stats['fetchrows_sec'] + 0), 2).'s]'.' Column['.($db_stats['fetchcol'] + 0).'/'.round(($db_stats['fetchcol_sec'] + 0), 2).'s]'.' Update['.($db_stats['update'] + 0).'/'.round(($db_stats['update_sec'] + 0), 2).'s]'.' Insert['.($db_stats['insert'] + 0).'/'.round(($db_stats['insert_sec'] + 0), 2).'s]'.' Delete['.($db_stats['delete'] + 0).'/'.round(($db_stats['delete_sec'] + 0), 2).'s]');
echo "\n";
if (!isset($options['q'])) {
printStats();
}
logfile($string);
rrdtool_close();

1
tests/bootstrap.php
View File

@ -32,7 +32,6 @@ require $install_dir . '/includes/defaults.inc.php';
$config['install_dir'] = $install_dir;
$config['mib_dir'] = $install_dir . '/mibs';
$config['snmpget'] = 'snmpget';
$runtime_stats = array('snmpget' => 0, 'snmpwalk' => 0);
// initialize the class loader and add custom mappings
require $install_dir . '/LibreNMS/ClassLoader.php';