From b6e18db766bf15bf576d95d569d260c106efbdf6 Mon Sep 17 00:00:00 2001 From: Tony Murray Date: Wed, 10 May 2017 02:57:10 -0500 Subject: [PATCH] fix: page/graph load speed: part 1 (#6611) * fix: page/graph load speed: part 1 Speed up load_all_os() by caching the full $config['os'] array Function time goes from 1.13648s -> 0.00073s The cache is updated during discovery, but only if it is >22hrs old. Additionally, the cache is not used and a full yaml load is forced if the cache is >24hrs old. The cache time is controlled by $config['os_def_cache_time'], I don't see any reason anyone would change that time. * Delete Session.php * Include the cache directory. --- cache/.gitignore | 4 +++ discovery.php | 2 ++ includes/common.php | 69 +++++++++++++++++++++++++++++---------- includes/defaults.inc.php | 3 ++ includes/init.php | 8 ++--- tests/OSDiscoveryTest.php | 5 ++- 6 files changed, 66 insertions(+), 25 deletions(-) create mode 100644 cache/.gitignore diff --git a/cache/.gitignore b/cache/.gitignore new file mode 100644 index 0000000000..5e7d2734cf --- /dev/null +++ b/cache/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore diff --git a/discovery.php b/discovery.php index 4028a4536e..467c29c8bb 100755 --- a/discovery.php +++ b/discovery.php @@ -118,6 +118,8 @@ if (get_lock('schema') === false) { require 'includes/sql-schema/update.php'; } +update_os_cache(); // will only update if needed + $discovered_devices = 0; if (!empty($config['distributed_poller_group'])) { diff --git a/includes/common.php b/includes/common.php index f35de22c8e..7658d0e2ee 100644 --- a/includes/common.php +++ b/includes/common.php @@ -1576,30 +1576,63 @@ function load_os(&$device) } /** - * @param array $restricted + * Load all OS, optionally load just the OS used by existing devices + * Default cache time is 1 day. Controlled by os_def_cache_time. + * + * @param bool $existing Only load OS that have existing OS in the database + * @param bool $cached Load os definitions from the cache file */ -function load_all_os($restricted = array()) +function load_all_os($existing = false, $cached = true) { global $config; - if (!empty($restricted)) { - $list = $restricted; + $cache_file = $config['install_dir'] . '/cache/os_defs.cache'; + + if ($cached && is_file($cache_file) && (time() - filemtime($cache_file) < $config['os_def_cache_time'])) { + // Cached + $os_defs = unserialize(file_get_contents($cache_file)); + + if ($existing) { + // remove unneeded os + $os_defs = array_diff_key($os_defs, dbFetchColumn('SELECT DISTINCT(`os`) FROM `devices`')); + } + + $config['os'] = array_replace_recursive($os_defs, $config['os']); } else { - $list = glob($config['install_dir'].'/includes/definitions/*.yaml'); - if (count($list) == count($config['os'])) { - // already fully loaded - return; + // load from yaml + if ($existing) { + $os_list = array_map(function ($os) use ($config) { + return $config['install_dir'] . '/includes/definitions/'. $os . '.yaml'; + }, dbFetchColumn('SELECT DISTINCT(`os`) FROM `devices`')); + } else { + $os_list = glob($config['install_dir'].'/includes/definitions/*.yaml'); + } + + foreach ($os_list as $file) { + $tmp = Symfony\Component\Yaml\Yaml::parse(file_get_contents($file)); + + if (isset($config['os'][$tmp['os']])) { + $config['os'][$tmp['os']] = array_replace_recursive($tmp, $config['os'][$tmp['os']]); + } else { + $config['os'][$tmp['os']] = $tmp; + } } } +} - foreach ($list as $file) { - $tmp = Symfony\Component\Yaml\Yaml::parse( - file_get_contents($file) - ); - if (isset($config['os'][$tmp['os']])) { - $config['os'][$tmp['os']] = array_replace_recursive($tmp, $config['os'][$tmp['os']]); - } else { - $config['os'][$tmp['os']] = $tmp; - } +/** + * Update the OS cache file cache/os_defs.cache + */ +function update_os_cache() +{ + global $config; + $cache_file = $config['install_dir'] . '/cache/os_defs.cache'; + $cache_keep_time = $config['os_def_cache_time'] - 7200; // 2hr buffer + + if (!is_file($cache_file) || time() - filemtime($cache_file) > $cache_keep_time) { + d_echo('Updating os_def.cache... '); + load_all_os(false, false); + file_put_contents($cache_file, serialize($config['os'])); + d_echo("Done\n"); } } @@ -1704,7 +1737,7 @@ function get_user_pref($name, $default = null, $user_id = null) { global $user_prefs; - if (array_key_exists($name, $user_prefs)) { + if (is_array($user_prefs) && array_key_exists($name, $user_prefs)) { return $user_prefs[$name]; } diff --git a/includes/defaults.inc.php b/includes/defaults.inc.php index deaf711a88..6d3d5c849f 100644 --- a/includes/defaults.inc.php +++ b/includes/defaults.inc.php @@ -191,6 +191,9 @@ $config['snmp']['v3'][0]['cryptoalgo'] = 'AES'; // Devices must respond to icmp by default $config['icmp_check'] = true; +// The amount of time to keep the OS cache +$config['os_def_cache_time'] = 86400; + // Autodiscovery Settings $config['autodiscovery']['xdp'] = true; $config['autodiscovery']['xdp_exclude']['sysdesc_regexp'][] = '/-K9W8-/'; // Cisco Lightweight Access Point diff --git a/includes/init.php b/includes/init.php index 42cc18fa0f..a4bc80a9e5 100644 --- a/includes/init.php +++ b/includes/init.php @@ -152,12 +152,8 @@ if (module_selected('web', $init_modules)) { $config['title_image'] = 'images/librenms_logo_'.$config['site_style'].'.svg'; } require $install_dir . '/html/includes/vars.inc.php'; - $tmp_list = dbFetchRows('SELECT DISTINCT(`os`) FROM `devices`'); - $os_list = array(); - foreach ($tmp_list as $k => $v) { - $os_list[] = $config['install_dir'].'/includes/definitions/'. $v['os'] . '.yaml'; - } - load_all_os($os_list); + + load_all_os(true); } $console_color = new Console_Color2(); diff --git a/tests/OSDiscoveryTest.php b/tests/OSDiscoveryTest.php index 6526bf7b86..bbe83783e1 100644 --- a/tests/OSDiscoveryTest.php +++ b/tests/OSDiscoveryTest.php @@ -143,7 +143,10 @@ class OSDiscoveryTest extends \PHPUnit_Framework_TestCase { global $config; - load_all_os(); // make sure all OS are loaded + // make sure all OS are loaded + if (count($config['os']) < count(glob($config['install_dir'].'/includes/definitions/*.yaml'))) { + load_all_os(); + } $excluded_os = array( 'default',