From 12f8bb2040aafcd0335fdff02c4151478bbabaa0 Mon Sep 17 00:00:00 2001 From: Tony Murray Date: Thu, 3 Aug 2023 19:29:30 -0500 Subject: [PATCH] MAC Vendor OUI use scheduler (#15187) * MAC Vendor OUI use scheduler Add command to update `lnms maintenance:fetch-ouis` Show vendor column in tables if mac_oui.enabled is set to true Improve scheduler validation handle non-standard install directories and systems without systemd Add index to table to improve speed and improve mac->vendor lookup speed Scheduled weekly with random wait to prevent stampeding herd issues for upstream drop oui update from daily * MAC Vendor OUI use scheduler Add command to update `lnms maintenance:fetch-ouis` Show vendor column in tables if mac_oui.enabled is set to true * Lint fixes and better prefix detection * update schema file --- LibreNMS/Util/Rewrite.php | 24 +++- LibreNMS/Validations/Scheduler.php | 45 +++++- app/Console/Commands/MaintenanceFetchOuis.php | 136 ++++++++++++++++++ app/Console/Kernel.php | 16 +++ daily.php | 9 -- daily.sh | 6 - ..._08_02_120455_vendor_ouis_unique_index.php | 30 ++++ dist/librenms-scheduler.cron | 1 + dist/librenms-scheduler.service | 2 +- includes/functions.php | 77 ---------- includes/html/pages/device/nac.inc.php | 8 +- includes/html/pages/device/port/arp.inc.php | 6 +- includes/html/pages/device/port/fdb.inc.php | 6 +- includes/html/pages/device/ports/fdb.inc.php | 6 +- includes/html/pages/search/arp.inc.php | 2 +- includes/html/pages/search/fdb.inc.php | 6 +- includes/html/pages/search/mac.inc.php | 7 +- lang/en/commands.php | 18 +++ misc/db_schema.yaml | 3 +- 19 files changed, 273 insertions(+), 135 deletions(-) create mode 100644 app/Console/Commands/MaintenanceFetchOuis.php create mode 100644 database/migrations/2023_08_02_120455_vendor_ouis_unique_index.php create mode 100644 dist/librenms-scheduler.cron diff --git a/LibreNMS/Util/Rewrite.php b/LibreNMS/Util/Rewrite.php index 1f6ff52d10..450e882ad4 100644 --- a/LibreNMS/Util/Rewrite.php +++ b/LibreNMS/Util/Rewrite.php @@ -26,6 +26,7 @@ namespace LibreNMS\Util; use App\Models\Device; +use Illuminate\Support\Arr; use Illuminate\Support\Facades\DB; use LibreNMS\Config; @@ -151,20 +152,29 @@ class Rewrite * Extract the OUI and match it against database values * * @param string $mac - * @return string|null + * @return string */ - public static function readableOUI($mac) + public static function readableOUI($mac): string { $oui = substr($mac, 0, 6); - $result = DB::table('vendor_ouis')->where('oui', $oui)->value('vendor'); + $results = DB::table('vendor_ouis') + ->where('oui', 'like', "$oui%") // possible matches + ->orderBy('oui', 'desc') // so we can check longer ones first if we have them + ->pluck('vendor', 'oui'); - if ($result === 'IEEE Registration Authority') { - // Then we may have a shorter prefix, so let's try them one after the other, ordered by probability - $result = DB::table('vendor_ouis')->whereIn('oui', [substr($mac, 0, 9), substr($mac, 0, 7)])->value('vendor'); + if (count($results) == 1) { + return Arr::first($results); } - return $result ?: ''; + // Then we may have a shorter prefix, so let's try them one after the other + foreach ($results as $oui => $vendor) { + if (str_starts_with($mac, $oui)) { + return $vendor; + } + } + + return ''; } /** diff --git a/LibreNMS/Validations/Scheduler.php b/LibreNMS/Validations/Scheduler.php index b18e9d3ddb..22ef54b5b5 100644 --- a/LibreNMS/Validations/Scheduler.php +++ b/LibreNMS/Validations/Scheduler.php @@ -23,6 +23,8 @@ namespace LibreNMS\Validations; use Illuminate\Support\Facades\Cache; +use LibreNMS\Config; +use LibreNMS\ValidationResult; use LibreNMS\Validator; class Scheduler extends BaseValidation @@ -36,8 +38,47 @@ class Scheduler extends BaseValidation public function validate(Validator $validator): void { if (! Cache::has('scheduler_working')) { - $validator->fail('Scheduler is not running', - "cp /opt/librenms/dist/librenms-scheduler.service /opt/librenms/dist/librenms-scheduler.timer /etc/systemd/system/\nsystemctl enable librenms-scheduler.timer\nsystemctl start librenms-scheduler.timer"); + $commands = $this->generateCommands($validator); + $validator->result(ValidationResult::fail('Scheduler is not running')->setFix($commands)); } } + + /** + * @param Validator $validator + * @return array + */ + private function generateCommands(Validator $validator): array + { + $commands = []; + $systemctl_bin = Config::locateBinary('systemctl'); + $base_dir = rtrim($validator->getBaseDir(), '/'); + + if (is_executable($systemctl_bin)) { + // systemd exists + if ($base_dir === '/opt/librenms') { + // standard install dir + $commands[] = 'sudo cp /opt/librenms/dist/librenms-scheduler.service /opt/librenms/dist/librenms-scheduler.timer /etc/systemd/system/'; + } else { + // non-standard install dir + $commands[] = "sudo sh -c 'sed \"s#/opt/librenms#$base_dir#\" $base_dir/dist/librenms-scheduler.service > /etc/systemd/system/librenms-scheduler.service'"; + $commands[] = "sudo sh -c 'sed \"s#/opt/librenms#$base_dir#\" $base_dir/dist/librenms-scheduler.timer > /etc/systemd/system/librenms-scheduler.timer'"; + } + $commands[] = 'sudo systemctl enable librenms-scheduler.timer'; + $commands[] = 'sudo systemctl start librenms-scheduler.timer'; + + return $commands; + } + + // non-systemd use cron + if ($base_dir === '/opt/librenms') { + $commands[] = 'sudo cp /opt/librenms/dist/librenms-scheduler.cron /etc/cron.d/'; + + return $commands; + } + + // non-standard install dir + $commands[] = "sudo sh -c 'sed \"s#/opt/librenms#$base_dir#\" $base_dir/dist/librenms-scheduler.cron > /etc/cron.d/librenms-scheduler.cron'"; + + return $commands; + } } diff --git a/app/Console/Commands/MaintenanceFetchOuis.php b/app/Console/Commands/MaintenanceFetchOuis.php new file mode 100644 index 0000000000..7f97e8a3fa --- /dev/null +++ b/app/Console/Commands/MaintenanceFetchOuis.php @@ -0,0 +1,136 @@ +addOption('force', null, InputOption::VALUE_NONE); + $this->addOption('wait', null, InputOption::VALUE_NONE); + } + + /** + * Execute the console command. + */ + public function handle(): int + { + $force = $this->option('force'); + + if (Config::get('mac_oui.enabled') !== true && ! $force) { + $this->line(trans('commands.maintenance:fetch-ouis.disabled', ['setting' => 'mac_oui.enabled'])); + + if (! $this->confirm(trans('commands.maintenance:fetch-ouis.enable_question'))) { + return 0; + } + + Config::persist('mac_oui.enabled', true); + } + + // We want to refresh after at least 6 days + $lock = Cache::lock('vendor_oui_db_refresh', 86400 * $this->min_refresh_days); + if (! $lock->get() && ! $force) { + $this->warn(trans('commands.maintenance:fetch-ouis.recently_fetched')); + + return 0; + } + + // wait for 0-15 minutes to prevent stampeding herd + if ($this->option('wait')) { + $seconds = rand(1, $this->max_wait_seconds); + $minutes = (int) round($seconds / 60); + $this->info(trans_choice('commands.maintenance:fetch-ouis.waiting', $minutes, ['minutes' => $minutes])); + sleep($seconds); + } + + $this->line(trans('commands.maintenance:fetch-ouis.starting')); + + try { + $this->line(' -> ' . trans('commands.maintenance:fetch-ouis.downloading') . ' ...'); + $csv_data = \LibreNMS\Util\Http::client()->get($this->mac_oui_url)->body(); + + // convert the csv into an array to be consumed by upsert + $this->line(' -> ' . trans('commands.maintenance:fetch-ouis.processing') . ' ...'); + $ouis = $this->buildOuiList($csv_data); + + $this->line(' -> ' . trans('commands.maintenance:fetch-ouis.saving') . ' ...'); + $count = 0; + foreach (array_chunk($ouis, $this->upsert_chunk_size) as $oui_chunk) { + $count += DB::table('vendor_ouis')->upsert($oui_chunk, 'oui'); + } + + $this->info(trans_choice('commands.maintenance:fetch-ouis.success', $count, ['count' => $count])); + + return 0; + } catch (\Exception|\ErrorException $e) { + $this->error(trans('commands.maintenance:fetch-ouis.error')); + $this->error('Exception: ' . get_class($e)); + $this->error($e); + + $lock->release(); // We did not succeed, so we'll try again next time + + return 1; + } + } + + private function buildOuiList(string $csv_data): array + { + $ouis = []; + + foreach (explode("\n", rtrim($csv_data)) as $csv_line) { + // skip comments + if (str_starts_with($csv_line, '#')) { + continue; + } + + [$oui, $vendor] = str_getcsv($csv_line, "\t"); + + $oui = strtolower(str_replace(':', '', $oui)); // normalize oui + $prefix_index = strpos($oui, '/'); + + // check for non-/24 oui + if ($prefix_index !== false) { + // find prefix length + $prefix_length = (int) substr($oui, $prefix_index + 1); + + // 4 bits per character: /28 = 7 /36 = 9 + $substring_length = (int) floor($prefix_length / 4); + + $oui = substr($oui, 0, $substring_length); + } + + // Add to the list of vendor ids + $ouis[] = [ + 'vendor' => $vendor, + 'oui' => $oui, + ]; + + if ($this->verbosity == OutputInterface::VERBOSITY_DEBUG) { + $this->line(trans('commands.maintenance:fetch-ouis.vendor_update', ['vendor' => $vendor, 'oui' => $oui])); + } + } + + return $ouis; + } +} diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 939fd1c5ea..7c02a0762a 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -2,9 +2,11 @@ namespace App\Console; +use App\Console\Commands\MaintenanceFetchOuis; use Illuminate\Console\Scheduling\Schedule; use Illuminate\Foundation\Console\Kernel as ConsoleKernel; use Illuminate\Support\Facades\Cache; +use LibreNMS\Config; use LibreNMS\Util\Debug; use LibreNMS\Util\Version; @@ -19,6 +21,7 @@ class Kernel extends ConsoleKernel protected function schedule(Schedule $schedule): void { $this->scheduleMarkWorking($schedule); + $this->scheduleMaintenance($schedule); // should be after all others } /** @@ -73,4 +76,17 @@ class Kernel extends ConsoleKernel Cache::put('scheduler_working', now(), now()->addMinutes(6)); })->everyFiveMinutes(); } + + /** + * Schedule maintenance tasks + */ + private function scheduleMaintenance(Schedule $schedule): void + { + $maintenance_log_file = Config::get('log_dir') . '/maintenance.log'; + + $schedule->command(MaintenanceFetchOuis::class, ['--wait']) + ->weeklyOn(0, '1:00') + ->onOneServer() + ->appendOutputTo($maintenance_log_file); + } } diff --git a/daily.php b/daily.php index 5b80606775..2ee8dd4451 100644 --- a/daily.php +++ b/daily.php @@ -358,15 +358,6 @@ if ($options['f'] === 'peeringdb') { } } -if ($options['f'] === 'mac_oui') { - $lock = Cache::lock('vendor_oui_db', 86000); - if ($lock->get()) { - $res = mac_oui_to_database(); - $lock->release(); - exit($res); - } -} - if ($options['f'] === 'refresh_os_cache') { echo 'Clearing OS cache' . PHP_EOL; if (is_file(Config::get('install_dir') . '/cache/os_defs.cache')) { diff --git a/daily.sh b/daily.sh index 13a54e519c..f6b790fd51 100755 --- a/daily.sh +++ b/daily.sh @@ -341,7 +341,6 @@ main () { # and clean up the db. status_run 'Updating SQL-Schema' './lnms migrate --force --no-interaction --isolated' status_run 'Cleaning up DB' "'$DAILY_SCRIPT' cleanup" - status_run 'Updating Mac OUI data' "$DAILY_SCRIPT mac_oui" ;; post-pull) # re-check dependencies after pull with the new code @@ -371,7 +370,6 @@ main () { status_run 'Cleaning up DB' "$DAILY_SCRIPT cleanup" status_run 'Fetching notifications' "$DAILY_SCRIPT notifications" status_run 'Caching PeeringDB data' "$DAILY_SCRIPT peeringdb" - status_run 'Caching Mac OUI data' "$DAILY_SCRIPT mac_oui" ;; cleanup) # Cleanups @@ -406,10 +404,6 @@ main () { peeringdb) options=("peeringdb") call_daily_php "${options[@]}" - ;; - mac_oui) - options=("mac_oui") - call_daily_php "${options[@]}" esac fi } diff --git a/database/migrations/2023_08_02_120455_vendor_ouis_unique_index.php b/database/migrations/2023_08_02_120455_vendor_ouis_unique_index.php new file mode 100644 index 0000000000..d940ce6e50 --- /dev/null +++ b/database/migrations/2023_08_02_120455_vendor_ouis_unique_index.php @@ -0,0 +1,30 @@ +string('oui', 12)->change(); + $table->unique(['oui']); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('vendor_ouis', function (Blueprint $table) { + $table->dropUnique('vendor_ouis_oui_unique'); + $table->string('oui')->change(); + }); + } +}; diff --git a/dist/librenms-scheduler.cron b/dist/librenms-scheduler.cron new file mode 100644 index 0000000000..7b8fbd84a0 --- /dev/null +++ b/dist/librenms-scheduler.cron @@ -0,0 +1 @@ +* * * * * php /opt/librenms/artisan schedule:run --no-ansi --no-interaction > /dev/null 2>&1 diff --git a/dist/librenms-scheduler.service b/dist/librenms-scheduler.service index 1cd2ee4d4c..ee20ae894d 100644 --- a/dist/librenms-scheduler.service +++ b/dist/librenms-scheduler.service @@ -6,6 +6,6 @@ Type=oneshot StandardOutput=null StandardError=null WorkingDirectory=/opt/librenms/ -ExecStart=/usr/bin/env php artisan schedule:run +ExecStart=/usr/bin/env php artisan schedule:run --no-ansi --no-interaction User=librenms Group=librenms diff --git a/includes/functions.php b/includes/functions.php index 1d266a01c0..174289b609 100755 --- a/includes/functions.php +++ b/includes/functions.php @@ -1163,83 +1163,6 @@ function q_bridge_bits2indices($hex_data) return $indices; } -/** - * Function to generate Mac OUI Cache - */ -function mac_oui_to_database() -{ - // Refresh timer - $mac_oui_refresh_int_min = 86400 * rand(7, 11); // 7 days + a random number between 0 and 4 days - - $lock = Cache::lock('vendor_oui_db_refresh', $mac_oui_refresh_int_min); // We want to refresh after at least $mac_oui_refresh_int_min - - if (Config::get('mac_oui.enabled') !== true) { - echo 'Mac OUI integration disabled' . PHP_EOL; - - return 0; - } - - if ($lock->get()) { - echo 'Storing Mac OUI in the database' . PHP_EOL; - try { - $mac_oui_url = 'https://www.wireshark.org/download/automated/data/manuf'; - //$mac_oui_url = 'https://gitlab.com/wireshark/wireshark/-/raw/master/manuf'; - //$mac_oui_url_mirror = 'https://raw.githubusercontent.com/wireshark/wireshark/master/manuf'; - - echo ' -> Downloading ...' . PHP_EOL; - $get = \LibreNMS\Util\Http::client()->get($mac_oui_url); - echo ' -> Processing CSV ...' . PHP_EOL; - $csv_data = $get->body(); - - // Process each line of the CSV data - foreach (explode("\n", $csv_data) as $csv_line) { - unset($oui); - $entry = str_getcsv($csv_line, "\t"); - - $length = strlen($entry[0]); - $prefix = strtolower(str_replace(':', '', $entry[0])); - $vendor = $entry[1]; - - if (is_array($entry) && count($entry) >= 2 && $length == 8) { - // We have a standard OUI xx:xx:xx - $oui = $prefix; - } elseif (is_array($entry) && count($entry) >= 2 && $length == 20) { - // We have a smaller range (xx:xx:xx:X or xx:xx:xx:xx:X) - if (substr($prefix, -2) == '28') { - $oui = substr($prefix, 0, 7); - } elseif (substr($prefix, -2) == '36') { - $oui = substr($prefix, 0, 9); - } - } - - if (isset($oui)) { - // Store the OUI for the vendor in the database - DB::table('vendor_ouis')->insert([ - 'vendor' => $vendor, - 'oui' => $oui, - ]); - - echo "Adding $oui for $vendor" . PHP_EOL; - } - } - } catch (Exception $e) { - echo 'Error processing Mac OUI:' . PHP_EOL; - echo 'Exception: ' . get_class($e) . PHP_EOL; - echo $e->getMessage() . PHP_EOL; - - $lock->release(); // We did not succeed, so we'll try again next time - - return 1; - } - } else { - echo 'Not able to acquire lock, skipping mac database update' . PHP_EOL; - - return 1; - } - - return 0; -} - /** * Function to generate PeeringDB Cache */ diff --git a/includes/html/pages/device/nac.inc.php b/includes/html/pages/device/nac.inc.php index 68c77d9e4c..ff9a11c206 100644 --- a/includes/html/pages/device/nac.inc.php +++ b/includes/html/pages/device/nac.inc.php @@ -37,11 +37,7 @@ if ($device['os'] === 'vrp') { Port MAC Address -Vendor'; -} -?> + Vendor IP Address >Vlan Domain @@ -89,7 +85,7 @@ if (\LibreNMS\Config::get('mac_oui.enabled') === true) { "nac_authz": function (column, row) { var value = row[column.id]; - if (value === 'authorizationSuccess' || value === 'sussess') { + if (value === 'authorizationSuccess' || value === 'sussess') { //typo in huawei MIB so we must keep sussess return ""; } else if (value === 'authorizationFailed') { diff --git a/includes/html/pages/device/port/arp.inc.php b/includes/html/pages/device/port/arp.inc.php index 73ac281f3c..94ee1a3ab0 100644 --- a/includes/html/pages/device/port/arp.inc.php +++ b/includes/html/pages/device/port/arp.inc.php @@ -5,11 +5,7 @@ $no_refresh = true; MAC address -Vendor'; -} -?> + Vendor IPv4 address Remote device Remote interface diff --git a/includes/html/pages/device/port/fdb.inc.php b/includes/html/pages/device/port/fdb.inc.php index 5426dd8e96..c89bbb7729 100644 --- a/includes/html/pages/device/port/fdb.inc.php +++ b/includes/html/pages/device/port/fdb.inc.php @@ -6,11 +6,7 @@ $no_refresh = true; MAC Address -Vendor'; -} -?> + Vendor IPv4 Address Port Vlan diff --git a/includes/html/pages/device/ports/fdb.inc.php b/includes/html/pages/device/ports/fdb.inc.php index 91aeb6bf10..55dec42481 100644 --- a/includes/html/pages/device/ports/fdb.inc.php +++ b/includes/html/pages/device/ports/fdb.inc.php @@ -5,11 +5,7 @@ $no_refresh = true; MAC Address -Vendor'; -} -?> + Vendor IPv4 Address Port Description diff --git a/includes/html/pages/search/arp.inc.php b/includes/html/pages/search/arp.inc.php index 2e6623bab7..cd6861fdd6 100644 --- a/includes/html/pages/search/arp.inc.php +++ b/includes/html/pages/search/arp.inc.php @@ -6,7 +6,7 @@ MAC Address - Vendor + Vendor IP Address Device Interface diff --git a/includes/html/pages/search/fdb.inc.php b/includes/html/pages/search/fdb.inc.php index e52a4948a6..285954e66e 100755 --- a/includes/html/pages/search/fdb.inc.php +++ b/includes/html/pages/search/fdb.inc.php @@ -7,11 +7,7 @@ Device MAC Address -Vendor'; -} -?> + Vendor IPv4 Address Port Vlan diff --git a/includes/html/pages/search/mac.inc.php b/includes/html/pages/search/mac.inc.php index 82b8a60881..98fed098ea 100644 --- a/includes/html/pages/search/mac.inc.php +++ b/includes/html/pages/search/mac.inc.php @@ -8,11 +8,8 @@ Device Interface MAC Address -Vendor'; -} -?> Description + Vendor + Description diff --git a/lang/en/commands.php b/lang/en/commands.php index 92b660e0c3..c74be9299f 100644 --- a/lang/en/commands.php +++ b/lang/en/commands.php @@ -157,6 +157,24 @@ return [ 'optionValue' => 'Selected :option is invalid. Should be one of: :values', ], ], + 'maintenance:fetch-ouis' => [ + 'description' => 'Fetch MAC OUIs and cache them to display vendor names for MAC addresses', + 'options' => [ + 'force' => 'Ignore any settings or locks that prevent the command from being run', + 'wait' => 'Wait a random amount of time, used by the scedueler to prevent server strain', + ], + 'disabled' => 'Mac OUI integration disabled (:setting)', + 'enable_question' => 'Enable Mac OUI integration and scheduled fetching?', + 'recently_fetched' => 'MAC OUI Database fetched recently, skipping update.', + 'waiting' => 'Waiting :minutes minute before attempting MAC OUI update|Waiting :minutes minutes before attempting MAC OUI update', + 'starting' => 'Storing Mac OUI in the database', + 'downloading' => 'Downloading', + 'processing' => 'Processing CSV', + 'saving' => 'Saving results', + 'success' => 'Successfully updated OUI/Vendor mappings. :count modified OUI|Successfully updated. :count modified OUIs', + 'error' => 'Error processing Mac OUI:', + 'vendor_update' => 'Adding OUI :oui for :vendor', + ], 'plugin:disable' => [ 'description' => 'Disable all plugins with the given name', 'arguments' => [ diff --git a/misc/db_schema.yaml b/misc/db_schema.yaml index b1166570e8..7c487c0eab 100644 --- a/misc/db_schema.yaml +++ b/misc/db_schema.yaml @@ -2088,9 +2088,10 @@ vendor_ouis: Columns: - { Field: id, Type: 'bigint unsigned', 'Null': false, Extra: auto_increment } - { Field: vendor, Type: varchar(255), 'Null': false, Extra: '' } - - { Field: oui, Type: varchar(255), 'Null': false, Extra: '' } + - { Field: oui, Type: varchar(12), 'Null': false, Extra: '' } Indexes: PRIMARY: { Name: PRIMARY, Columns: [id], Unique: true, Type: BTREE } + vendor_ouis_oui_unique: { Name: vendor_ouis_oui_unique, Columns: [oui], Unique: true, Type: BTREE } vlans: Columns: - { Field: vlan_id, Type: 'int unsigned', 'Null': false, Extra: auto_increment }