From 1cceafb88712abf6dff42bbaad874d8bbaaa94e2 Mon Sep 17 00:00:00 2001 From: Tony Murray Date: Wed, 17 Jul 2024 16:05:07 -0500 Subject: [PATCH] Improve Snmpsim usage to ease testing (#15471) * Snmpsim use python venv Patch to enable listening while minimizing output Update lnms dev:simulate, tests, and ./scripts/save-test-data.php removed old option to start snmpsim from older scripts, use lnms dev:simulate * Apply fixes from StyleCI * various fixes * Remove patch official package is updated --------- Co-authored-by: StyleCI Bot --- .github/workflows/test.yml | 8 +- LibreNMS/Util/CiHelper.php | 2 +- LibreNMS/Util/ModuleTestHelper.php | 8 +- LibreNMS/Util/Snmpsim.php | 190 +++++++-------------------- app/Console/Commands/DevSimulate.php | 60 +++++---- lang/en/commands.php | 1 + phpstan-baseline.neon | 10 -- scripts/collect-snmp-data.php | 8 -- scripts/save-test-data.php | 16 +-- tests/OSDiscoveryTest.php | 4 +- tests/bootstrap.php | 5 +- 11 files changed, 100 insertions(+), 212 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bf0dd3cc71..fdf4f074d2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -99,7 +99,7 @@ jobs: name: Pip install run: | python3 -m pip install --upgrade pip - python3 -m pip install --upgrade --user snmpsim pylint python-memcached mysqlclient + python3 -m pip install --upgrade --user pylint python-memcached mysqlclient - name: Composer validate run: | @@ -128,6 +128,10 @@ jobs: name: Composer install run: | composer install --prefer-dist --no-interaction --no-progress + - + name: Snmpsim setup + run: | + php lnms dev:simulate --setup-venv - name: Artisan dusk:chrome-driver if: matrix.skip-web-check != '1' @@ -168,7 +172,7 @@ jobs: name: Start SNMP if: matrix.skip-unit-check != '1' run: | - ~/.local/bin/snmpsim-command-responder-lite --data-dir=tests/snmpsim --agent-udpv4-endpoint=127.1.6.2:1162 --log-level=error --logging-method=file:/tmp/snmpsimd.log & + .python_venvs/snmpsim/bin/snmpsim-command-responder-lite --data-dir=tests/snmpsim --agent-udpv4-endpoint=127.1.6.2:1162 --log-level=error --logging-method=file:/tmp/snmpsimd.log & - name: lnms dev:check ci run: | diff --git a/LibreNMS/Util/CiHelper.php b/LibreNMS/Util/CiHelper.php index 0d038ed0a8..b276fc8240 100644 --- a/LibreNMS/Util/CiHelper.php +++ b/LibreNMS/Util/CiHelper.php @@ -295,7 +295,7 @@ class CiHelper $py_lint_cmd = [$this->checkPythonExec('pylint'), '-E', '-j', '0']; $files = $this->flags['full'] - ? explode(PHP_EOL, rtrim(shell_exec("find . -name '*.py' -not -path './vendor/*' -not -path './tests/*'"))) + ? explode(PHP_EOL, rtrim(shell_exec("find . -name '*.py' -not -path './vendor/*' -not -path './tests/*' -not -path './.python_venvs/*'"))) : $this->changed['python']; $py_lint_cmd = array_merge($py_lint_cmd, $files); diff --git a/LibreNMS/Util/ModuleTestHelper.php b/LibreNMS/Util/ModuleTestHelper.php index 741fbbf749..d70c924374 100644 --- a/LibreNMS/Util/ModuleTestHelper.php +++ b/LibreNMS/Util/ModuleTestHelper.php @@ -560,17 +560,17 @@ class ModuleTestHelper } // Remove existing device in case it didn't get removed previously - if (($existing_device = device_by_name($snmpsim->getIp())) && isset($existing_device['device_id'])) { + if (($existing_device = device_by_name($snmpsim->ip)) && isset($existing_device['device_id'])) { delete_device($existing_device['device_id']); } // Add the test device try { $new_device = new Device([ - 'hostname' => $snmpsim->getIp(), + 'hostname' => $snmpsim->ip, 'version' => 'v2c', 'community' => $this->file_name, - 'port' => $snmpsim->getPort(), + 'port' => $snmpsim->port, 'disabled' => 1, // disable to block normal pollers ]); (new ValidateDeviceAndCreate($new_device, true))->execute(); @@ -647,7 +647,7 @@ class ModuleTestHelper $data = array_merge_recursive($data, $this->dumpDb($device_id, $polled_modules, 'poller')); // Remove the test device, we don't need the debug from this - if ($device['hostname'] == $snmpsim->getIp()) { + if ($device['hostname'] == $snmpsim->ip) { Debug::set(false); delete_device($device_id); } diff --git a/LibreNMS/Util/Snmpsim.php b/LibreNMS/Util/Snmpsim.php index e5255f2b8f..eac7ef9ec9 100644 --- a/LibreNMS/Util/Snmpsim.php +++ b/LibreNMS/Util/Snmpsim.php @@ -25,166 +25,70 @@ namespace LibreNMS\Util; -use App; -use LibreNMS\Config; -use LibreNMS\Proc; +use Symfony\Component\Process\Process; -class Snmpsim +class Snmpsim extends Process { - private $snmprec_dir; - private $ip; - private $port; - private $log; - /** @var Proc */ - private $proc; + public readonly string $snmprec_dir; - public function __construct($ip = '127.1.6.1', $port = 1161, $log = '/tmp/snmpsimd.log') + public function __construct( + public readonly string $ip = '127.1.6.1', + public readonly int $port = 1161, + public readonly ?string $log_method = null) { - $this->ip = $ip; - $this->port = $port; - $this->log = $log; - $this->snmprec_dir = Config::get('install_dir') . '/tests/snmpsim/'; + $this->snmprec_dir = base_path('tests/snmpsim'); + + $cmd = [ + $this->getVenvPath('bin/snmpsim-command-responder-lite'), + "--data-dir={$this->snmprec_dir}", + "--agent-udpv4-endpoint={$this->ip}:{$this->port}", + '--log-level=error', + ]; + + if ($this->log_method !== null) { + $cmd[] = "--logging-method=$this->log_method"; + } + + parent::__construct($cmd, base_path()); + $this->setTimeout(null); // no timeout by default } - /** - * Run snmpsimd and fork it into the background - * Captures all output to the log - * - * @param int $wait Wait for x seconds after starting before returning - */ - public function fork($wait = 2) + public function waitForStartup(): string { - if ($this->isRunning()) { - echo "Snmpsim is already running!\n"; + $listen = $this->ip . ':' . $this->port; + $this->waitUntil(function ($type, $buffer) use ($listen, &$last) { + $last = $buffer; - return; - } + return $type == Process::ERR && str_contains($buffer, $listen); + }); - $cmd = $this->getCmd(); + return trim($last); + } - if (App::runningInConsole()) { - echo "Starting snmpsim listening on {$this->ip}:{$this->port}... \n"; - d_echo($cmd); - } + public function isVenvSetUp(): bool + { + return is_executable($this->getVenvPath('bin/snmpsim-command-responder-lite')); + } - $this->proc = new Proc($cmd); + public function setupVenv($print_output = false): void + { + $snmpsim_venv_path = $this->getVenvPath(); - if ($wait) { - sleep($wait); - } + if (! $this->isVenvSetUp()) { + \Log::info('Setting up snmpsim virtual env in ' . $snmpsim_venv_path); - if (App::runningInConsole() && ! $this->proc->isRunning()) { - // if starting failed, run snmpsim again and output to the console and validate the data - passthru($this->getCmd(false) . ' --validate-data'); + $setupProcess = new Process(['python', '-m', 'venv', $snmpsim_venv_path]); + $setupProcess->setTty($print_output); + $setupProcess->run(); - if (! is_executable($this->findSnmpsimd())) { - echo "\nCould not find snmpsim, you can install it with 'pip install snmpsim-lextudio'. If it is already installed, make sure snmpsimd, snmpsim-command-responder or snmpsimd.py is in PATH\n"; - } else { - echo "\nFailed to start Snmpsim. Scroll up for error.\n"; - } - exit(1); + $installProcess = new Process([$snmpsim_venv_path . '/bin/pip', 'install', 'snmpsim']); + $installProcess->setTty($print_output); + $installProcess->run(); } } - /** - * Stop and start the running snmpsim process - */ - public function restart() + public function getVenvPath(string $subdir = ''): string { - $this->stop(); - $this->proc = new Proc($this->getCmd()); - } - - public function stop() - { - if (isset($this->proc)) { - if ($this->proc->isRunning()) { - $this->proc->terminate(); - } - unset($this->proc); - } - } - - /** - * Run snmpsimd but keep it in the foreground - * Outputs to stdout - */ - public function run() - { - echo "Starting snmpsim listening on {$this->ip}:{$this->port}... \n"; - shell_exec($this->getCmd(false)); - } - - public function isRunning() - { - if (isset($this->proc)) { - return $this->proc->isRunning(); - } - - return false; - } - - /** - * @return string - */ - public function getDir() - { - return $this->snmprec_dir; - } - - /** - * @return string - */ - public function getIp() - { - return $this->ip; - } - - /** - * @return int - */ - public function getPort() - { - return $this->port; - } - - /** - * Generate the command for snmpsimd - * - * @param bool $with_log - * @return string - */ - private function getCmd($with_log = true) - { - $cmd = $this->findSnmpsimd(); - - $cmd .= " --data-dir={$this->snmprec_dir} --agent-udpv4-endpoint={$this->ip}:{$this->port}"; - - if (is_null($this->log)) { - $cmd .= ' --logging-method=null'; - } elseif ($with_log) { - $cmd .= " --logging-method=file:{$this->log}"; - } - - return $cmd; - } - - public function __destruct() - { - // unset $this->proc to make sure it isn't referenced - unset($this->proc); - } - - public function findSnmpsimd() - { - $cmd = Config::locateBinary('snmpsimd'); - if (! is_executable($cmd)) { - $cmd = Config::locateBinary('snmpsim-command-responder'); - if (! is_executable($cmd)) { - $cmd = Config::locateBinary('snmpsimd.py'); - } - } - - return $cmd; + return base_path('.python_venvs/snmpsim/' . $subdir); } } diff --git a/app/Console/Commands/DevSimulate.php b/app/Console/Commands/DevSimulate.php index 1d06eb8545..6ba57da8a5 100644 --- a/app/Console/Commands/DevSimulate.php +++ b/app/Console/Commands/DevSimulate.php @@ -8,7 +8,7 @@ use Illuminate\Support\Str; use LibreNMS\Util\Snmpsim; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputOption; -use Symfony\Component\Process\Process; +use Symfony\Component\Process\Exception\ProcessSignaledException; class DevSimulate extends LnmsCommand { @@ -38,6 +38,7 @@ class DevSimulate extends LnmsCommand $this->addArgument('file', InputArgument::OPTIONAL); $this->addOption('multiple', 'm', InputOption::VALUE_NONE); $this->addOption('remove', 'r', InputOption::VALUE_NONE); + $this->addOption('setup-venv', mode: InputOption::VALUE_NONE); } /** @@ -45,12 +46,8 @@ class DevSimulate extends LnmsCommand * * @return int */ - public function handle() + public function handle(): int { - $this->snmpsim = new Snmpsim(); - $snmprec_dir = $this->snmpsim->getDir(); - $listen = $this->snmpsim->getIp() . ':' . $this->snmpsim->getPort(); - $file = $this->argument('file'); if ($file && ! file_exists(base_path("tests/snmpsim/$file.snmprec"))) { $this->error("$file does not exist"); @@ -58,45 +55,52 @@ class DevSimulate extends LnmsCommand return 1; } - $snmpsim = new Process([ - $this->snmpsim->findSnmpsimd(), - "--data-dir=$snmprec_dir", - "--agent-udpv4-endpoint=$listen", - ]); - $snmpsim->setTimeout(null); + $this->snmpsim = new Snmpsim; + if (! $this->snmpsim->isVenvSetUp()) { + $this->line(trans('commands.dev:simulate.setup', ['dir' => $this->snmpsim->getVenvPath()])); + $this->snmpsim->setupVenv($this->getOutput()->isVeryVerbose()); + } - $snmpsim->run(function ($type, $buffer) use ($listen) { - if (Process::ERR === $type) { - if (Str::contains($buffer, $listen)) { - $this->line(trim($buffer)); - $this->started(); - $this->line(trans('commands.dev:simulate.exit')); - } - } - }); + if ($this->option('setup-venv')) { + return 0; // venv is set up exit + } - if (! $snmpsim->isSuccessful()) { - $this->line($snmpsim->getErrorOutput()); + $this->snmpsim->start(); + $this->line($this->snmpsim->waitForStartup()); + $this->started(); + $this->line(trans('commands.dev:simulate.exit')); + try { + $this->snmpsim->wait(); + } catch(ProcessSignaledException $e) { + $this->error($e->getMessage()); + + return 1; + } + + if (! $this->snmpsim->isSuccessful()) { + $this->line($this->snmpsim->getErrorOutput()); + + return 1; } return 0; } - private function started() + private function started(): void { if ($file = $this->argument('file')) { $this->addDevice($file); } } - private function addDevice($community) + private function addDevice($community): void { $hostname = $this->option('multiple') ? $community : 'snmpsim'; $device = Device::firstOrNew(['hostname' => $hostname]); $action = $device->exists ? 'updated' : 'added'; - $device->overwrite_ip = $this->snmpsim->getIp(); - $device->port = $this->snmpsim->getPort(); + $device->overwrite_ip = $this->snmpsim->ip; + $device->port = $this->snmpsim->port; $device->snmpver = 'v2c'; $device->transport = 'udp'; $device->community = $community; @@ -112,7 +116,7 @@ class DevSimulate extends LnmsCommand } } - private function queueRemoval($device_id) + private function queueRemoval($device_id): void { if (function_exists('pcntl_signal')) { pcntl_signal(SIGINT, function () { diff --git a/lang/en/commands.php b/lang/en/commands.php index 60af6fbb37..0e646662c1 100644 --- a/lang/en/commands.php +++ b/lang/en/commands.php @@ -63,6 +63,7 @@ return [ 'exit' => 'Ctrl-C to stop', 'removed' => 'Device :id removed', 'updated' => 'Device :hostname (:id) updated', + 'setup' => 'Setting up snmpsim venv in :dir', ], 'device:add' => [ 'description' => 'Add a new device', diff --git a/phpstan-baseline.neon b/phpstan-baseline.neon index 5d8a453fad..72bc85515c 100644 --- a/phpstan-baseline.neon +++ b/phpstan-baseline.neon @@ -150,16 +150,6 @@ parameters: count: 1 path: LibreNMS/Util/Graph.php - - - message: "#^Property LibreNMS\\\\Util\\\\Snmpsim\\:\\:\\$proc \\(LibreNMS\\\\Proc\\) in isset\\(\\) is not nullable\\.$#" - count: 2 - path: LibreNMS/Util/Snmpsim.php - - - - message: "#^Unreachable statement \\- code above always terminates\\.$#" - count: 1 - path: LibreNMS/Util/Snmpsim.php - - message: "#^Static method Silber\\\\Bouncer\\\\BouncerFacade\\:\\:role\\(\\) invoked with 0 parameters, 1 required\\.$#" count: 1 diff --git a/scripts/collect-snmp-data.php b/scripts/collect-snmp-data.php index 8e06b00bc0..144c2d475e 100755 --- a/scripts/collect-snmp-data.php +++ b/scripts/collect-snmp-data.php @@ -5,7 +5,6 @@ use Illuminate\Support\Str; use LibreNMS\Exceptions\InvalidModuleException; use LibreNMS\Util\Debug; use LibreNMS\Util\ModuleTestHelper; -use LibreNMS\Util\Snmpsim; $install_dir = realpath(__DIR__ . '/..'); chdir($install_dir); @@ -23,18 +22,11 @@ $options = getopt( 'variant:', 'file:', 'debug', - 'snmpsim', 'full', 'help', ] ); -if (isset($options['snmpsim'])) { - $snmpsim = new Snmpsim(); - $snmpsim->run(); - exit; -} - if (isset($options['v'])) { $variant = $options['v']; } elseif (isset($options['variant'])) { diff --git a/scripts/save-test-data.php b/scripts/save-test-data.php index 88546909b5..12b0003651 100755 --- a/scripts/save-test-data.php +++ b/scripts/save-test-data.php @@ -20,7 +20,6 @@ $options = getopt( 'no-save', 'file:', 'debug', - 'snmpsim', 'help', ] ); @@ -32,12 +31,6 @@ Debug::setVerbose( Debug::set(isset($options['d']) || isset($options['debug'])) ); -if (isset($options['snmpsim'])) { - $snmpsim = new Snmpsim(); - $snmpsim->run(); - exit; -} - if (isset($options['h']) || isset($options['help']) || ! (isset($options['o']) || isset($options['os']) || isset($options['m']) || isset($options['modules'])) @@ -120,9 +113,10 @@ if (isset($options['f'])) { // Now use the saved data to update the saved database data $snmpsim = new Snmpsim(); -$snmpsim->fork(); -$snmpsim_ip = $snmpsim->getIp(); -$snmpsim_port = $snmpsim->getPort(); +$snmpsim->setupVenv(); +$snmpsim->start(); +echo "Waiting for snmpsim to initialize...\n"; +$snmpsim->waitForStartup(); if (! $snmpsim->isRunning()) { echo "Failed to start snmpsim, make sure it is installed, working, and there are no bad snmprec files.\n"; @@ -130,8 +124,6 @@ if (! $snmpsim->isRunning()) { exit(1); } -echo "Pausing 10 seconds to allow snmpsim to initialize...\n"; -sleep(10); echo "\n"; try { diff --git a/tests/OSDiscoveryTest.php b/tests/OSDiscoveryTest.php index 87e7f211af..2a926c74b3 100644 --- a/tests/OSDiscoveryTest.php +++ b/tests/OSDiscoveryTest.php @@ -143,9 +143,9 @@ class OSDiscoveryTest extends TestCase private function genDevice($community): Device { return new Device([ - 'hostname' => $this->getSnmpsim()->getIP(), + 'hostname' => $this->getSnmpsim()->ip, 'snmpver' => 'v2c', - 'port' => $this->getSnmpsim()->getPort(), + 'port' => $this->getSnmpsim()->port, 'timeout' => 3, 'retries' => 0, 'snmp_max_repeaters' => 10, diff --git a/tests/bootstrap.php b/tests/bootstrap.php index ed13c14aff..5e4aac38c4 100644 --- a/tests/bootstrap.php +++ b/tests/bootstrap.php @@ -36,10 +36,11 @@ chdir($install_dir); ini_set('display_errors', '1'); //error_reporting(E_ALL & ~E_WARNING); -$snmpsim = new Snmpsim('127.1.6.2', 1162, null); +$snmpsim = new Snmpsim('127.1.6.2', 1162); if (getenv('SNMPSIM')) { if (! getenv('GITHUB_ACTIONS')) { - $snmpsim->fork(6); + $snmpsim->setupVenv(); + $snmpsim->start(); } // make PHP hold on a reference to $snmpsim so it doesn't get destructed