clones/librenms
1
0
Fork 0
mirror of https://github.com/librenms/librenms.git synced 2024-09-21 02:18:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve Snmpsim usage to ease testing (#15471)

Browse Source 
* Snmpsim use python venv
Patch to enable listening while minimizing output
Update lnms dev:simulate, tests, and ./scripts/save-test-data.php
removed old option to start snmpsim from older scripts, use lnms dev:simulate

* Apply fixes from StyleCI

* various fixes

* Remove patch official package is updated

---------

Co-authored-by: StyleCI Bot <bot@styleci.io>
This commit is contained in:
Tony Murray 2024-07-17 16:05:07 -05:00 committed by GitHub
parent 10669226fc
commit 1cceafb887
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 100 additions and 212 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.github/workflows/test.yml vendored
View File

@ -99,7 +99,7 @@ jobs:
name: Pip install
run: |
python3 -m pip install --upgrade pip
python3 -m pip install --upgrade --user snmpsim pylint python-memcached mysqlclient
python3 -m pip install --upgrade --user pylint python-memcached mysqlclient
-
name: Composer validate
run: |
@ -128,6 +128,10 @@ jobs:
name: Composer install
run: |
composer install --prefer-dist --no-interaction --no-progress
-
name: Snmpsim setup
run: |
php lnms dev:simulate --setup-venv
-
name: Artisan dusk:chrome-driver
if: matrix.skip-web-check != '1'
@ -168,7 +172,7 @@ jobs:
name: Start SNMP
if: matrix.skip-unit-check != '1'
run: |
~/.local/bin/snmpsim-command-responder-lite --data-dir=tests/snmpsim --agent-udpv4-endpoint=127.1.6.2:1162 --log-level=error --logging-method=file:/tmp/snmpsimd.log &
.python_venvs/snmpsim/bin/snmpsim-command-responder-lite --data-dir=tests/snmpsim --agent-udpv4-endpoint=127.1.6.2:1162 --log-level=error --logging-method=file:/tmp/snmpsimd.log &
-
name: lnms dev:check ci
run: |

2
LibreNMS/Util/CiHelper.php
View File

@ -295,7 +295,7 @@ class CiHelper
$py_lint_cmd = [$this->checkPythonExec('pylint'), '-E', '-j', '0'];
$files = $this->flags['full']
? explode(PHP_EOL, rtrim(shell_exec("find . -name '*.py' -not -path './vendor/*' -not -path './tests/*'")))
? explode(PHP_EOL, rtrim(shell_exec("find . -name '*.py' -not -path './vendor/*' -not -path './tests/*' -not -path './.python_venvs/*'")))
: $this->changed['python'];
$py_lint_cmd = array_merge($py_lint_cmd, $files);

8
LibreNMS/Util/ModuleTestHelper.php
View File

@ -560,17 +560,17 @@ class ModuleTestHelper
}
// Remove existing device in case it didn't get removed previously
if (($existing_device = device_by_name($snmpsim->getIp())) && isset($existing_device['device_id'])) {
if (($existing_device = device_by_name($snmpsim->ip)) && isset($existing_device['device_id'])) {
delete_device($existing_device['device_id']);
}
// Add the test device
try {
$new_device = new Device([
'hostname' => $snmpsim->getIp(),
'hostname' => $snmpsim->ip,
'version' => 'v2c',
'community' => $this->file_name,
'port' => $snmpsim->getPort(),
'port' => $snmpsim->port,
'disabled' => 1, // disable to block normal pollers
]);
(new ValidateDeviceAndCreate($new_device, true))->execute();
@ -647,7 +647,7 @@ class ModuleTestHelper
$data = array_merge_recursive($data, $this->dumpDb($device_id, $polled_modules, 'poller'));
// Remove the test device, we don't need the debug from this
if ($device['hostname'] == $snmpsim->getIp()) {
if ($device['hostname'] == $snmpsim->ip) {
Debug::set(false);
delete_device($device_id);
}

190
LibreNMS/Util/Snmpsim.php
View File

@ -25,166 +25,70 @@
namespace LibreNMS\Util;
use App;
use LibreNMS\Config;
use LibreNMS\Proc;
use Symfony\Component\Process\Process;
class Snmpsim
class Snmpsim extends Process
{
private $snmprec_dir;
private $ip;
private $port;
private $log;
/** @var Proc */
private $proc;
public readonly string $snmprec_dir;
public function __construct($ip = '127.1.6.1', $port = 1161, $log = '/tmp/snmpsimd.log')
public function __construct(
public readonly string $ip = '127.1.6.1',
public readonly int $port = 1161,
public readonly ?string $log_method = null)
{
$this->ip = $ip;
$this->port = $port;
$this->log = $log;
$this->snmprec_dir = Config::get('install_dir') . '/tests/snmpsim/';
$this->snmprec_dir = base_path('tests/snmpsim');
$cmd = [
$this->getVenvPath('bin/snmpsim-command-responder-lite'),
"--data-dir={$this->snmprec_dir}",
"--agent-udpv4-endpoint={$this->ip}:{$this->port}",
'--log-level=error',
];
if ($this->log_method !== null) {
$cmd[] = "--logging-method=$this->log_method";
}
parent::__construct($cmd, base_path());
$this->setTimeout(null); // no timeout by default
}
/**
* Run snmpsimd and fork it into the background
* Captures all output to the log
*
* @param int $wait Wait for x seconds after starting before returning
*/
public function fork($wait = 2)
public function waitForStartup(): string
{
if ($this->isRunning()) {
echo "Snmpsim is already running!\n";
$listen = $this->ip . ':' . $this->port;
$this->waitUntil(function ($type, $buffer) use ($listen, &$last) {
$last = $buffer;
return;
}
return $type == Process::ERR && str_contains($buffer, $listen);
});
$cmd = $this->getCmd();
return trim($last);
}
if (App::runningInConsole()) {
echo "Starting snmpsim listening on {$this->ip}:{$this->port}... \n";
d_echo($cmd);
}
public function isVenvSetUp(): bool
{
return is_executable($this->getVenvPath('bin/snmpsim-command-responder-lite'));
}
$this->proc = new Proc($cmd);
public function setupVenv($print_output = false): void
{
$snmpsim_venv_path = $this->getVenvPath();
if ($wait) {
sleep($wait);
}
if (! $this->isVenvSetUp()) {
\Log::info('Setting up snmpsim virtual env in ' . $snmpsim_venv_path);
if (App::runningInConsole() && ! $this->proc->isRunning()) {
// if starting failed, run snmpsim again and output to the console and validate the data
passthru($this->getCmd(false) . ' --validate-data');
$setupProcess = new Process(['python', '-m', 'venv', $snmpsim_venv_path]);
$setupProcess->setTty($print_output);
$setupProcess->run();
if (! is_executable($this->findSnmpsimd())) {
echo "\nCould not find snmpsim, you can install it with 'pip install snmpsim-lextudio'. If it is already installed, make sure snmpsimd, snmpsim-command-responder or snmpsimd.py is in PATH\n";
} else {
echo "\nFailed to start Snmpsim. Scroll up for error.\n";
}
exit(1);
$installProcess = new Process([$snmpsim_venv_path . '/bin/pip', 'install', 'snmpsim']);
$installProcess->setTty($print_output);
$installProcess->run();
}
}
/**
* Stop and start the running snmpsim process
*/
public function restart()
public function getVenvPath(string $subdir = ''): string
{
$this->stop();
$this->proc = new Proc($this->getCmd());
}
public function stop()
{
if (isset($this->proc)) {
if ($this->proc->isRunning()) {
$this->proc->terminate();
}
unset($this->proc);
}
}
/**
* Run snmpsimd but keep it in the foreground
* Outputs to stdout
*/
public function run()
{
echo "Starting snmpsim listening on {$this->ip}:{$this->port}... \n";
shell_exec($this->getCmd(false));
}
public function isRunning()
{
if (isset($this->proc)) {
return $this->proc->isRunning();
}
return false;
}
/**
* @return string
*/
public function getDir()
{
return $this->snmprec_dir;
}
/**
* @return string
*/
public function getIp()
{
return $this->ip;
}
/**
* @return int
*/
public function getPort()
{
return $this->port;
}
/**
* Generate the command for snmpsimd
*
* @param bool $with_log
* @return string
*/
private function getCmd($with_log = true)
{
$cmd = $this->findSnmpsimd();
$cmd .= " --data-dir={$this->snmprec_dir} --agent-udpv4-endpoint={$this->ip}:{$this->port}";
if (is_null($this->log)) {
$cmd .= ' --logging-method=null';
} elseif ($with_log) {
$cmd .= " --logging-method=file:{$this->log}";
}
return $cmd;
}
public function __destruct()
{
// unset $this->proc to make sure it isn't referenced
unset($this->proc);
}
public function findSnmpsimd()
{
$cmd = Config::locateBinary('snmpsimd');
if (! is_executable($cmd)) {
$cmd = Config::locateBinary('snmpsim-command-responder');
if (! is_executable($cmd)) {
$cmd = Config::locateBinary('snmpsimd.py');
}
}
return $cmd;
return base_path('.python_venvs/snmpsim/' . $subdir);
}
}

60
app/Console/Commands/DevSimulate.php
View File

@ -8,7 +8,7 @@ use Illuminate\Support\Str;
use LibreNMS\Util\Snmpsim;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Process\Process;
use Symfony\Component\Process\Exception\ProcessSignaledException;
class DevSimulate extends LnmsCommand
{
@ -38,6 +38,7 @@ class DevSimulate extends LnmsCommand
$this->addArgument('file', InputArgument::OPTIONAL);
$this->addOption('multiple', 'm', InputOption::VALUE_NONE);
$this->addOption('remove', 'r', InputOption::VALUE_NONE);
$this->addOption('setup-venv', mode: InputOption::VALUE_NONE);
}
/**
@ -45,12 +46,8 @@ class DevSimulate extends LnmsCommand
*
* @return int
*/
public function handle()
public function handle(): int
{
$this->snmpsim = new Snmpsim();
$snmprec_dir = $this->snmpsim->getDir();
$listen = $this->snmpsim->getIp() . ':' . $this->snmpsim->getPort();
$file = $this->argument('file');
if ($file && ! file_exists(base_path("tests/snmpsim/$file.snmprec"))) {
$this->error("$file does not exist");
@ -58,45 +55,52 @@ class DevSimulate extends LnmsCommand
return 1;
}
$snmpsim = new Process([
$this->snmpsim->findSnmpsimd(),
"--data-dir=$snmprec_dir",
"--agent-udpv4-endpoint=$listen",
]);
$snmpsim->setTimeout(null);
$this->snmpsim = new Snmpsim;
if (! $this->snmpsim->isVenvSetUp()) {
$this->line(trans('commands.dev:simulate.setup', ['dir' => $this->snmpsim->getVenvPath()]));
$this->snmpsim->setupVenv($this->getOutput()->isVeryVerbose());
}
$snmpsim->run(function ($type, $buffer) use ($listen) {
if (Process::ERR === $type) {
if (Str::contains($buffer, $listen)) {
$this->line(trim($buffer));
$this->started();
$this->line(trans('commands.dev:simulate.exit'));
}
}
});
if ($this->option('setup-venv')) {
return 0; // venv is set up exit
}
if (! $snmpsim->isSuccessful()) {
$this->line($snmpsim->getErrorOutput());
$this->snmpsim->start();
$this->line($this->snmpsim->waitForStartup());
$this->started();
$this->line(trans('commands.dev:simulate.exit'));
try {
$this->snmpsim->wait();
} catch(ProcessSignaledException $e) {
$this->error($e->getMessage());
return 1;
}
if (! $this->snmpsim->isSuccessful()) {
$this->line($this->snmpsim->getErrorOutput());
return 1;
}
return 0;
}
private function started()
private function started(): void
{
if ($file = $this->argument('file')) {
$this->addDevice($file);
}
}
private function addDevice($community)
private function addDevice($community): void
{
$hostname = $this->option('multiple') ? $community : 'snmpsim';
$device = Device::firstOrNew(['hostname' => $hostname]);
$action = $device->exists ? 'updated' : 'added';
$device->overwrite_ip = $this->snmpsim->getIp();
$device->port = $this->snmpsim->getPort();
$device->overwrite_ip = $this->snmpsim->ip;
$device->port = $this->snmpsim->port;
$device->snmpver = 'v2c';
$device->transport = 'udp';
$device->community = $community;
@ -112,7 +116,7 @@ class DevSimulate extends LnmsCommand
}
}
private function queueRemoval($device_id)
private function queueRemoval($device_id): void
{
if (function_exists('pcntl_signal')) {
pcntl_signal(SIGINT, function () {

1
lang/en/commands.php
View File

@ -63,6 +63,7 @@ return [
'exit' => 'Ctrl-C to stop',
'removed' => 'Device :id removed',
'updated' => 'Device :hostname (:id) updated',
'setup' => 'Setting up snmpsim venv in :dir',
],
'device:add' => [
'description' => 'Add a new device',

10
phpstan-baseline.neon
View File

@ -150,16 +150,6 @@ parameters:
count: 1
path: LibreNMS/Util/Graph.php
-
message: "#^Property LibreNMS\\\\Util\\\\Snmpsim\\:\\:\\$proc \\(LibreNMS\\\\Proc\\) in isset\\(\\) is not nullable\\.$#"
count: 2
path: LibreNMS/Util/Snmpsim.php
-
message: "#^Unreachable statement \\- code above always terminates\\.$#"
count: 1
path: LibreNMS/Util/Snmpsim.php
-
message: "#^Static method Silber\\\\Bouncer\\\\BouncerFacade\\:\\:role\\(\\) invoked with 0 parameters, 1 required\\.$#"
count: 1

8
scripts/collect-snmp-data.php
View File

@ -5,7 +5,6 @@ use Illuminate\Support\Str;
use LibreNMS\Exceptions\InvalidModuleException;
use LibreNMS\Util\Debug;
use LibreNMS\Util\ModuleTestHelper;
use LibreNMS\Util\Snmpsim;
$install_dir = realpath(__DIR__ . '/..');
chdir($install_dir);
@ -23,18 +22,11 @@ $options = getopt(
'variant:',
'file:',
'debug',
'snmpsim',
'full',
'help',
]
);
if (isset($options['snmpsim'])) {
$snmpsim = new Snmpsim();
$snmpsim->run();
exit;
}
if (isset($options['v'])) {
$variant = $options['v'];
} elseif (isset($options['variant'])) {

16
scripts/save-test-data.php
View File

@ -20,7 +20,6 @@ $options = getopt(
'no-save',
'file:',
'debug',
'snmpsim',
'help',
]
);
@ -32,12 +31,6 @@ Debug::setVerbose(
Debug::set(isset($options['d']) || isset($options['debug']))
);
if (isset($options['snmpsim'])) {
$snmpsim = new Snmpsim();
$snmpsim->run();
exit;
}
if (isset($options['h'])
|| isset($options['help'])
|| ! (isset($options['o']) || isset($options['os']) || isset($options['m']) || isset($options['modules']))
@ -120,9 +113,10 @@ if (isset($options['f'])) {
// Now use the saved data to update the saved database data
$snmpsim = new Snmpsim();
$snmpsim->fork();
$snmpsim_ip = $snmpsim->getIp();
$snmpsim_port = $snmpsim->getPort();
$snmpsim->setupVenv();
$snmpsim->start();
echo "Waiting for snmpsim to initialize...\n";
$snmpsim->waitForStartup();
if (! $snmpsim->isRunning()) {
echo "Failed to start snmpsim, make sure it is installed, working, and there are no bad snmprec files.\n";
@ -130,8 +124,6 @@ if (! $snmpsim->isRunning()) {
exit(1);
}
echo "Pausing 10 seconds to allow snmpsim to initialize...\n";
sleep(10);
echo "\n";
try {

4
tests/OSDiscoveryTest.php
View File

@ -143,9 +143,9 @@ class OSDiscoveryTest extends TestCase
private function genDevice($community): Device
{
return new Device([
'hostname' => $this->getSnmpsim()->getIP(),
'hostname' => $this->getSnmpsim()->ip,
'snmpver' => 'v2c',
'port' => $this->getSnmpsim()->getPort(),
'port' => $this->getSnmpsim()->port,
'timeout' => 3,
'retries' => 0,
'snmp_max_repeaters' => 10,

5
tests/bootstrap.php
View File

@ -36,10 +36,11 @@ chdir($install_dir);
ini_set('display_errors', '1');
//error_reporting(E_ALL & ~E_WARNING);
$snmpsim = new Snmpsim('127.1.6.2', 1162, null);
$snmpsim = new Snmpsim('127.1.6.2', 1162);
if (getenv('SNMPSIM')) {
if (! getenv('GITHUB_ACTIONS')) {
$snmpsim->fork(6);
$snmpsim->setupVenv();
$snmpsim->start();
}
// make PHP hold on a reference to $snmpsim so it doesn't get destructed