Install new python dependencies during daily maintenance (#13186)

* Improve check_requirements script to dynamically read requirements.txt

* Don't dynamically load requirements for python2

* Make sure we check python3 dependencies with python3 binary

* Quote variable to fix SC2086

* Add dynamic_check_requirements.py

* Use dynamic_check_requirements.py for python3

* Revert "Don't dynamically load requirements for python2"

This reverts commit 4485c8fcf9e0075fcf212e4f8bbc59ed75e144b5.

* Revert "Improve check_requirements script to dynamically read requirements.txt"

This reverts commit a9c83350d9de6b1b3b0a313dca23af749a08300a.

* Add LIBRENMS_DIR variable to exec, check for permission errors

* Make sure we check for permission errors on pip install

* Fix shellcheck SC2046 missing quotations

* Make sure we install pip packages as librenms user for Python3

* revert daily.sh, update dependencies in composer

* And in the validation

* should be executable

Co-authored-by: Tony Murray <murraytony@gmail.com>

LibreNMS/Validations/Python.php

@@ -74,7 +74,7 @@ class Python extends BaseValidation
 
     private function checkExtensions(Validator $validator)
     {
-        $pythonExtensions = '/scripts/check_requirements.py';
+        $pythonExtensions = '/scripts/dynamic_check_requirements.py';
         $process = new Process([Config::get('install_dir') . $pythonExtensions, '-v']);
         $process->run();
 

composer.json

@@ -137,7 +137,7 @@
             "@php artisan key:generate --ansi"
         ],
         "python-requirements": [
-            "scripts/check_requirements.py || pip3 install --user -r requirements.txt || :"
+            "scripts/dynamic_check_requirements.py || pip3 install --user -r requirements.txt || :"
         ]
     },
     "support": {

scripts/dynamic_check_requirements.py

@@ -0,0 +1,69 @@
+#! /usr/bin/env python3
+#  -*- coding: utf-8 -*-
+#
+# This file is part of LibreNMS
+
+__author__ = "Orsiris de Jong"
+__copyright__ = "Copyright (C) 2021 LibreNMS"
+
+import os
+import sys
+
+import pkg_resources
+
+args = sys.argv
+
+# verbose flag
+verbose = "-v" in args
+
+
+def _read_file(filename):
+    if sys.version_info[0] > 2:
+        with open(filename, "r", encoding="utf-8") as file_handle:
+            return file_handle.read()
+    else:
+        # With python 2.7, open has no encoding parameter, resulting in TypeError
+        # Fix with io.open (slow but works)
+        from io import open as io_open
+
+        with io_open(filename, "r", encoding="utf-8") as file_handle:
+            return file_handle.read()
+
+
+def parse_requirements(filename):
+    """
+    There is a parse_requirements function in pip but it keeps changing import path
+    Let's build a simple one
+    """
+    try:
+        requirements_txt = _read_file(filename)
+        install_requires = [
+            str(requirement)
+            for requirement in pkg_resources.parse_requirements(requirements_txt)
+        ]
+        return install_requires
+    except OSError:
+        print(
+            'WARNING: No requirements.txt file found as "{}". Please check path or create an empty one'.format(
+                filename
+            )
+        )
+        sys.exit(3)
+
+
+base_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
+requirements = parse_requirements(os.path.join(base_dir, "requirements.txt"))
+if verbose:
+    print("Required packages:", requirements)
+
+try:
+    pkg_resources.require(requirements)
+except pkg_resources.DistributionNotFound as req:
+    if verbose:
+        print("Package not found: {}".format(req))
+    sys.exit(1)
+except pkg_resources.VersionConflict as req:
+    if verbose:
+        print("Required version not satisfied: {}".format(req))
+    sys.exit(2)
+sys.exit(0)