Infer character encoding for ifAlias and sysLocation (#13248)

* Infer character encoding for ifAlias and sysLocation
Tries to convert character encoding for non-UTF-8 encoded strings.
This will only work for snmp strings that are type-hinted, not ones forced to ASCII with -Oa or similar
Only works for your default charset or Windows-1251 or LATIN1.
You can set your character encoding in .env with CHARSET

Please save us all and just use UTF-8

* style fixes

* less Yoda

* ensure return type

* fall back to passed string

* don't convert strings with any unprintable characters (such as line return)

* Update LibreNMS/Util/StringHelpers.php

Co-authored-by: Jellyfrog <Jellyfrog@users.noreply.github.com>

* Fix CP850

* fix space

Co-authored-by: Jellyfrog <Jellyfrog@users.noreply.github.com>

LibreNMS/OS/Traits/YamlOSDiscovery.php

@@ -28,6 +28,7 @@ namespace LibreNMS\OS\Traits;
 use App\Models\Device;
 use App\Models\Location;
 use Illuminate\Support\Arr;
+use LibreNMS\Util\StringHelpers;
 use Log;
 
 trait YamlOSDiscovery
@@ -93,8 +94,10 @@ trait YamlOSDiscovery
 
         Log::debug('Yaml location data:', $data);
 
+        $location = $this->findFirst($data, $name, $numeric) ?? snmp_get($this->getDeviceArray(), 'SNMPv2-MIB::sysLocation.0', '-Oqv');
+
         return new Location([
-            'location' => $this->findFirst($data, $name, $numeric) ?? snmp_get($this->getDeviceArray(), 'SNMPv2-MIB::sysLocation.0', '-Oqv'),
+            'location' => StringHelpers::inferEncoding($location),
             'lat' => $this->findFirst($data, $lat, $numeric),
             'lng' => $this->findFirst($data, $lng, $numeric),
         ]);

LibreNMS/Util/StringHelpers.php

@@ -93,4 +93,36 @@ class StringHelpers
     {
         return ucwords(implode(' ', preg_split('/(?=[A-Z])/', $string)));
     }
+
+    /**
+     * Sometimes devices store strings as non-unicode strings and return them directly.
+     * NetSnmp parses those as UTF-8, try to convert the string if it contains non-printable ascii characters.
+     *
+     * @param  string|null  $string
+     * @return string
+     */
+    public static function inferEncoding(?string $string): ?string
+    {
+        if (empty($string) || preg_match('//u', $string) || ! function_exists('iconv')) {
+            return $string;
+        }
+
+        $charset = config('app.charset');
+
+        if (($converted = @iconv($charset, 'UTF-8', $string)) !== false) {
+            return (string) $converted;
+        }
+
+        if ($charset !== 'Windows-1252' && ($converted = @iconv('Windows-1252', 'UTF-8', $string)) !== false) {
+            return (string) $converted;
+        }
+
+        if ($charset !== 'CP850' && ($converted = @iconv('CP850', 'UTF-8', $string)) !== false) {
+            return (string) $converted;
+        }
+
+        \Log::debug('Failed to convert string: ' . $string);
+
+        return $string;
+    }
 }

config/app.php

@@ -247,4 +247,5 @@ return [
         'Rrd' => App\Facades\Rrd::class,
     ],
 
+    'charset' => env('CHARSET', ini_get('php.output_encoding') ?: ini_get('default_charset') ?: 'UTF-8'),
 ];

includes/discovery/ports.inc.php

@@ -3,6 +3,7 @@
 // Build SNMP Cache Array
 use App\Models\PortGroup;
 use LibreNMS\Config;
+use LibreNMS\Util\StringHelpers;
 
 $port_stats = [];
 $port_stats = snmpwalk_cache_oid($device, 'ifDescr', $port_stats, 'IF-MIB');
@@ -60,6 +61,7 @@ $default_port_group = Config::get('default_port_group');
 // New interface detection
 foreach ($port_stats as $ifIndex => $snmp_data) {
     $snmp_data['ifIndex'] = $ifIndex; // Store ifIndex in port entry
+    $snmp_data['ifAlias'] = StringHelpers::inferEncoding($snmp_data['ifAlias']);
 
     // Get port_id according to port_association_mode used for this device
     $port_id = get_port_id($ports_mapped, $snmp_data, $port_association_mode);

includes/polling/ports.inc.php

@@ -669,6 +669,8 @@ foreach ($ports as $port) {
             if ($oid == 'ifAlias') {
                 if ($attribs['ifName:' . $port['ifName']]) {
                     $this_port['ifAlias'] = $port['ifAlias'];
+                } else {
+                    $this_port['ifAlias'] = \LibreNMS\Util\StringHelpers::inferEncoding($this_port['ifAlias']);
                 }
             }
             if ($oid == 'ifSpeed') {

tests/Unit/Util/StringHelperTest.php

@@ -0,0 +1,51 @@
+<?php
+/*
+ * StringHelperTest.php
+ *
+ * -Description-
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @package    LibreNMS
+ * @link       http://librenms.org
+ * @copyright  2021 Tony Murray
+ * @author     Tony Murray <murraytony@gmail.com>
+ */
+
+namespace LibreNMS\Tests\Unit\Util;
+
+use LibreNMS\Tests\TestCase;
+use LibreNMS\Util\StringHelpers;
+
+class StringHelperTest extends TestCase
+{
+    /**
+     * A basic feature test example.
+     *
+     * @return void
+     */
+    public function testInferEncoding()
+    {
+        $this->assertEquals(null, StringHelpers::inferEncoding(null));
+        $this->assertEquals('', StringHelpers::inferEncoding(''));
+        $this->assertEquals('~null', StringHelpers::inferEncoding('~null'));
+        $this->assertEquals('Øverbyvegen', StringHelpers::inferEncoding('Øverbyvegen'));
+
+        $this->assertEquals('Øverbyvegen', StringHelpers::inferEncoding(base64_decode('w5h2ZXJieXZlZ2Vu')));
+        $this->assertEquals('Øverbyvegen', StringHelpers::inferEncoding(base64_decode('2HZlcmJ5dmVnZW4=')));
+
+        config(['app.charset' => 'Shift_JIS']);
+        $this->assertEquals('コンサート', StringHelpers::inferEncoding(base64_decode('g1KDk4NUgVuDZw==')));
+    }
+}