Import: validated encoding on DB error

MySQL gives useless error messages for invalid UTF8 characters and
confuses users with an 'Invalid datetime format' message.

Once storing imported data fails, the original data will now be scanned
for invalid UTF-8 characters. If such are found, a dedicated exception
with more details is thrown. Otherwise the original exception will be
forwarded

fixes #2143

doc/82-Changelog.md

@@ -27,6 +27,7 @@ next (will be 1.8.0)
 * FEATURE: New Property Modifier: ListToObject (#2062)
 * FEATURE: Property Modifier: convert binary UUID to HEX presentation (#2138)
 * FEATURE: Import Sources now allow to download previewed data as JSON (#2096)
+* FEATURE: UTF8 validation for failed imports gives better error message (#2143)
 * FIX: LDAP Import is now able to paginate limited results (#2019)
 
 ### REST API

library/Director/Data/RecursiveUtf8Validator.php

@@ -0,0 +1,59 @@
+<?php
+
+namespace Icinga\Module\Director\Data;
+
+use InvalidArgumentException;
+use ipl\Html\Error;
+
+class RecursiveUtf8Validator
+{
+    protected static $rowNum;
+
+    protected static $column;
+
+    /**
+     * @param array $rows Usually array of stdClass
+     * @return bool
+     */
+    public static function validateRows($rows)
+    {
+        foreach ($rows as self::$rowNum => $row) {
+            foreach ($row as self::$column => $value) {
+                static::assertUtf8($value);
+            }
+        }
+
+        return true;
+    }
+
+    protected static function assertUtf8($value)
+    {
+        if (\is_string($value)) {
+            static::assertUtf8String($value);
+        } elseif (\is_array($value) || $value instanceof \stdClass) {
+            foreach ((array) $value as $k => $v) {
+                static::assertUtf8($k);
+                static::assertUtf8($v);
+            }
+        } elseif ($value !== null && !\is_scalar($value)) {
+            throw new InvalidArgumentException("Cannot validate " . Error::getPhpTypeName($value));
+        }
+    }
+
+    protected static function assertUtf8String($string)
+    {
+        if (@\iconv('UTF-8', 'UTF-8', $string) != $string) {
+            $row = self::$rowNum;
+            if (is_int($row)) {
+                $row++;
+            }
+            throw new InvalidArgumentException(\sprintf(
+                'Invalid UTF-8 on row %s, column %s: "%s" (%s)',
+                $row,
+                self::$column,
+                \iconv('UTF-8', 'UTF-8//IGNORE', $string),
+                '0x' . \bin2hex($string)
+            ));
+        }
+    }
+}

library/Director/Import/Import.php

@@ -5,6 +5,7 @@ namespace Icinga\Module\Director\Import;
 use Exception;
 use Icinga\Application\Benchmark;
 use Icinga\Exception\IcingaException;
+use Icinga\Module\Director\Data\RecursiveUtf8Validator;
 use Icinga\Module\Director\Db;
 use Icinga\Module\Director\Hook\ImportSourceHook;
 use Icinga\Module\Director\Objects\ImportSource;
@@ -335,7 +336,13 @@ class Import
 
             $this->rowsetExists = true;
         } catch (Exception $e) {
+            try {
                 $db->rollBack();
+            } catch (Exception $e) {
+                // Well...
+            }
+            // Eventually throws details for invalid UTF8 characters
+            RecursiveUtf8Validator::validateRows($this->data);
             throw $e;
         }
     }

test/php/library/Director/Data/RecursiveUtf8ValidatorTest.php

@@ -0,0 +1,45 @@
+<?php
+
+namespace Tests\Icinga\Module\Director\IcingaConfig;
+
+use Icinga\Module\Director\Data\RecursiveUtf8Validator;
+use Icinga\Module\Director\Test\BaseTestCase;
+
+class RecursiveUtf8ValidatorTest extends BaseTestCase
+{
+    /**
+     * @expectedException \InvalidArgumentException
+     */
+    public function testDetectInvalidUtf8Character()
+    {
+        RecursiveUtf8Validator::validateRows([
+            (object) [
+                'name'  => 'test 1',
+                'value' => 'something',
+            ],
+            (object) [
+                'name'  => 'test 2',
+                'value' => "some\xa1\xa2thing",
+            ],
+        ]);
+    }
+
+    public function testAcceptValidUtf8Characters()
+    {
+        $this->assertTrue(RecursiveUtf8Validator::validateRows([
+            (object) [
+                'name'  => 'test 1',
+                'value' => "Some 🍻",
+            ],
+            (object) [
+                'name'  => 'test 2',
+                'value' => [
+                    (object) [
+                        'its' => true,
+                        ['💩']
+                    ]
+                ],
+            ],
+        ]));
+    }
+}