Import: validated encoding on DB error

MySQL gives useless error messages for invalid UTF8 characters and
confuses users with an 'Invalid datetime format' message.

Once storing imported data fails, the original data will now be scanned
for invalid UTF-8 characters. If such are found, a dedicated exception
with more details is thrown. Otherwise the original exception will be
forwarded

fixes #2143
This commit is contained in:
Thomas Gelf 2020-07-10 16:54:20 +02:00
parent b1245ac6d3
commit 36141c5f98
4 changed files with 113 additions and 1 deletions

View File

@ -27,6 +27,7 @@ next (will be 1.8.0)
* FEATURE: New Property Modifier: ListToObject (#2062)
* FEATURE: Property Modifier: convert binary UUID to HEX presentation (#2138)
* FEATURE: Import Sources now allow to download previewed data as JSON (#2096)
* FEATURE: UTF8 validation for failed imports gives better error message (#2143)
* FIX: LDAP Import is now able to paginate limited results (#2019)
### REST API

View File

@ -0,0 +1,59 @@
<?php
namespace Icinga\Module\Director\Data;
use InvalidArgumentException;
use ipl\Html\Error;
class RecursiveUtf8Validator
{
protected static $rowNum;
protected static $column;
/**
* @param array $rows Usually array of stdClass
* @return bool
*/
public static function validateRows($rows)
{
foreach ($rows as self::$rowNum => $row) {
foreach ($row as self::$column => $value) {
static::assertUtf8($value);
}
}
return true;
}
protected static function assertUtf8($value)
{
if (\is_string($value)) {
static::assertUtf8String($value);
} elseif (\is_array($value) || $value instanceof \stdClass) {
foreach ((array) $value as $k => $v) {
static::assertUtf8($k);
static::assertUtf8($v);
}
} elseif ($value !== null && !\is_scalar($value)) {
throw new InvalidArgumentException("Cannot validate " . Error::getPhpTypeName($value));
}
}
protected static function assertUtf8String($string)
{
if (@\iconv('UTF-8', 'UTF-8', $string) != $string) {
$row = self::$rowNum;
if (is_int($row)) {
$row++;
}
throw new InvalidArgumentException(\sprintf(
'Invalid UTF-8 on row %s, column %s: "%s" (%s)',
$row,
self::$column,
\iconv('UTF-8', 'UTF-8//IGNORE', $string),
'0x' . \bin2hex($string)
));
}
}
}

View File

@ -5,6 +5,7 @@ namespace Icinga\Module\Director\Import;
use Exception;
use Icinga\Application\Benchmark;
use Icinga\Exception\IcingaException;
use Icinga\Module\Director\Data\RecursiveUtf8Validator;
use Icinga\Module\Director\Db;
use Icinga\Module\Director\Hook\ImportSourceHook;
use Icinga\Module\Director\Objects\ImportSource;
@ -335,7 +336,13 @@ class Import
$this->rowsetExists = true;
} catch (Exception $e) {
$db->rollBack();
try {
$db->rollBack();
} catch (Exception $e) {
// Well...
}
// Eventually throws details for invalid UTF8 characters
RecursiveUtf8Validator::validateRows($this->data);
throw $e;
}
}

View File

@ -0,0 +1,45 @@
<?php
namespace Tests\Icinga\Module\Director\IcingaConfig;
use Icinga\Module\Director\Data\RecursiveUtf8Validator;
use Icinga\Module\Director\Test\BaseTestCase;
class RecursiveUtf8ValidatorTest extends BaseTestCase
{
/**
* @expectedException \InvalidArgumentException
*/
public function testDetectInvalidUtf8Character()
{
RecursiveUtf8Validator::validateRows([
(object) [
'name' => 'test 1',
'value' => 'something',
],
(object) [
'name' => 'test 2',
'value' => "some\xa1\xa2thing",
],
]);
}
public function testAcceptValidUtf8Characters()
{
$this->assertTrue(RecursiveUtf8Validator::validateRows([
(object) [
'name' => 'test 1',
'value' => "Some 🍻",
],
(object) [
'name' => 'test 2',
'value' => [
(object) [
'its' => true,
['💩']
]
],
],
]));
}
}