mirror of https://github.com/Icinga/icinga2.git
parent
6bb44ed91a
commit
46d0d807dc
|
@ -0,0 +1,18 @@
|
||||||
|
#! /usr/bin/perl
|
||||||
|
|
||||||
|
$release_files = 'source/utf8.h source/utf8/core.h source/utf8/checked.h source/utf8/unchecked.h doc/utf8cpp.html doc/ReleaseNotes';
|
||||||
|
|
||||||
|
# First get the latest version
|
||||||
|
`svn update`;
|
||||||
|
|
||||||
|
# Then construct the name of the zip file
|
||||||
|
$argc = @ARGV;
|
||||||
|
if ($argc > 0) {
|
||||||
|
$zip_name = $ARGV[0];
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
$zip_name = "utf8";
|
||||||
|
}
|
||||||
|
|
||||||
|
# Zip the files to an archive
|
||||||
|
`zip $zip_name $release_files`;
|
|
@ -0,0 +1,5 @@
|
||||||
|
CC = g++
|
||||||
|
CFLAGS = -g -Wall -pedantic
|
||||||
|
|
||||||
|
docsample: docsample.cpp ../source/utf8.h
|
||||||
|
$(CC) $(CFLAGS) docsample.cpp -odocsample
|
|
@ -0,0 +1,52 @@
|
||||||
|
#include "../source/utf8.h"
|
||||||
|
#include <iostream>
|
||||||
|
#include <fstream>
|
||||||
|
#include <string>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
|
||||||
|
using namespace std;
|
||||||
|
|
||||||
|
int main(int argc, char** argv)
|
||||||
|
{
|
||||||
|
if (argc != 2) {
|
||||||
|
cout << "\nUsage: docsample filename\n";
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
const char* test_file_path = argv[1];
|
||||||
|
// Open the test file (must be UTF-8 encoded)
|
||||||
|
ifstream fs8(test_file_path);
|
||||||
|
if (!fs8.is_open()) {
|
||||||
|
cout << "Could not open " << test_file_path << endl;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
unsigned line_count = 1;
|
||||||
|
string line;
|
||||||
|
// Play with all the lines in the file
|
||||||
|
while (getline(fs8, line)) {
|
||||||
|
// check for invalid utf-8 (for a simple yes/no check, there is also utf8::is_valid function)
|
||||||
|
string::iterator end_it = utf8::find_invalid(line.begin(), line.end());
|
||||||
|
if (end_it != line.end()) {
|
||||||
|
cout << "Invalid UTF-8 encoding detected at line " << line_count << "\n";
|
||||||
|
cout << "This part is fine: " << string(line.begin(), end_it) << "\n";
|
||||||
|
}
|
||||||
|
// Get the line length (at least for the valid part)
|
||||||
|
int length = utf8::distance(line.begin(), end_it);
|
||||||
|
cout << "Length of line " << line_count << " is " << length << "\n";
|
||||||
|
|
||||||
|
// Convert it to utf-16
|
||||||
|
vector<unsigned short> utf16line;
|
||||||
|
utf8::utf8to16(line.begin(), end_it, back_inserter(utf16line));
|
||||||
|
// And back to utf-8;
|
||||||
|
string utf8line;
|
||||||
|
utf8::utf16to8(utf16line.begin(), utf16line.end(), back_inserter(utf8line));
|
||||||
|
// Confirm that the conversion went OK:
|
||||||
|
if (utf8line != string(line.begin(), end_it))
|
||||||
|
cout << "Error in UTF-16 conversion at line: " << line_count << "\n";
|
||||||
|
|
||||||
|
line_count++;
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
|
@ -0,0 +1,34 @@
|
||||||
|
// Copyright 2006 Nemanja Trifunovic
|
||||||
|
|
||||||
|
/*
|
||||||
|
Permission is hereby granted, free of charge, to any person or organization
|
||||||
|
obtaining a copy of the software and accompanying documentation covered by
|
||||||
|
this license (the "Software") to use, reproduce, display, distribute,
|
||||||
|
execute, and transmit the Software, and to prepare derivative works of the
|
||||||
|
Software, and to permit third-parties to whom the Software is furnished to
|
||||||
|
do so, all subject to the following:
|
||||||
|
|
||||||
|
The copyright notices in the Software and this entire statement, including
|
||||||
|
the above license grant, this restriction and the following disclaimer,
|
||||||
|
must be included in all copies of the Software, in whole or in part, and
|
||||||
|
all derivative works of the Software, unless such copies or derivative
|
||||||
|
works are solely in the form of machine-executable object code generated by
|
||||||
|
a source language processor.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
|
||||||
|
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
|
||||||
|
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
|
||||||
|
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||||
|
DEALINGS IN THE SOFTWARE.
|
||||||
|
*/
|
||||||
|
|
||||||
|
|
||||||
|
#ifndef UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731
|
||||||
|
#define UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731
|
||||||
|
|
||||||
|
#include "utf8/checked.h"
|
||||||
|
#include "utf8/unchecked.h"
|
||||||
|
|
||||||
|
#endif // header guard
|
|
@ -0,0 +1,327 @@
|
||||||
|
// Copyright 2006 Nemanja Trifunovic
|
||||||
|
|
||||||
|
/*
|
||||||
|
Permission is hereby granted, free of charge, to any person or organization
|
||||||
|
obtaining a copy of the software and accompanying documentation covered by
|
||||||
|
this license (the "Software") to use, reproduce, display, distribute,
|
||||||
|
execute, and transmit the Software, and to prepare derivative works of the
|
||||||
|
Software, and to permit third-parties to whom the Software is furnished to
|
||||||
|
do so, all subject to the following:
|
||||||
|
|
||||||
|
The copyright notices in the Software and this entire statement, including
|
||||||
|
the above license grant, this restriction and the following disclaimer,
|
||||||
|
must be included in all copies of the Software, in whole or in part, and
|
||||||
|
all derivative works of the Software, unless such copies or derivative
|
||||||
|
works are solely in the form of machine-executable object code generated by
|
||||||
|
a source language processor.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
|
||||||
|
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
|
||||||
|
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
|
||||||
|
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||||
|
DEALINGS IN THE SOFTWARE.
|
||||||
|
*/
|
||||||
|
|
||||||
|
|
||||||
|
#ifndef UTF8_FOR_CPP_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
|
||||||
|
#define UTF8_FOR_CPP_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
|
||||||
|
|
||||||
|
#include "core.h"
|
||||||
|
#include <stdexcept>
|
||||||
|
|
||||||
|
namespace utf8
|
||||||
|
{
|
||||||
|
// Base for the exceptions that may be thrown from the library
|
||||||
|
class exception : public ::std::exception {
|
||||||
|
};
|
||||||
|
|
||||||
|
// Exceptions that may be thrown from the library functions.
|
||||||
|
class invalid_code_point : public exception {
|
||||||
|
uint32_t cp;
|
||||||
|
public:
|
||||||
|
invalid_code_point(uint32_t cp) : cp(cp) {}
|
||||||
|
virtual const char* what() const throw() { return "Invalid code point"; }
|
||||||
|
uint32_t code_point() const {return cp;}
|
||||||
|
};
|
||||||
|
|
||||||
|
class invalid_utf8 : public exception {
|
||||||
|
uint8_t u8;
|
||||||
|
public:
|
||||||
|
invalid_utf8 (uint8_t u) : u8(u) {}
|
||||||
|
virtual const char* what() const throw() { return "Invalid UTF-8"; }
|
||||||
|
uint8_t utf8_octet() const {return u8;}
|
||||||
|
};
|
||||||
|
|
||||||
|
class invalid_utf16 : public exception {
|
||||||
|
uint16_t u16;
|
||||||
|
public:
|
||||||
|
invalid_utf16 (uint16_t u) : u16(u) {}
|
||||||
|
virtual const char* what() const throw() { return "Invalid UTF-16"; }
|
||||||
|
uint16_t utf16_word() const {return u16;}
|
||||||
|
};
|
||||||
|
|
||||||
|
class not_enough_room : public exception {
|
||||||
|
public:
|
||||||
|
virtual const char* what() const throw() { return "Not enough space"; }
|
||||||
|
};
|
||||||
|
|
||||||
|
/// The library API - functions intended to be called by the users
|
||||||
|
|
||||||
|
template <typename octet_iterator>
|
||||||
|
octet_iterator append(uint32_t cp, octet_iterator result)
|
||||||
|
{
|
||||||
|
if (!utf8::internal::is_code_point_valid(cp))
|
||||||
|
throw invalid_code_point(cp);
|
||||||
|
|
||||||
|
if (cp < 0x80) // one octet
|
||||||
|
*(result++) = static_cast<uint8_t>(cp);
|
||||||
|
else if (cp < 0x800) { // two octets
|
||||||
|
*(result++) = static_cast<uint8_t>((cp >> 6) | 0xc0);
|
||||||
|
*(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80);
|
||||||
|
}
|
||||||
|
else if (cp < 0x10000) { // three octets
|
||||||
|
*(result++) = static_cast<uint8_t>((cp >> 12) | 0xe0);
|
||||||
|
*(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f) | 0x80);
|
||||||
|
*(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80);
|
||||||
|
}
|
||||||
|
else { // four octets
|
||||||
|
*(result++) = static_cast<uint8_t>((cp >> 18) | 0xf0);
|
||||||
|
*(result++) = static_cast<uint8_t>(((cp >> 12) & 0x3f) | 0x80);
|
||||||
|
*(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f) | 0x80);
|
||||||
|
*(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename octet_iterator, typename output_iterator>
|
||||||
|
output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, uint32_t replacement)
|
||||||
|
{
|
||||||
|
while (start != end) {
|
||||||
|
octet_iterator sequence_start = start;
|
||||||
|
internal::utf_error err_code = utf8::internal::validate_next(start, end);
|
||||||
|
switch (err_code) {
|
||||||
|
case internal::UTF8_OK :
|
||||||
|
for (octet_iterator it = sequence_start; it != start; ++it)
|
||||||
|
*out++ = *it;
|
||||||
|
break;
|
||||||
|
case internal::NOT_ENOUGH_ROOM:
|
||||||
|
throw not_enough_room();
|
||||||
|
case internal::INVALID_LEAD:
|
||||||
|
out = utf8::append (replacement, out);
|
||||||
|
++start;
|
||||||
|
break;
|
||||||
|
case internal::INCOMPLETE_SEQUENCE:
|
||||||
|
case internal::OVERLONG_SEQUENCE:
|
||||||
|
case internal::INVALID_CODE_POINT:
|
||||||
|
out = utf8::append (replacement, out);
|
||||||
|
++start;
|
||||||
|
// just one replacement mark for the sequence
|
||||||
|
while (start != end && utf8::internal::is_trail(*start))
|
||||||
|
++start;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename octet_iterator, typename output_iterator>
|
||||||
|
inline output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out)
|
||||||
|
{
|
||||||
|
static const uint32_t replacement_marker = utf8::internal::mask16(0xfffd);
|
||||||
|
return utf8::replace_invalid(start, end, out, replacement_marker);
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename octet_iterator>
|
||||||
|
uint32_t next(octet_iterator& it, octet_iterator end)
|
||||||
|
{
|
||||||
|
uint32_t cp = 0;
|
||||||
|
internal::utf_error err_code = utf8::internal::validate_next(it, end, cp);
|
||||||
|
switch (err_code) {
|
||||||
|
case internal::UTF8_OK :
|
||||||
|
break;
|
||||||
|
case internal::NOT_ENOUGH_ROOM :
|
||||||
|
throw not_enough_room();
|
||||||
|
case internal::INVALID_LEAD :
|
||||||
|
case internal::INCOMPLETE_SEQUENCE :
|
||||||
|
case internal::OVERLONG_SEQUENCE :
|
||||||
|
throw invalid_utf8(*it);
|
||||||
|
case internal::INVALID_CODE_POINT :
|
||||||
|
throw invalid_code_point(cp);
|
||||||
|
}
|
||||||
|
return cp;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename octet_iterator>
|
||||||
|
uint32_t peek_next(octet_iterator it, octet_iterator end)
|
||||||
|
{
|
||||||
|
return utf8::next(it, end);
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename octet_iterator>
|
||||||
|
uint32_t prior(octet_iterator& it, octet_iterator start)
|
||||||
|
{
|
||||||
|
// can't do much if it == start
|
||||||
|
if (it == start)
|
||||||
|
throw not_enough_room();
|
||||||
|
|
||||||
|
octet_iterator end = it;
|
||||||
|
// Go back until we hit either a lead octet or start
|
||||||
|
while (utf8::internal::is_trail(*(--it)))
|
||||||
|
if (it == start)
|
||||||
|
throw invalid_utf8(*it); // error - no lead byte in the sequence
|
||||||
|
return utf8::peek_next(it, end);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Deprecated in versions that include "prior"
|
||||||
|
template <typename octet_iterator>
|
||||||
|
uint32_t previous(octet_iterator& it, octet_iterator pass_start)
|
||||||
|
{
|
||||||
|
octet_iterator end = it;
|
||||||
|
while (utf8::internal::is_trail(*(--it)))
|
||||||
|
if (it == pass_start)
|
||||||
|
throw invalid_utf8(*it); // error - no lead byte in the sequence
|
||||||
|
octet_iterator temp = it;
|
||||||
|
return utf8::next(temp, end);
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename octet_iterator, typename distance_type>
|
||||||
|
void advance (octet_iterator& it, distance_type n, octet_iterator end)
|
||||||
|
{
|
||||||
|
for (distance_type i = 0; i < n; ++i)
|
||||||
|
utf8::next(it, end);
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename octet_iterator>
|
||||||
|
typename std::iterator_traits<octet_iterator>::difference_type
|
||||||
|
distance (octet_iterator first, octet_iterator last)
|
||||||
|
{
|
||||||
|
typename std::iterator_traits<octet_iterator>::difference_type dist;
|
||||||
|
for (dist = 0; first < last; ++dist)
|
||||||
|
utf8::next(first, last);
|
||||||
|
return dist;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename u16bit_iterator, typename octet_iterator>
|
||||||
|
octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result)
|
||||||
|
{
|
||||||
|
while (start != end) {
|
||||||
|
uint32_t cp = utf8::internal::mask16(*start++);
|
||||||
|
// Take care of surrogate pairs first
|
||||||
|
if (utf8::internal::is_lead_surrogate(cp)) {
|
||||||
|
if (start != end) {
|
||||||
|
uint32_t trail_surrogate = utf8::internal::mask16(*start++);
|
||||||
|
if (utf8::internal::is_trail_surrogate(trail_surrogate))
|
||||||
|
cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET;
|
||||||
|
else
|
||||||
|
throw invalid_utf16(static_cast<uint16_t>(trail_surrogate));
|
||||||
|
}
|
||||||
|
else
|
||||||
|
throw invalid_utf16(static_cast<uint16_t>(cp));
|
||||||
|
|
||||||
|
}
|
||||||
|
// Lone trail surrogate
|
||||||
|
else if (utf8::internal::is_trail_surrogate(cp))
|
||||||
|
throw invalid_utf16(static_cast<uint16_t>(cp));
|
||||||
|
|
||||||
|
result = utf8::append(cp, result);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename u16bit_iterator, typename octet_iterator>
|
||||||
|
u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result)
|
||||||
|
{
|
||||||
|
while (start < end) {
|
||||||
|
uint32_t cp = utf8::next(start, end);
|
||||||
|
if (cp > 0xffff) { //make a surrogate pair
|
||||||
|
*result++ = static_cast<uint16_t>((cp >> 10) + internal::LEAD_OFFSET);
|
||||||
|
*result++ = static_cast<uint16_t>((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
*result++ = static_cast<uint16_t>(cp);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename octet_iterator, typename u32bit_iterator>
|
||||||
|
octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result)
|
||||||
|
{
|
||||||
|
while (start != end)
|
||||||
|
result = utf8::append(*(start++), result);
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename octet_iterator, typename u32bit_iterator>
|
||||||
|
u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result)
|
||||||
|
{
|
||||||
|
while (start < end)
|
||||||
|
(*result++) = utf8::next(start, end);
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
// The iterator class
|
||||||
|
template <typename octet_iterator>
|
||||||
|
class iterator : public std::iterator <std::bidirectional_iterator_tag, uint32_t> {
|
||||||
|
octet_iterator it;
|
||||||
|
octet_iterator range_start;
|
||||||
|
octet_iterator range_end;
|
||||||
|
public:
|
||||||
|
iterator () {}
|
||||||
|
explicit iterator (const octet_iterator& octet_it,
|
||||||
|
const octet_iterator& range_start,
|
||||||
|
const octet_iterator& range_end) :
|
||||||
|
it(octet_it), range_start(range_start), range_end(range_end)
|
||||||
|
{
|
||||||
|
if (it < range_start || it > range_end)
|
||||||
|
throw std::out_of_range("Invalid utf-8 iterator position");
|
||||||
|
}
|
||||||
|
// the default "big three" are OK
|
||||||
|
octet_iterator base () const { return it; }
|
||||||
|
uint32_t operator * () const
|
||||||
|
{
|
||||||
|
octet_iterator temp = it;
|
||||||
|
return utf8::next(temp, range_end);
|
||||||
|
}
|
||||||
|
bool operator == (const iterator& rhs) const
|
||||||
|
{
|
||||||
|
if (range_start != rhs.range_start || range_end != rhs.range_end)
|
||||||
|
throw std::logic_error("Comparing utf-8 iterators defined with different ranges");
|
||||||
|
return (it == rhs.it);
|
||||||
|
}
|
||||||
|
bool operator != (const iterator& rhs) const
|
||||||
|
{
|
||||||
|
return !(operator == (rhs));
|
||||||
|
}
|
||||||
|
iterator& operator ++ ()
|
||||||
|
{
|
||||||
|
utf8::next(it, range_end);
|
||||||
|
return *this;
|
||||||
|
}
|
||||||
|
iterator operator ++ (int)
|
||||||
|
{
|
||||||
|
iterator temp = *this;
|
||||||
|
utf8::next(it, range_end);
|
||||||
|
return temp;
|
||||||
|
}
|
||||||
|
iterator& operator -- ()
|
||||||
|
{
|
||||||
|
utf8::prior(it, range_start);
|
||||||
|
return *this;
|
||||||
|
}
|
||||||
|
iterator operator -- (int)
|
||||||
|
{
|
||||||
|
iterator temp = *this;
|
||||||
|
utf8::prior(it, range_start);
|
||||||
|
return temp;
|
||||||
|
}
|
||||||
|
}; // class iterator
|
||||||
|
|
||||||
|
} // namespace utf8
|
||||||
|
|
||||||
|
#endif //header guard
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,329 @@
|
||||||
|
// Copyright 2006 Nemanja Trifunovic
|
||||||
|
|
||||||
|
/*
|
||||||
|
Permission is hereby granted, free of charge, to any person or organization
|
||||||
|
obtaining a copy of the software and accompanying documentation covered by
|
||||||
|
this license (the "Software") to use, reproduce, display, distribute,
|
||||||
|
execute, and transmit the Software, and to prepare derivative works of the
|
||||||
|
Software, and to permit third-parties to whom the Software is furnished to
|
||||||
|
do so, all subject to the following:
|
||||||
|
|
||||||
|
The copyright notices in the Software and this entire statement, including
|
||||||
|
the above license grant, this restriction and the following disclaimer,
|
||||||
|
must be included in all copies of the Software, in whole or in part, and
|
||||||
|
all derivative works of the Software, unless such copies or derivative
|
||||||
|
works are solely in the form of machine-executable object code generated by
|
||||||
|
a source language processor.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
|
||||||
|
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
|
||||||
|
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
|
||||||
|
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||||
|
DEALINGS IN THE SOFTWARE.
|
||||||
|
*/
|
||||||
|
|
||||||
|
|
||||||
|
#ifndef UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
|
||||||
|
#define UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
|
||||||
|
|
||||||
|
#include <iterator>
|
||||||
|
|
||||||
|
namespace utf8
|
||||||
|
{
|
||||||
|
// The typedefs for 8-bit, 16-bit and 32-bit unsigned integers
|
||||||
|
// You may need to change them to match your system.
|
||||||
|
// These typedefs have the same names as ones from cstdint, or boost/cstdint
|
||||||
|
typedef unsigned char uint8_t;
|
||||||
|
typedef unsigned short uint16_t;
|
||||||
|
typedef unsigned int uint32_t;
|
||||||
|
|
||||||
|
// Helper code - not intended to be directly called by the library users. May be changed at any time
|
||||||
|
namespace internal
|
||||||
|
{
|
||||||
|
// Unicode constants
|
||||||
|
// Leading (high) surrogates: 0xd800 - 0xdbff
|
||||||
|
// Trailing (low) surrogates: 0xdc00 - 0xdfff
|
||||||
|
const uint16_t LEAD_SURROGATE_MIN = 0xd800u;
|
||||||
|
const uint16_t LEAD_SURROGATE_MAX = 0xdbffu;
|
||||||
|
const uint16_t TRAIL_SURROGATE_MIN = 0xdc00u;
|
||||||
|
const uint16_t TRAIL_SURROGATE_MAX = 0xdfffu;
|
||||||
|
const uint16_t LEAD_OFFSET = LEAD_SURROGATE_MIN - (0x10000 >> 10);
|
||||||
|
const uint32_t SURROGATE_OFFSET = 0x10000u - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN;
|
||||||
|
|
||||||
|
// Maximum valid value for a Unicode code point
|
||||||
|
const uint32_t CODE_POINT_MAX = 0x0010ffffu;
|
||||||
|
|
||||||
|
template<typename octet_type>
|
||||||
|
inline uint8_t mask8(octet_type oc)
|
||||||
|
{
|
||||||
|
return static_cast<uint8_t>(0xff & oc);
|
||||||
|
}
|
||||||
|
template<typename u16_type>
|
||||||
|
inline uint16_t mask16(u16_type oc)
|
||||||
|
{
|
||||||
|
return static_cast<uint16_t>(0xffff & oc);
|
||||||
|
}
|
||||||
|
template<typename octet_type>
|
||||||
|
inline bool is_trail(octet_type oc)
|
||||||
|
{
|
||||||
|
return ((utf8::internal::mask8(oc) >> 6) == 0x2);
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename u16>
|
||||||
|
inline bool is_lead_surrogate(u16 cp)
|
||||||
|
{
|
||||||
|
return (cp >= LEAD_SURROGATE_MIN && cp <= LEAD_SURROGATE_MAX);
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename u16>
|
||||||
|
inline bool is_trail_surrogate(u16 cp)
|
||||||
|
{
|
||||||
|
return (cp >= TRAIL_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX);
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename u16>
|
||||||
|
inline bool is_surrogate(u16 cp)
|
||||||
|
{
|
||||||
|
return (cp >= LEAD_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX);
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename u32>
|
||||||
|
inline bool is_code_point_valid(u32 cp)
|
||||||
|
{
|
||||||
|
return (cp <= CODE_POINT_MAX && !utf8::internal::is_surrogate(cp));
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename octet_iterator>
|
||||||
|
inline typename std::iterator_traits<octet_iterator>::difference_type
|
||||||
|
sequence_length(octet_iterator lead_it)
|
||||||
|
{
|
||||||
|
uint8_t lead = utf8::internal::mask8(*lead_it);
|
||||||
|
if (lead < 0x80)
|
||||||
|
return 1;
|
||||||
|
else if ((lead >> 5) == 0x6)
|
||||||
|
return 2;
|
||||||
|
else if ((lead >> 4) == 0xe)
|
||||||
|
return 3;
|
||||||
|
else if ((lead >> 3) == 0x1e)
|
||||||
|
return 4;
|
||||||
|
else
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename octet_difference_type>
|
||||||
|
inline bool is_overlong_sequence(uint32_t cp, octet_difference_type length)
|
||||||
|
{
|
||||||
|
if (cp < 0x80) {
|
||||||
|
if (length != 1)
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
else if (cp < 0x800) {
|
||||||
|
if (length != 2)
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
else if (cp < 0x10000) {
|
||||||
|
if (length != 3)
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
enum utf_error {UTF8_OK, NOT_ENOUGH_ROOM, INVALID_LEAD, INCOMPLETE_SEQUENCE, OVERLONG_SEQUENCE, INVALID_CODE_POINT};
|
||||||
|
|
||||||
|
/// Helper for get_sequence_x
|
||||||
|
template <typename octet_iterator>
|
||||||
|
utf_error increase_safely(octet_iterator& it, octet_iterator end)
|
||||||
|
{
|
||||||
|
if (++it == end)
|
||||||
|
return NOT_ENOUGH_ROOM;
|
||||||
|
|
||||||
|
if (!utf8::internal::is_trail(*it))
|
||||||
|
return INCOMPLETE_SEQUENCE;
|
||||||
|
|
||||||
|
return UTF8_OK;
|
||||||
|
}
|
||||||
|
|
||||||
|
#define UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(IT, END) {utf_error ret = increase_safely(IT, END); if (ret != UTF8_OK) return ret;}
|
||||||
|
|
||||||
|
/// get_sequence_x functions decode utf-8 sequences of the length x
|
||||||
|
template <typename octet_iterator>
|
||||||
|
utf_error get_sequence_1(octet_iterator& it, octet_iterator end, uint32_t& code_point)
|
||||||
|
{
|
||||||
|
if (it == end)
|
||||||
|
return NOT_ENOUGH_ROOM;
|
||||||
|
|
||||||
|
code_point = utf8::internal::mask8(*it);
|
||||||
|
|
||||||
|
return UTF8_OK;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename octet_iterator>
|
||||||
|
utf_error get_sequence_2(octet_iterator& it, octet_iterator end, uint32_t& code_point)
|
||||||
|
{
|
||||||
|
if (it == end)
|
||||||
|
return NOT_ENOUGH_ROOM;
|
||||||
|
|
||||||
|
code_point = utf8::internal::mask8(*it);
|
||||||
|
|
||||||
|
UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
|
||||||
|
|
||||||
|
code_point = ((code_point << 6) & 0x7ff) + ((*it) & 0x3f);
|
||||||
|
|
||||||
|
return UTF8_OK;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename octet_iterator>
|
||||||
|
utf_error get_sequence_3(octet_iterator& it, octet_iterator end, uint32_t& code_point)
|
||||||
|
{
|
||||||
|
if (it == end)
|
||||||
|
return NOT_ENOUGH_ROOM;
|
||||||
|
|
||||||
|
code_point = utf8::internal::mask8(*it);
|
||||||
|
|
||||||
|
UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
|
||||||
|
|
||||||
|
code_point = ((code_point << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff);
|
||||||
|
|
||||||
|
UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
|
||||||
|
|
||||||
|
code_point += (*it) & 0x3f;
|
||||||
|
|
||||||
|
return UTF8_OK;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename octet_iterator>
|
||||||
|
utf_error get_sequence_4(octet_iterator& it, octet_iterator end, uint32_t& code_point)
|
||||||
|
{
|
||||||
|
if (it == end)
|
||||||
|
return NOT_ENOUGH_ROOM;
|
||||||
|
|
||||||
|
code_point = utf8::internal::mask8(*it);
|
||||||
|
|
||||||
|
UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
|
||||||
|
|
||||||
|
code_point = ((code_point << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff);
|
||||||
|
|
||||||
|
UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
|
||||||
|
|
||||||
|
code_point += (utf8::internal::mask8(*it) << 6) & 0xfff;
|
||||||
|
|
||||||
|
UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
|
||||||
|
|
||||||
|
code_point += (*it) & 0x3f;
|
||||||
|
|
||||||
|
return UTF8_OK;
|
||||||
|
}
|
||||||
|
|
||||||
|
#undef UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR
|
||||||
|
|
||||||
|
template <typename octet_iterator>
|
||||||
|
utf_error validate_next(octet_iterator& it, octet_iterator end, uint32_t& code_point)
|
||||||
|
{
|
||||||
|
// Save the original value of it so we can go back in case of failure
|
||||||
|
// Of course, it does not make much sense with i.e. stream iterators
|
||||||
|
octet_iterator original_it = it;
|
||||||
|
|
||||||
|
uint32_t cp = 0;
|
||||||
|
// Determine the sequence length based on the lead octet
|
||||||
|
typedef typename std::iterator_traits<octet_iterator>::difference_type octet_difference_type;
|
||||||
|
const octet_difference_type length = utf8::internal::sequence_length(it);
|
||||||
|
|
||||||
|
// Get trail octets and calculate the code point
|
||||||
|
utf_error err = UTF8_OK;
|
||||||
|
switch (length) {
|
||||||
|
case 0:
|
||||||
|
return INVALID_LEAD;
|
||||||
|
case 1:
|
||||||
|
err = utf8::internal::get_sequence_1(it, end, cp);
|
||||||
|
break;
|
||||||
|
case 2:
|
||||||
|
err = utf8::internal::get_sequence_2(it, end, cp);
|
||||||
|
break;
|
||||||
|
case 3:
|
||||||
|
err = utf8::internal::get_sequence_3(it, end, cp);
|
||||||
|
break;
|
||||||
|
case 4:
|
||||||
|
err = utf8::internal::get_sequence_4(it, end, cp);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (err == UTF8_OK) {
|
||||||
|
// Decoding succeeded. Now, security checks...
|
||||||
|
if (utf8::internal::is_code_point_valid(cp)) {
|
||||||
|
if (!utf8::internal::is_overlong_sequence(cp, length)){
|
||||||
|
// Passed! Return here.
|
||||||
|
code_point = cp;
|
||||||
|
++it;
|
||||||
|
return UTF8_OK;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
err = OVERLONG_SEQUENCE;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
err = INVALID_CODE_POINT;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Failure branch - restore the original value of the iterator
|
||||||
|
it = original_it;
|
||||||
|
return err;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename octet_iterator>
|
||||||
|
inline utf_error validate_next(octet_iterator& it, octet_iterator end) {
|
||||||
|
uint32_t ignored;
|
||||||
|
return utf8::internal::validate_next(it, end, ignored);
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace internal
|
||||||
|
|
||||||
|
/// The library API - functions intended to be called by the users
|
||||||
|
|
||||||
|
// Byte order mark
|
||||||
|
const uint8_t bom[] = {0xef, 0xbb, 0xbf};
|
||||||
|
|
||||||
|
template <typename octet_iterator>
|
||||||
|
octet_iterator find_invalid(octet_iterator start, octet_iterator end)
|
||||||
|
{
|
||||||
|
octet_iterator result = start;
|
||||||
|
while (result != end) {
|
||||||
|
utf8::internal::utf_error err_code = utf8::internal::validate_next(result, end);
|
||||||
|
if (err_code != internal::UTF8_OK)
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename octet_iterator>
|
||||||
|
inline bool is_valid(octet_iterator start, octet_iterator end)
|
||||||
|
{
|
||||||
|
return (utf8::find_invalid(start, end) == end);
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename octet_iterator>
|
||||||
|
inline bool starts_with_bom (octet_iterator it, octet_iterator end)
|
||||||
|
{
|
||||||
|
return (
|
||||||
|
((it != end) && (utf8::internal::mask8(*it++)) == bom[0]) &&
|
||||||
|
((it != end) && (utf8::internal::mask8(*it++)) == bom[1]) &&
|
||||||
|
((it != end) && (utf8::internal::mask8(*it)) == bom[2])
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
//Deprecated in release 2.3
|
||||||
|
template <typename octet_iterator>
|
||||||
|
inline bool is_bom (octet_iterator it)
|
||||||
|
{
|
||||||
|
return (
|
||||||
|
(utf8::internal::mask8(*it++)) == bom[0] &&
|
||||||
|
(utf8::internal::mask8(*it++)) == bom[1] &&
|
||||||
|
(utf8::internal::mask8(*it)) == bom[2]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} // namespace utf8
|
||||||
|
|
||||||
|
#endif // header guard
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,228 @@
|
||||||
|
// Copyright 2006 Nemanja Trifunovic
|
||||||
|
|
||||||
|
/*
|
||||||
|
Permission is hereby granted, free of charge, to any person or organization
|
||||||
|
obtaining a copy of the software and accompanying documentation covered by
|
||||||
|
this license (the "Software") to use, reproduce, display, distribute,
|
||||||
|
execute, and transmit the Software, and to prepare derivative works of the
|
||||||
|
Software, and to permit third-parties to whom the Software is furnished to
|
||||||
|
do so, all subject to the following:
|
||||||
|
|
||||||
|
The copyright notices in the Software and this entire statement, including
|
||||||
|
the above license grant, this restriction and the following disclaimer,
|
||||||
|
must be included in all copies of the Software, in whole or in part, and
|
||||||
|
all derivative works of the Software, unless such copies or derivative
|
||||||
|
works are solely in the form of machine-executable object code generated by
|
||||||
|
a source language processor.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
|
||||||
|
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
|
||||||
|
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
|
||||||
|
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||||
|
DEALINGS IN THE SOFTWARE.
|
||||||
|
*/
|
||||||
|
|
||||||
|
|
||||||
|
#ifndef UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
|
||||||
|
#define UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
|
||||||
|
|
||||||
|
#include "core.h"
|
||||||
|
|
||||||
|
namespace utf8
|
||||||
|
{
|
||||||
|
namespace unchecked
|
||||||
|
{
|
||||||
|
template <typename octet_iterator>
|
||||||
|
octet_iterator append(uint32_t cp, octet_iterator result)
|
||||||
|
{
|
||||||
|
if (cp < 0x80) // one octet
|
||||||
|
*(result++) = static_cast<uint8_t>(cp);
|
||||||
|
else if (cp < 0x800) { // two octets
|
||||||
|
*(result++) = static_cast<uint8_t>((cp >> 6) | 0xc0);
|
||||||
|
*(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80);
|
||||||
|
}
|
||||||
|
else if (cp < 0x10000) { // three octets
|
||||||
|
*(result++) = static_cast<uint8_t>((cp >> 12) | 0xe0);
|
||||||
|
*(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f) | 0x80);
|
||||||
|
*(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80);
|
||||||
|
}
|
||||||
|
else { // four octets
|
||||||
|
*(result++) = static_cast<uint8_t>((cp >> 18) | 0xf0);
|
||||||
|
*(result++) = static_cast<uint8_t>(((cp >> 12) & 0x3f)| 0x80);
|
||||||
|
*(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f) | 0x80);
|
||||||
|
*(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename octet_iterator>
|
||||||
|
uint32_t next(octet_iterator& it)
|
||||||
|
{
|
||||||
|
uint32_t cp = utf8::internal::mask8(*it);
|
||||||
|
typename std::iterator_traits<octet_iterator>::difference_type length = utf8::internal::sequence_length(it);
|
||||||
|
switch (length) {
|
||||||
|
case 1:
|
||||||
|
break;
|
||||||
|
case 2:
|
||||||
|
it++;
|
||||||
|
cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f);
|
||||||
|
break;
|
||||||
|
case 3:
|
||||||
|
++it;
|
||||||
|
cp = ((cp << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff);
|
||||||
|
++it;
|
||||||
|
cp += (*it) & 0x3f;
|
||||||
|
break;
|
||||||
|
case 4:
|
||||||
|
++it;
|
||||||
|
cp = ((cp << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff);
|
||||||
|
++it;
|
||||||
|
cp += (utf8::internal::mask8(*it) << 6) & 0xfff;
|
||||||
|
++it;
|
||||||
|
cp += (*it) & 0x3f;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
++it;
|
||||||
|
return cp;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename octet_iterator>
|
||||||
|
uint32_t peek_next(octet_iterator it)
|
||||||
|
{
|
||||||
|
return utf8::unchecked::next(it);
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename octet_iterator>
|
||||||
|
uint32_t prior(octet_iterator& it)
|
||||||
|
{
|
||||||
|
while (utf8::internal::is_trail(*(--it))) ;
|
||||||
|
octet_iterator temp = it;
|
||||||
|
return utf8::unchecked::next(temp);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Deprecated in versions that include prior, but only for the sake of consistency (see utf8::previous)
|
||||||
|
template <typename octet_iterator>
|
||||||
|
inline uint32_t previous(octet_iterator& it)
|
||||||
|
{
|
||||||
|
return utf8::unchecked::prior(it);
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename octet_iterator, typename distance_type>
|
||||||
|
void advance (octet_iterator& it, distance_type n)
|
||||||
|
{
|
||||||
|
for (distance_type i = 0; i < n; ++i)
|
||||||
|
utf8::unchecked::next(it);
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename octet_iterator>
|
||||||
|
typename std::iterator_traits<octet_iterator>::difference_type
|
||||||
|
distance (octet_iterator first, octet_iterator last)
|
||||||
|
{
|
||||||
|
typename std::iterator_traits<octet_iterator>::difference_type dist;
|
||||||
|
for (dist = 0; first < last; ++dist)
|
||||||
|
utf8::unchecked::next(first);
|
||||||
|
return dist;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename u16bit_iterator, typename octet_iterator>
|
||||||
|
octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result)
|
||||||
|
{
|
||||||
|
while (start != end) {
|
||||||
|
uint32_t cp = utf8::internal::mask16(*start++);
|
||||||
|
// Take care of surrogate pairs first
|
||||||
|
if (utf8::internal::is_lead_surrogate(cp)) {
|
||||||
|
uint32_t trail_surrogate = utf8::internal::mask16(*start++);
|
||||||
|
cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET;
|
||||||
|
}
|
||||||
|
result = utf8::unchecked::append(cp, result);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename u16bit_iterator, typename octet_iterator>
|
||||||
|
u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result)
|
||||||
|
{
|
||||||
|
while (start < end) {
|
||||||
|
uint32_t cp = utf8::unchecked::next(start);
|
||||||
|
if (cp > 0xffff) { //make a surrogate pair
|
||||||
|
*result++ = static_cast<uint16_t>((cp >> 10) + internal::LEAD_OFFSET);
|
||||||
|
*result++ = static_cast<uint16_t>((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
*result++ = static_cast<uint16_t>(cp);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename octet_iterator, typename u32bit_iterator>
|
||||||
|
octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result)
|
||||||
|
{
|
||||||
|
while (start != end)
|
||||||
|
result = utf8::unchecked::append(*(start++), result);
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename octet_iterator, typename u32bit_iterator>
|
||||||
|
u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result)
|
||||||
|
{
|
||||||
|
while (start < end)
|
||||||
|
(*result++) = utf8::unchecked::next(start);
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
// The iterator class
|
||||||
|
template <typename octet_iterator>
|
||||||
|
class iterator : public std::iterator <std::bidirectional_iterator_tag, uint32_t> {
|
||||||
|
octet_iterator it;
|
||||||
|
public:
|
||||||
|
iterator () {}
|
||||||
|
explicit iterator (const octet_iterator& octet_it): it(octet_it) {}
|
||||||
|
// the default "big three" are OK
|
||||||
|
octet_iterator base () const { return it; }
|
||||||
|
uint32_t operator * () const
|
||||||
|
{
|
||||||
|
octet_iterator temp = it;
|
||||||
|
return utf8::unchecked::next(temp);
|
||||||
|
}
|
||||||
|
bool operator == (const iterator& rhs) const
|
||||||
|
{
|
||||||
|
return (it == rhs.it);
|
||||||
|
}
|
||||||
|
bool operator != (const iterator& rhs) const
|
||||||
|
{
|
||||||
|
return !(operator == (rhs));
|
||||||
|
}
|
||||||
|
iterator& operator ++ ()
|
||||||
|
{
|
||||||
|
::std::advance(it, utf8::internal::sequence_length(it));
|
||||||
|
return *this;
|
||||||
|
}
|
||||||
|
iterator operator ++ (int)
|
||||||
|
{
|
||||||
|
iterator temp = *this;
|
||||||
|
::std::advance(it, utf8::internal::sequence_length(it));
|
||||||
|
return temp;
|
||||||
|
}
|
||||||
|
iterator& operator -- ()
|
||||||
|
{
|
||||||
|
utf8::unchecked::prior(it);
|
||||||
|
return *this;
|
||||||
|
}
|
||||||
|
iterator operator -- (int)
|
||||||
|
{
|
||||||
|
iterator temp = *this;
|
||||||
|
utf8::unchecked::prior(it);
|
||||||
|
return temp;
|
||||||
|
}
|
||||||
|
}; // class iterator
|
||||||
|
|
||||||
|
} // namespace utf8::unchecked
|
||||||
|
} // namespace utf8
|
||||||
|
|
||||||
|
|
||||||
|
#endif // header guard
|
||||||
|
|
Binary file not shown.
|
@ -0,0 +1,212 @@
|
||||||
|
|
||||||
|
UTF-8 encoded sample plain-text file
|
||||||
|
‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
|
||||||
|
|
||||||
|
Markus Kuhn [ˈmaʳkʊs kuːn] <http://www.cl.cam.ac.uk/~mgk25/> — 2002-07-25
|
||||||
|
|
||||||
|
|
||||||
|
The ASCII compatible UTF-8 encoding used in this plain-text file
|
||||||
|
is defined in Unicode, ISO 10646-1, and RFC 2279.
|
||||||
|
|
||||||
|
|
||||||
|
Using Unicode/UTF-8, you can write in emails and source code things such as
|
||||||
|
|
||||||
|
Mathematics and sciences:
|
||||||
|
|
||||||
|
∮ E⋅da = Q, n → ∞, ∑ f(i) = ∏ g(i), ⎧⎡⎛┌─────┐⎞⎤⎫
|
||||||
|
⎪⎢⎜│a²+b³ ⎟⎥⎪
|
||||||
|
∀x∈ℝ: ⌈x⌉ = −⌊−x⌋, α ∧ ¬β = ¬(¬α ∨ β), ⎪⎢⎜│───── ⎟⎥⎪
|
||||||
|
⎪⎢⎜⎷ c₈ ⎟⎥⎪
|
||||||
|
ℕ ⊆ ℕ₀ ⊂ ℤ ⊂ ℚ ⊂ ℝ ⊂ ℂ, ⎨⎢⎜ ⎟⎥⎬
|
||||||
|
⎪⎢⎜ ∞ ⎟⎥⎪
|
||||||
|
⊥ < a ≠ b ≡ c ≤ d ≪ ⊤ ⇒ (⟦A⟧ ⇔ ⟪B⟫), ⎪⎢⎜ ⎲ ⎟⎥⎪
|
||||||
|
⎪⎢⎜ ⎳aⁱ-bⁱ⎟⎥⎪
|
||||||
|
2H₂ + O₂ ⇌ 2H₂O, R = 4.7 kΩ, ⌀ 200 mm ⎩⎣⎝i=1 ⎠⎦⎭
|
||||||
|
|
||||||
|
Linguistics and dictionaries:
|
||||||
|
|
||||||
|
ði ıntəˈnæʃənəl fəˈnɛtık əsoʊsiˈeıʃn
|
||||||
|
Y [ˈʏpsilɔn], Yen [jɛn], Yoga [ˈjoːgɑ]
|
||||||
|
|
||||||
|
APL:
|
||||||
|
|
||||||
|
((V⍳V)=⍳⍴V)/V←,V ⌷←⍳→⍴∆∇⊃‾⍎⍕⌈
|
||||||
|
|
||||||
|
Nicer typography in plain text files:
|
||||||
|
|
||||||
|
╔══════════════════════════════════════════╗
|
||||||
|
║ ║
|
||||||
|
║ • ‘single’ and “double” quotes ║
|
||||||
|
║ ║
|
||||||
|
║ • Curly apostrophes: “We’ve been here” ║
|
||||||
|
║ ║
|
||||||
|
║ • Latin-1 apostrophe and accents: '´` ║
|
||||||
|
║ ║
|
||||||
|
║ • ‚deutsche‘ „Anführungszeichen“ ║
|
||||||
|
║ ║
|
||||||
|
║ • †, ‡, ‰, •, 3–4, —, −5/+5, ™, … ║
|
||||||
|
║ ║
|
||||||
|
║ • ASCII safety test: 1lI|, 0OD, 8B ║
|
||||||
|
║ ╭─────────╮ ║
|
||||||
|
║ • the euro symbol: │ 14.95 € │ ║
|
||||||
|
║ ╰─────────╯ ║
|
||||||
|
╚══════════════════════════════════════════╝
|
||||||
|
|
||||||
|
Combining characters:
|
||||||
|
|
||||||
|
STARGΛ̊TE SG-1, a = v̇ = r̈, a⃑ ⊥ b⃑
|
||||||
|
|
||||||
|
Greek (in Polytonic):
|
||||||
|
|
||||||
|
The Greek anthem:
|
||||||
|
|
||||||
|
Σὲ γνωρίζω ἀπὸ τὴν κόψη
|
||||||
|
τοῦ σπαθιοῦ τὴν τρομερή,
|
||||||
|
σὲ γνωρίζω ἀπὸ τὴν ὄψη
|
||||||
|
ποὺ μὲ βία μετράει τὴ γῆ.
|
||||||
|
|
||||||
|
᾿Απ᾿ τὰ κόκκαλα βγαλμένη
|
||||||
|
τῶν ῾Ελλήνων τὰ ἱερά
|
||||||
|
καὶ σὰν πρῶτα ἀνδρειωμένη
|
||||||
|
χαῖρε, ὦ χαῖρε, ᾿Ελευθεριά!
|
||||||
|
|
||||||
|
From a speech of Demosthenes in the 4th century BC:
|
||||||
|
|
||||||
|
Οὐχὶ ταὐτὰ παρίσταταί μοι γιγνώσκειν, ὦ ἄνδρες ᾿Αθηναῖοι,
|
||||||
|
ὅταν τ᾿ εἰς τὰ πράγματα ἀποβλέψω καὶ ὅταν πρὸς τοὺς
|
||||||
|
λόγους οὓς ἀκούω· τοὺς μὲν γὰρ λόγους περὶ τοῦ
|
||||||
|
τιμωρήσασθαι Φίλιππον ὁρῶ γιγνομένους, τὰ δὲ πράγματ᾿
|
||||||
|
εἰς τοῦτο προήκοντα, ὥσθ᾿ ὅπως μὴ πεισόμεθ᾿ αὐτοὶ
|
||||||
|
πρότερον κακῶς σκέψασθαι δέον. οὐδέν οὖν ἄλλο μοι δοκοῦσιν
|
||||||
|
οἱ τὰ τοιαῦτα λέγοντες ἢ τὴν ὑπόθεσιν, περὶ ἧς βουλεύεσθαι,
|
||||||
|
οὐχὶ τὴν οὖσαν παριστάντες ὑμῖν ἁμαρτάνειν. ἐγὼ δέ, ὅτι μέν
|
||||||
|
ποτ᾿ ἐξῆν τῇ πόλει καὶ τὰ αὑτῆς ἔχειν ἀσφαλῶς καὶ Φίλιππον
|
||||||
|
τιμωρήσασθαι, καὶ μάλ᾿ ἀκριβῶς οἶδα· ἐπ᾿ ἐμοῦ γάρ, οὐ πάλαι
|
||||||
|
γέγονεν ταῦτ᾿ ἀμφότερα· νῦν μέντοι πέπεισμαι τοῦθ᾿ ἱκανὸν
|
||||||
|
προλαβεῖν ἡμῖν εἶναι τὴν πρώτην, ὅπως τοὺς συμμάχους
|
||||||
|
σώσομεν. ἐὰν γὰρ τοῦτο βεβαίως ὑπάρξῃ, τότε καὶ περὶ τοῦ
|
||||||
|
τίνα τιμωρήσεταί τις καὶ ὃν τρόπον ἐξέσται σκοπεῖν· πρὶν δὲ
|
||||||
|
τὴν ἀρχὴν ὀρθῶς ὑποθέσθαι, μάταιον ἡγοῦμαι περὶ τῆς
|
||||||
|
τελευτῆς ὁντινοῦν ποιεῖσθαι λόγον.
|
||||||
|
|
||||||
|
Δημοσθένους, Γ´ ᾿Ολυνθιακὸς
|
||||||
|
|
||||||
|
Georgian:
|
||||||
|
|
||||||
|
From a Unicode conference invitation:
|
||||||
|
|
||||||
|
გთხოვთ ახლავე გაიაროთ რეგისტრაცია Unicode-ის მეათე საერთაშორისო
|
||||||
|
კონფერენციაზე დასასწრებად, რომელიც გაიმართება 10-12 მარტს,
|
||||||
|
ქ. მაინცში, გერმანიაში. კონფერენცია შეჰკრებს ერთად მსოფლიოს
|
||||||
|
ექსპერტებს ისეთ დარგებში როგორიცაა ინტერნეტი და Unicode-ი,
|
||||||
|
ინტერნაციონალიზაცია და ლოკალიზაცია, Unicode-ის გამოყენება
|
||||||
|
ოპერაციულ სისტემებსა, და გამოყენებით პროგრამებში, შრიფტებში,
|
||||||
|
ტექსტების დამუშავებასა და მრავალენოვან კომპიუტერულ სისტემებში.
|
||||||
|
|
||||||
|
Russian:
|
||||||
|
|
||||||
|
From a Unicode conference invitation:
|
||||||
|
|
||||||
|
Зарегистрируйтесь сейчас на Десятую Международную Конференцию по
|
||||||
|
Unicode, которая состоится 10-12 марта 1997 года в Майнце в Германии.
|
||||||
|
Конференция соберет широкий круг экспертов по вопросам глобального
|
||||||
|
Интернета и Unicode, локализации и интернационализации, воплощению и
|
||||||
|
применению Unicode в различных операционных системах и программных
|
||||||
|
приложениях, шрифтах, верстке и многоязычных компьютерных системах.
|
||||||
|
|
||||||
|
Thai (UCS Level 2):
|
||||||
|
|
||||||
|
Excerpt from a poetry on The Romance of The Three Kingdoms (a Chinese
|
||||||
|
classic 'San Gua'):
|
||||||
|
|
||||||
|
[----------------------------|------------------------]
|
||||||
|
๏ แผ่นดินฮั่นเสื่อมโทรมแสนสังเวช พระปกเกศกองบู๊กู้ขึ้นใหม่
|
||||||
|
สิบสองกษัตริย์ก่อนหน้าแลถัดไป สององค์ไซร้โง่เขลาเบาปัญญา
|
||||||
|
ทรงนับถือขันทีเป็นที่พึ่ง บ้านเมืองจึงวิปริตเป็นนักหนา
|
||||||
|
โฮจิ๋นเรียกทัพทั่วหัวเมืองมา หมายจะฆ่ามดชั่วตัวสำคัญ
|
||||||
|
เหมือนขับไสไล่เสือจากเคหา รับหมาป่าเข้ามาเลยอาสัญ
|
||||||
|
ฝ่ายอ้องอุ้นยุแยกให้แตกกัน ใช้สาวนั้นเป็นชนวนชื่นชวนใจ
|
||||||
|
พลันลิฉุยกุยกีกลับก่อเหตุ ช่างอาเพศจริงหนาฟ้าร้องไห้
|
||||||
|
ต้องรบราฆ่าฟันจนบรรลัย ฤๅหาใครค้ำชูกู้บรรลังก์ ฯ
|
||||||
|
|
||||||
|
(The above is a two-column text. If combining characters are handled
|
||||||
|
correctly, the lines of the second column should be aligned with the
|
||||||
|
| character above.)
|
||||||
|
|
||||||
|
Ethiopian:
|
||||||
|
|
||||||
|
Proverbs in the Amharic language:
|
||||||
|
|
||||||
|
ሰማይ አይታረስ ንጉሥ አይከሰስ።
|
||||||
|
ብላ ካለኝ እንደአባቴ በቆመጠኝ።
|
||||||
|
ጌጥ ያለቤቱ ቁምጥና ነው።
|
||||||
|
ደሀ በሕልሙ ቅቤ ባይጠጣ ንጣት በገደለው።
|
||||||
|
የአፍ ወለምታ በቅቤ አይታሽም።
|
||||||
|
አይጥ በበላ ዳዋ ተመታ።
|
||||||
|
ሲተረጉሙ ይደረግሙ።
|
||||||
|
ቀስ በቀስ፥ ዕንቁላል በእግሩ ይሄዳል።
|
||||||
|
ድር ቢያብር አንበሳ ያስር።
|
||||||
|
ሰው እንደቤቱ እንጅ እንደ ጉረቤቱ አይተዳደርም።
|
||||||
|
እግዜር የከፈተውን ጉሮሮ ሳይዘጋው አይድርም።
|
||||||
|
የጎረቤት ሌባ፥ ቢያዩት ይስቅ ባያዩት ያጠልቅ።
|
||||||
|
ሥራ ከመፍታት ልጄን ላፋታት።
|
||||||
|
ዓባይ ማደሪያ የለው፥ ግንድ ይዞ ይዞራል።
|
||||||
|
የእስላም አገሩ መካ የአሞራ አገሩ ዋርካ።
|
||||||
|
ተንጋሎ ቢተፉ ተመልሶ ባፉ።
|
||||||
|
ወዳጅህ ማር ቢሆን ጨርስህ አትላሰው።
|
||||||
|
እግርህን በፍራሽህ ልክ ዘርጋ።
|
||||||
|
|
||||||
|
Runes:
|
||||||
|
|
||||||
|
ᚻᛖ ᚳᚹᚫᚦ ᚦᚫᛏ ᚻᛖ ᛒᚢᛞᛖ ᚩᚾ ᚦᚫᛗ ᛚᚪᚾᛞᛖ ᚾᚩᚱᚦᚹᛖᚪᚱᛞᚢᛗ ᚹᛁᚦ ᚦᚪ ᚹᛖᛥᚫ
|
||||||
|
|
||||||
|
(Old English, which transcribed into Latin reads 'He cwaeth that he
|
||||||
|
bude thaem lande northweardum with tha Westsae.' and means 'He said
|
||||||
|
that he lived in the northern land near the Western Sea.')
|
||||||
|
|
||||||
|
Braille:
|
||||||
|
|
||||||
|
⡌⠁⠧⠑ ⠼⠁⠒ ⡍⠜⠇⠑⠹⠰⠎ ⡣⠕⠌
|
||||||
|
|
||||||
|
⡍⠜⠇⠑⠹ ⠺⠁⠎ ⠙⠑⠁⠙⠒ ⠞⠕ ⠃⠑⠛⠔ ⠺⠊⠹⠲ ⡹⠻⠑ ⠊⠎ ⠝⠕ ⠙⠳⠃⠞
|
||||||
|
⠱⠁⠞⠑⠧⠻ ⠁⠃⠳⠞ ⠹⠁⠞⠲ ⡹⠑ ⠗⠑⠛⠊⠌⠻ ⠕⠋ ⠙⠊⠎ ⠃⠥⠗⠊⠁⠇ ⠺⠁⠎
|
||||||
|
⠎⠊⠛⠝⠫ ⠃⠹ ⠹⠑ ⠊⠇⠻⠛⠹⠍⠁⠝⠂ ⠹⠑ ⠊⠇⠻⠅⠂ ⠹⠑ ⠥⠝⠙⠻⠞⠁⠅⠻⠂
|
||||||
|
⠁⠝⠙ ⠹⠑ ⠡⠊⠑⠋ ⠍⠳⠗⠝⠻⠲ ⡎⠊⠗⠕⠕⠛⠑ ⠎⠊⠛⠝⠫ ⠊⠞⠲ ⡁⠝⠙
|
||||||
|
⡎⠊⠗⠕⠕⠛⠑⠰⠎ ⠝⠁⠍⠑ ⠺⠁⠎ ⠛⠕⠕⠙ ⠥⠏⠕⠝ ⠰⡡⠁⠝⠛⠑⠂ ⠋⠕⠗ ⠁⠝⠹⠹⠔⠛ ⠙⠑
|
||||||
|
⠡⠕⠎⠑ ⠞⠕ ⠏⠥⠞ ⠙⠊⠎ ⠙⠁⠝⠙ ⠞⠕⠲
|
||||||
|
|
||||||
|
⡕⠇⠙ ⡍⠜⠇⠑⠹ ⠺⠁⠎ ⠁⠎ ⠙⠑⠁⠙ ⠁⠎ ⠁ ⠙⠕⠕⠗⠤⠝⠁⠊⠇⠲
|
||||||
|
|
||||||
|
⡍⠔⠙⠖ ⡊ ⠙⠕⠝⠰⠞ ⠍⠑⠁⠝ ⠞⠕ ⠎⠁⠹ ⠹⠁⠞ ⡊ ⠅⠝⠪⠂ ⠕⠋ ⠍⠹
|
||||||
|
⠪⠝ ⠅⠝⠪⠇⠫⠛⠑⠂ ⠱⠁⠞ ⠹⠻⠑ ⠊⠎ ⠏⠜⠞⠊⠊⠥⠇⠜⠇⠹ ⠙⠑⠁⠙ ⠁⠃⠳⠞
|
||||||
|
⠁ ⠙⠕⠕⠗⠤⠝⠁⠊⠇⠲ ⡊ ⠍⠊⠣⠞ ⠙⠁⠧⠑ ⠃⠑⠲ ⠔⠊⠇⠔⠫⠂ ⠍⠹⠎⠑⠇⠋⠂ ⠞⠕
|
||||||
|
⠗⠑⠛⠜⠙ ⠁ ⠊⠕⠋⠋⠔⠤⠝⠁⠊⠇ ⠁⠎ ⠹⠑ ⠙⠑⠁⠙⠑⠌ ⠏⠊⠑⠊⠑ ⠕⠋ ⠊⠗⠕⠝⠍⠕⠝⠛⠻⠹
|
||||||
|
⠔ ⠹⠑ ⠞⠗⠁⠙⠑⠲ ⡃⠥⠞ ⠹⠑ ⠺⠊⠎⠙⠕⠍ ⠕⠋ ⠳⠗ ⠁⠝⠊⠑⠌⠕⠗⠎
|
||||||
|
⠊⠎ ⠔ ⠹⠑ ⠎⠊⠍⠊⠇⠑⠆ ⠁⠝⠙ ⠍⠹ ⠥⠝⠙⠁⠇⠇⠪⠫ ⠙⠁⠝⠙⠎
|
||||||
|
⠩⠁⠇⠇ ⠝⠕⠞ ⠙⠊⠌⠥⠗⠃ ⠊⠞⠂ ⠕⠗ ⠹⠑ ⡊⠳⠝⠞⠗⠹⠰⠎ ⠙⠕⠝⠑ ⠋⠕⠗⠲ ⡹⠳
|
||||||
|
⠺⠊⠇⠇ ⠹⠻⠑⠋⠕⠗⠑ ⠏⠻⠍⠊⠞ ⠍⠑ ⠞⠕ ⠗⠑⠏⠑⠁⠞⠂ ⠑⠍⠏⠙⠁⠞⠊⠊⠁⠇⠇⠹⠂ ⠹⠁⠞
|
||||||
|
⡍⠜⠇⠑⠹ ⠺⠁⠎ ⠁⠎ ⠙⠑⠁⠙ ⠁⠎ ⠁ ⠙⠕⠕⠗⠤⠝⠁⠊⠇⠲
|
||||||
|
|
||||||
|
(The first couple of paragraphs of "A Christmas Carol" by Dickens)
|
||||||
|
|
||||||
|
Compact font selection example text:
|
||||||
|
|
||||||
|
ABCDEFGHIJKLMNOPQRSTUVWXYZ /0123456789
|
||||||
|
abcdefghijklmnopqrstuvwxyz £©µÀÆÖÞßéöÿ
|
||||||
|
–—‘“”„†•…‰™œŠŸž€ ΑΒΓΔΩαβγδω АБВГДабвгд
|
||||||
|
∀∂∈ℝ∧∪≡∞ ↑↗↨↻⇣ ┐┼╔╘░►☺♀ fi<>⑀₂ἠḂӥẄɐː⍎אԱა
|
||||||
|
|
||||||
|
Greetings in various languages:
|
||||||
|
|
||||||
|
Hello world, Καλημέρα κόσμε, コンニチハ
|
||||||
|
|
||||||
|
Box drawing alignment tests: █
|
||||||
|
▉
|
||||||
|
╔══╦══╗ ┌──┬──┐ ╭──┬──╮ ╭──┬──╮ ┏━━┳━━┓ ┎┒┏┑ ╷ ╻ ┏┯┓ ┌┰┐ ▊ ╱╲╱╲╳╳╳
|
||||||
|
║┌─╨─┐║ │╔═╧═╗│ │╒═╪═╕│ │╓─╁─╖│ ┃┌─╂─┐┃ ┗╃╄┙ ╶┼╴╺╋╸┠┼┨ ┝╋┥ ▋ ╲╱╲╱╳╳╳
|
||||||
|
║│╲ ╱│║ │║ ║│ ││ │ ││ │║ ┃ ║│ ┃│ ╿ │┃ ┍╅╆┓ ╵ ╹ ┗┷┛ └┸┘ ▌ ╱╲╱╲╳╳╳
|
||||||
|
╠╡ ╳ ╞╣ ├╢ ╟┤ ├┼─┼─┼┤ ├╫─╂─╫┤ ┣┿╾┼╼┿┫ ┕┛┖┚ ┌┄┄┐ ╎ ┏┅┅┓ ┋ ▍ ╲╱╲╱╳╳╳
|
||||||
|
║│╱ ╲│║ │║ ║│ ││ │ ││ │║ ┃ ║│ ┃│ ╽ │┃ ░░▒▒▓▓██ ┊ ┆ ╎ ╏ ┇ ┋ ▎
|
||||||
|
║└─╥─┘║ │╚═╤═╝│ │╘═╪═╛│ │╙─╀─╜│ ┃└─╂─┘┃ ░░▒▒▓▓██ ┊ ┆ ╎ ╏ ┇ ┋ ▏
|
||||||
|
╚══╩══╝ └──┴──┘ ╰──┴──╯ ╰──┴──╯ ┗━━┻━━┛ ▗▄▖▛▀▜ └╌╌┘ ╎ ┗╍╍┛ ┋ ▁▂▃▄▅▆▇█
|
||||||
|
▝▀▘▙▄▟
|
|
@ -0,0 +1,167 @@
|
||||||
|
? *Unicode Transcriptions* Notes <#Notes>
|
||||||
|
|
||||||
|
Glyphs <http://www.macchiato.com/unicode/show.html> | Samples
|
||||||
|
<http://www.macchiato.com/unicode/Unicode_transcriptions.html> | Charts
|
||||||
|
<http://www.macchiato.com/unicode/charts.html> | UTF
|
||||||
|
<http://www.macchiato.com/unicode/convert.html> | Forms
|
||||||
|
<http://www-4.ibm.com/software/developer/library/utfencodingforms/> |
|
||||||
|
Home <http://www.macchiato.com>.
|
||||||
|
<http://member.linkexchange.com/cgi-bin/fc/fastcounter-login?750641>
|
||||||
|
|
||||||
|
Name Text Image
|
||||||
|
Arabic (Arabic) يونِكود ?
|
||||||
|
Arabic (Persian) یونیکُد / ?/
|
||||||
|
Armenian Յունիկօդ
|
||||||
|
Bengali য়ূনিকোড
|
||||||
|
Bopomofo ㄊㄨㄥ˅ ㄧˋ ㄇㄚ˅
|
||||||
|
ㄨㄢˋ ㄍㄨㄛˊ ㄇㄚ˅
|
||||||
|
Braille
|
||||||
|
Buhid
|
||||||
|
Canadian Aboriginal ᔫᗂᑰᑦ
|
||||||
|
Cherokee ᏳᏂᎪᏛ
|
||||||
|
Cypriot
|
||||||
|
Cyrillic (Russian) Юникод ?
|
||||||
|
Deseret (English) ???????
|
||||||
|
Devanagari (Hindi) यूनिकोड ?
|
||||||
|
Ethiopic ዩኒኮድ
|
||||||
|
Georgian უნიკოდი ?
|
||||||
|
Gothic
|
||||||
|
Greek Γιούνικοντ
|
||||||
|
Gujarati યૂનિકોડ
|
||||||
|
Gurmukhi ਯੂਨਿਕੋਡ
|
||||||
|
Han (Chinese) 统一码 ?
|
||||||
|
統一碼 ?
|
||||||
|
万国码 ?
|
||||||
|
萬國碼 ?
|
||||||
|
Hangul 유니코드
|
||||||
|
Hanunoo
|
||||||
|
Hebrew יוניקוד
|
||||||
|
Hebrew (pointed) יוּנִיקוׁד
|
||||||
|
Hebrew (Yiddish) יוניקאָד ?
|
||||||
|
Hiragana (Japanese) ゆにこおど
|
||||||
|
Katakana (Japanese) ユニコード ?
|
||||||
|
Kannada ಯೂನಿಕೋಡ್
|
||||||
|
Khmer យូនីគោដ
|
||||||
|
Lao
|
||||||
|
Latin Unicode Unicode
|
||||||
|
Latin (IPA <#English_Pronunciation>) ˈjunɪˌkoːd ?
|
||||||
|
Latin (Am. Dict. <#American_Dictionary>) Ūnĭcōde̽ ?
|
||||||
|
Limbu
|
||||||
|
Linear B
|
||||||
|
Malayalam യൂനികോഡ്
|
||||||
|
Mongolian
|
||||||
|
Myanmar
|
||||||
|
Ogham ᚔᚒᚅᚔᚉᚑᚇ / /
|
||||||
|
Old Italic
|
||||||
|
Oriya ୟୂନିକୋଡ
|
||||||
|
Osmanya
|
||||||
|
Runic (Anglo-Saxon) ᛡᚢᚾᛁᚳᚩᛞ
|
||||||
|
Shavian
|
||||||
|
Sinhala යණනිකෞද්
|
||||||
|
Syriac ܝܘܢܝܩܘܕ
|
||||||
|
Tagbanwa
|
||||||
|
Tagalog
|
||||||
|
Tai Le
|
||||||
|
Tamil யூனிகோட்
|
||||||
|
Telugu యూనికోడ్
|
||||||
|
Thaana
|
||||||
|
Thai ยูนืโคด
|
||||||
|
Tibetan (Dzongkha) ཨུ་ནི་ཀོཌྲ།
|
||||||
|
Ugaritic
|
||||||
|
Yi
|
||||||
|
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
|
||||||
|
There are different ways to transcribe the word “Unicode”, depending on
|
||||||
|
the language and script. In some cases there is only one language that
|
||||||
|
customarily uses a given script; in others there are many languages. The
|
||||||
|
goal here is at a minimum to collect at least one transcription for each
|
||||||
|
script in a language customarily written in that script, with more
|
||||||
|
languages if possible. If the transcription is the same for multiple
|
||||||
|
languages in a script, then a single representative language is used.
|
||||||
|
|
||||||
|
Still missing are transcriptions for the items above in RED (in at least
|
||||||
|
one language). I would appreciate any other transcriptions, or
|
||||||
|
corrections for the ones listed here. Send to mark3@macchiato.com
|
||||||
|
<mailto:mark3@macchiato.com>, using the directions below:
|
||||||
|
|
||||||
|
* *Supplying Missing Items*
|
||||||
|
o Most Latin-script languages will follow the spelling, and
|
||||||
|
change the pronunciation. For any that would not, it would
|
||||||
|
be good to have the alternate spelling.
|
||||||
|
o For non-Latin scripts the goal is to match the English
|
||||||
|
pronunciation — /*not*/ spelling. Above is the IPA <#IPA>
|
||||||
|
(in phonemic transcription) that should be matched as
|
||||||
|
closely as possible (without sounding affected in the target
|
||||||
|
language)
|
||||||
|
o Text would be best in either the UTF-8 text, or the code
|
||||||
|
points in hex HTML. E.g. either of the following:
|
||||||
|
+ "Юникод"
|
||||||
|
+ "Юникод"
|
||||||
|
+ Note: for / supplementary characters/
|
||||||
|
<http://www.unicode.org/glossary/#supplementary_character>,
|
||||||
|
there should be one hex number per code point, not two
|
||||||
|
surrogates
|
||||||
|
<http://www.unicode.org/glossary/#surrogate_code_point>:
|
||||||
|
# 𐀀 /*not*/ �&xDC00;
|
||||||
|
o If you have a good font, I'd also appreciate a GIF. It
|
||||||
|
should be *96 x 24* bits, with the text centered, in black
|
||||||
|
on white (plus grays if smoothed).
|
||||||
|
* *Other Comments*
|
||||||
|
o Because some browsers won't handle the text, both text and
|
||||||
|
GIF image are supplied. If you can’t read the text columns,
|
||||||
|
see Display Problems
|
||||||
|
<http://www.unicode.org/help/display_problems.html>.
|
||||||
|
o The Chinese versions (inc. Bopomofo) are translations, not
|
||||||
|
transcriptions, since "transcription in Chinese is pretty
|
||||||
|
lame" [J. Becker].
|
||||||
|
o There are other "translations" of Unicode that may be in
|
||||||
|
use, such as the Vietnamese "Thống Nhất Mã".
|
||||||
|
o For sample pages in different languages on the Unicode site,
|
||||||
|
see What is Unicode?
|
||||||
|
<http://www.unicode.org/unicode/standard/WhatIsUnicode.html>
|
||||||
|
o Americans are not generally used to IPA, and find a variety
|
||||||
|
of different systems in their dictionaries. This one leaves
|
||||||
|
the base letters as they are, and uses diacritics for
|
||||||
|
pronunciation.
|
||||||
|
* *Etymology of /Unicode/*
|
||||||
|
o Coined by J. Becker. Not related to previous usages, such as:
|
||||||
|
+ A telegraphic code in which one word or set of letters
|
||||||
|
represents a sentence or phrase; a telegram or message
|
||||||
|
in this. (late 19th century, OED)
|
||||||
|
o According to my references, the prefix "uni" is directly
|
||||||
|
from Latin while the word "code" is through French.
|
||||||
|
o The original Indo-European apparently would have been
|
||||||
|
*oino-kau-do ("one strike give"): *kau apparently being
|
||||||
|
related to such English words as: hew, haggle, hoe, hag,
|
||||||
|
hay, hack, caudad, caudal, caudate, caudex, coda, codex,
|
||||||
|
codicil, coward, incus, and Kovač (personal name: "smith").
|
||||||
|
+ I will leave the exact derivations to the exegetes,
|
||||||
|
but I like the association with "haggle" myself.
|
||||||
|
* *Contributions*
|
||||||
|
o This draws on contributions or comments from:
|
||||||
|
+ Dixon Au
|
||||||
|
+ Joe Becker
|
||||||
|
+ Maurice Bauhahn
|
||||||
|
+ Abel Cheung
|
||||||
|
+ Peter Constable
|
||||||
|
+ Michael Everson
|
||||||
|
+ Christopher John Fynn
|
||||||
|
+ Michael Kaplan
|
||||||
|
+ George Kiraz
|
||||||
|
+ Abdul Malik
|
||||||
|
+ Siva Nataraja
|
||||||
|
+ Roozbeh Pournader
|
||||||
|
+ Jonathan Rosenne
|
||||||
|
+ Jungshik Shin
|
||||||
|
|
||||||
|
------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
Terms of Use <http://www.macchiato.com/terms_of_use.html>. Last updated:
|
||||||
|
MED - 04/20/2003 15:30:33.
|
||||||
|
<http://member.linkexchange.com/cgi-bin/fc/fastcounter-login?750641>
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,126 @@
|
||||||
|
Sentences that contain all letters commonly used in a language
|
||||||
|
--------------------------------------------------------------
|
||||||
|
|
||||||
|
Markus Kuhn <http://www.cl.cam.ac.uk/~mgk25/> -- 2001-09-02
|
||||||
|
|
||||||
|
This file is UTF-8 encoded.
|
||||||
|
|
||||||
|
|
||||||
|
Danish (da)
|
||||||
|
---------
|
||||||
|
|
||||||
|
Quizdeltagerne spiste jordbær med fløde, mens cirkusklovnen
|
||||||
|
Wolther spillede på xylofon.
|
||||||
|
(= Quiz contestants were eating strawbery with cream while Wolther
|
||||||
|
the circus clown played on xylophone.)
|
||||||
|
|
||||||
|
German (de)
|
||||||
|
-----------
|
||||||
|
|
||||||
|
Falsches Üben von Xylophonmusik quält jeden größeren Zwerg
|
||||||
|
(= Wrongful practicing of xylophone music tortures every larger dwarf)
|
||||||
|
|
||||||
|
Zwölf Boxkämpfer jagten Eva quer über den Sylter Deich
|
||||||
|
(= Twelve boxing fighters hunted Eva across the dike of Sylt)
|
||||||
|
|
||||||
|
Heizölrückstoßabdämpfung
|
||||||
|
(= fuel oil recoil absorber)
|
||||||
|
(jqvwxy missing, but all non-ASCII letters in one word)
|
||||||
|
|
||||||
|
English (en)
|
||||||
|
------------
|
||||||
|
|
||||||
|
The quick brown fox jumps over the lazy dog
|
||||||
|
|
||||||
|
Spanish (es)
|
||||||
|
------------
|
||||||
|
|
||||||
|
El pingüino Wenceslao hizo kilómetros bajo exhaustiva lluvia y
|
||||||
|
frío, añoraba a su querido cachorro.
|
||||||
|
(Contains every letter and every accent, but not every combination
|
||||||
|
of vowel + acute.)
|
||||||
|
|
||||||
|
French (fr)
|
||||||
|
-----------
|
||||||
|
|
||||||
|
Portez ce vieux whisky au juge blond qui fume sur son île intérieure, à
|
||||||
|
côté de l'alcôve ovoïde, où les bûches se consument dans l'âtre, ce
|
||||||
|
qui lui permet de penser à la cænogenèse de l'être dont il est question
|
||||||
|
dans la cause ambiguë entendue à Moÿ, dans un capharnaüm qui,
|
||||||
|
pense-t-il, diminue çà et là la qualité de son œuvre.
|
||||||
|
|
||||||
|
l'île exiguë
|
||||||
|
Où l'obèse jury mûr
|
||||||
|
Fête l'haï volapük,
|
||||||
|
Âne ex aéquo au whist,
|
||||||
|
Ôtez ce vœu déçu.
|
||||||
|
|
||||||
|
Le cœur déçu mais l'âme plutôt naïve, Louÿs rêva de crapaüter en
|
||||||
|
canoë au delà des îles, près du mälström où brûlent les novæ.
|
||||||
|
|
||||||
|
Irish Gaelic (ga)
|
||||||
|
-----------------
|
||||||
|
|
||||||
|
D'fhuascail Íosa, Úrmhac na hÓighe Beannaithe, pór Éava agus Ádhaimh
|
||||||
|
|
||||||
|
Hungarian (hu)
|
||||||
|
--------------
|
||||||
|
|
||||||
|
Árvíztűrő tükörfúrógép
|
||||||
|
(= flood-proof mirror-drilling machine, only all non-ASCII letters)
|
||||||
|
|
||||||
|
Icelandic (is)
|
||||||
|
--------------
|
||||||
|
|
||||||
|
Kæmi ný öxi hér ykist þjófum nú bæði víl og ádrepa
|
||||||
|
|
||||||
|
Sævör grét áðan því úlpan var ónýt
|
||||||
|
(some ASCII letters missing)
|
||||||
|
|
||||||
|
Japanese (jp)
|
||||||
|
-------------
|
||||||
|
|
||||||
|
Hiragana: (Iroha)
|
||||||
|
|
||||||
|
いろはにほへとちりぬるを
|
||||||
|
わかよたれそつねならむ
|
||||||
|
うゐのおくやまけふこえて
|
||||||
|
あさきゆめみしゑひもせす
|
||||||
|
|
||||||
|
Katakana:
|
||||||
|
|
||||||
|
イロハニホヘト チリヌルヲ ワカヨタレソ ツネナラム
|
||||||
|
ウヰノオクヤマ ケフコエテ アサキユメミシ ヱヒモセスン
|
||||||
|
|
||||||
|
Hebrew (iw)
|
||||||
|
-----------
|
||||||
|
|
||||||
|
? דג סקרן שט בים מאוכזב ולפתע מצא לו חברה איך הקליטה
|
||||||
|
|
||||||
|
Polish (pl)
|
||||||
|
-----------
|
||||||
|
|
||||||
|
Pchnąć w tę łódź jeża lub ośm skrzyń fig
|
||||||
|
(= To push a hedgehog or eight bins of figs in this boat)
|
||||||
|
|
||||||
|
Russian (ru)
|
||||||
|
------------
|
||||||
|
|
||||||
|
В чащах юга жил бы цитрус? Да, но фальшивый экземпляр!
|
||||||
|
(= Would a citrus live in the bushes of south? Yes, but only a fake one!)
|
||||||
|
|
||||||
|
Thai (th)
|
||||||
|
---------
|
||||||
|
|
||||||
|
[--------------------------|------------------------]
|
||||||
|
๏ เป็นมนุษย์สุดประเสริฐเลิศคุณค่า กว่าบรรดาฝูงสัตว์เดรัจฉาน
|
||||||
|
จงฝ่าฟันพัฒนาวิชาการ อย่าล้างผลาญฤๅเข่นฆ่าบีฑาใคร
|
||||||
|
ไม่ถือโทษโกรธแช่งซัดฮึดฮัดด่า หัดอภัยเหมือนกีฬาอัชฌาสัย
|
||||||
|
ปฏิบัติประพฤติกฎกำหนดใจ พูดจาให้จ๊ะๆ จ๋าๆ น่าฟังเอย ฯ
|
||||||
|
|
||||||
|
[The copyright for the Thai example is owned by The Computer
|
||||||
|
Association of Thailand under the Royal Patronage of His Majesty the
|
||||||
|
King.]
|
||||||
|
|
||||||
|
Please let me know if you find others! Special thanks to the people
|
||||||
|
from all over the world who contributed these sentences.
|
|
@ -0,0 +1,19 @@
|
||||||
|
CC = g++
|
||||||
|
CFLAGS = -g
|
||||||
|
|
||||||
|
all: smoketest regressiontest negativetest utf8readertest
|
||||||
|
|
||||||
|
smoketest:
|
||||||
|
cd smoke_test && $(MAKE) $@
|
||||||
|
|
||||||
|
regressiontest:
|
||||||
|
cd regression_tests && $(MAKE) $@
|
||||||
|
|
||||||
|
negativetest:
|
||||||
|
cd negative && $(MAKE) $@
|
||||||
|
|
||||||
|
utf8readertest:
|
||||||
|
cd utf8reader && $(MAKE) $@
|
||||||
|
|
||||||
|
clean:
|
||||||
|
rm smoke_test/smoketest regression_tests/regressiontest negative/negative utf8reader/utf8reader
|
|
@ -0,0 +1,5 @@
|
||||||
|
CC = g++
|
||||||
|
CFLAGS = -g -Wall -pedantic
|
||||||
|
|
||||||
|
negativetest: negative.cpp ../../source/utf8.h ../../source/utf8/core.h ../../source/utf8/checked.h ../../source/utf8/unchecked.h
|
||||||
|
$(CC) $(CFLAGS) negative.cpp -onegative
|
|
@ -0,0 +1,53 @@
|
||||||
|
#include "../../source/utf8.h"
|
||||||
|
using namespace utf8;
|
||||||
|
|
||||||
|
#include <string>
|
||||||
|
#include <iostream>
|
||||||
|
#include <fstream>
|
||||||
|
#include <algorithm>
|
||||||
|
using namespace std;
|
||||||
|
|
||||||
|
const unsigned INVALID_LINES[] = { 75, 76, 83, 84, 85, 93, 102, 103, 105, 106, 107, 108, 109, 110, 114, 115, 116, 117, 124, 125, 130, 135, 140, 145, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 169, 175, 176, 177, 207, 208, 209, 210, 211, 220, 221, 222, 223, 224, 232, 233, 234, 235, 236, 247, 248, 249, 250, 251, 252, 253, 257, 258, 259, 260, 261, 262, 263, 264};
|
||||||
|
const unsigned* INVALID_LINES_END = INVALID_LINES + sizeof(INVALID_LINES)/sizeof(unsigned);
|
||||||
|
|
||||||
|
int main(int argc, char** argv)
|
||||||
|
{
|
||||||
|
string test_file_path;
|
||||||
|
if (argc == 2)
|
||||||
|
test_file_path = argv[1];
|
||||||
|
else {
|
||||||
|
cout << "Wrong number of arguments" << endl;
|
||||||
|
exit(0);
|
||||||
|
}
|
||||||
|
// Open the test file
|
||||||
|
ifstream fs8(test_file_path.c_str());
|
||||||
|
if (!fs8.is_open()) {
|
||||||
|
cout << "Could not open " << test_file_path << endl;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read it line by line
|
||||||
|
unsigned int line_count = 0;
|
||||||
|
char byte;
|
||||||
|
while (!fs8.eof()) {
|
||||||
|
string line;
|
||||||
|
while ((byte = static_cast<char>(fs8.get())) != '\n' && !fs8.eof())
|
||||||
|
line.push_back(byte);
|
||||||
|
|
||||||
|
line_count++;
|
||||||
|
bool expected_valid = (find(INVALID_LINES, INVALID_LINES_END, line_count) == INVALID_LINES_END);
|
||||||
|
// Print out lines that contain unexpected invalid UTF-8
|
||||||
|
if (!is_valid(line.begin(), line.end())) {
|
||||||
|
if (expected_valid)
|
||||||
|
cout << "Unexpected invalid utf-8 at line " << line_count << '\n';
|
||||||
|
|
||||||
|
// try fixing it:
|
||||||
|
string fixed_line;
|
||||||
|
replace_invalid(line.begin(), line.end(), back_inserter(fixed_line));
|
||||||
|
if (!is_valid(fixed_line.begin(), fixed_line.end()))
|
||||||
|
cout << "replace_invalid() resulted in an invalid utf-8 at line " << line_count << '\n';
|
||||||
|
}
|
||||||
|
else if (!expected_valid)
|
||||||
|
cout << "Invalid utf-8 NOT detected at line " << line_count << '\n';
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,5 @@
|
||||||
|
CC = g++
|
||||||
|
CFLAGS = -O3
|
||||||
|
|
||||||
|
iconvtest: iconvtest.cpp ../../source/utf8.h timer.h ../../source/utf8/core.h ../../source/utf8/checked.h ../../source/utf8/unchecked.h
|
||||||
|
$(CC) $(CFLAGS) iconvtest.cpp -oiconvtest
|
|
@ -0,0 +1,132 @@
|
||||||
|
#include <iconv.h>
|
||||||
|
#include <string.h>
|
||||||
|
#include "../../source/utf8.h"
|
||||||
|
#include "timer.h"
|
||||||
|
#include <fstream>
|
||||||
|
#include <algorithm>
|
||||||
|
#include <vector>
|
||||||
|
using namespace std;
|
||||||
|
|
||||||
|
using namespace utf8;
|
||||||
|
|
||||||
|
int main(int argc, char** argv)
|
||||||
|
{
|
||||||
|
if (argc != 2) {
|
||||||
|
cout << "\nUsage: iconvtest filename\n";
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
const char* test_file_path = argv[1];
|
||||||
|
// Open the test file (UTF-8 encoded text)
|
||||||
|
ifstream fs8(test_file_path, ios::binary);
|
||||||
|
if (!fs8.is_open()) {
|
||||||
|
cout << "Could not open " << test_file_path << endl;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
// get length
|
||||||
|
fs8.seekg(0, ios::end);
|
||||||
|
int length = fs8.tellg();
|
||||||
|
fs8.seekg(0, ios::beg);
|
||||||
|
|
||||||
|
// allocate the buffer (no vector - we are benchmarking conversions, not STL
|
||||||
|
char* buf = new char[length];
|
||||||
|
char* end_buf = buf + length;
|
||||||
|
// fill the data
|
||||||
|
fs8.read(buf, length);
|
||||||
|
fs8.close();
|
||||||
|
// the UTF-16 result will not be larger than this (I hope :) )
|
||||||
|
vector<unsigned char> temputf16;
|
||||||
|
utf8::utf8to16(buf, end_buf, back_inserter(temputf16));
|
||||||
|
int wlength = temputf16.size();
|
||||||
|
unsigned short* utf16buf = new unsigned short[wlength];
|
||||||
|
|
||||||
|
cout << "UTF8 to UTF-16\n";
|
||||||
|
{
|
||||||
|
memset (utf16buf, 0 , wlength * sizeof(unsigned short));
|
||||||
|
// utf-8 cpp:
|
||||||
|
cout << "utf8::utf8to16: ";
|
||||||
|
timer t(cout);
|
||||||
|
utf8::utf8to16(buf, buf + length, utf16buf);
|
||||||
|
t.print_time();
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
memset (utf16buf, 0 , wlength * sizeof(unsigned short));
|
||||||
|
// utf-8 cpp:
|
||||||
|
cout << "unchecked::utf8to16: ";
|
||||||
|
timer t(cout);
|
||||||
|
utf8::unchecked::utf8to16(buf, buf + length, utf16buf);
|
||||||
|
t.print_time();
|
||||||
|
}
|
||||||
|
|
||||||
|
// the UTF-16 result will not be larger than this (I hope :) )
|
||||||
|
unsigned short* utf16iconvbuf = new unsigned short[wlength];
|
||||||
|
{
|
||||||
|
memset (utf16iconvbuf, 0 , wlength * sizeof(unsigned short));
|
||||||
|
// iconv
|
||||||
|
cout << "iconv: ";
|
||||||
|
|
||||||
|
iconv_t cd = iconv_open("UTF-16LE", "UTF-8");
|
||||||
|
if (cd == iconv_t(-1)) {
|
||||||
|
cout << "Error openning the iconv stream";
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
char* inbuf = buf;
|
||||||
|
size_t in_bytes_left = length;
|
||||||
|
char* outbuf = (char*)utf16iconvbuf;
|
||||||
|
size_t out_bytes_left = wlength * sizeof (unsigned short);
|
||||||
|
{
|
||||||
|
timer t(cout);
|
||||||
|
iconv(cd, &inbuf, &in_bytes_left, &outbuf, &out_bytes_left);
|
||||||
|
t.print_time();
|
||||||
|
}
|
||||||
|
iconv_close(cd);
|
||||||
|
}
|
||||||
|
|
||||||
|
// just check the correctness while we are here:
|
||||||
|
if (!equal(utf16buf, utf16buf + wlength, utf16iconvbuf))
|
||||||
|
cout << "Different result!!!\n";
|
||||||
|
|
||||||
|
// the other way around
|
||||||
|
cout << "UTF16 to UTF-8\n";
|
||||||
|
{
|
||||||
|
//iconv
|
||||||
|
memset(buf, 0, length);
|
||||||
|
cout<< "iconv: ";
|
||||||
|
|
||||||
|
iconv_t cd = iconv_open("UTF-8", "UTF-16LE");
|
||||||
|
if (cd == iconv_t(-1)) {
|
||||||
|
cout << "Error openning the iconv stream";
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
char* inbuf = (char*)utf16buf;
|
||||||
|
size_t in_bytes_left = wlength * sizeof(unsigned short);
|
||||||
|
char* outbuf =buf;
|
||||||
|
size_t out_bytes_left = length;
|
||||||
|
{
|
||||||
|
timer t(cout);
|
||||||
|
iconv(cd, &inbuf, &in_bytes_left, &outbuf, &out_bytes_left);
|
||||||
|
t.print_time();
|
||||||
|
}
|
||||||
|
iconv_close(cd);
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
memset (buf, 0 , length);
|
||||||
|
// utf-8 cpp:
|
||||||
|
cout << "unchecked::utf16to8: ";
|
||||||
|
timer t(cout);
|
||||||
|
utf8::unchecked::utf16to8(utf16buf, utf16buf + wlength, buf);
|
||||||
|
t.print_time();
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
memset (buf, 0 , length);
|
||||||
|
cout << "utf16to8: ";
|
||||||
|
timer t(cout);
|
||||||
|
utf8::utf16to8(utf16buf, utf16buf + wlength, buf);
|
||||||
|
t.print_time();
|
||||||
|
}
|
||||||
|
|
||||||
|
delete [] buf;
|
||||||
|
delete [] utf16buf;
|
||||||
|
}
|
|
@ -0,0 +1,22 @@
|
||||||
|
#include <ctime>
|
||||||
|
#include <iostream>
|
||||||
|
struct timer {
|
||||||
|
timer(std::ostream& report) : report(report)
|
||||||
|
{start = std::clock();}
|
||||||
|
|
||||||
|
void print_time()
|
||||||
|
{
|
||||||
|
using namespace std;
|
||||||
|
clock_t now = clock();
|
||||||
|
unsigned milliseconds = (now - start)*1000 / CLOCKS_PER_SEC;
|
||||||
|
report << "Spent " << milliseconds << "ms here\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
std::clock_t start;
|
||||||
|
std::ostream& report;
|
||||||
|
|
||||||
|
private:
|
||||||
|
// just to surpress a VC++ 8.0 warning
|
||||||
|
timer& operator = (const timer&);
|
||||||
|
timer(const timer&);
|
||||||
|
};
|
|
@ -0,0 +1,110 @@
|
||||||
|
#include <windows.h>
|
||||||
|
#include "../../source/utf8.h"
|
||||||
|
#include "timer.h"
|
||||||
|
#include <fstream>
|
||||||
|
#include <algorithm>
|
||||||
|
#include <vector>
|
||||||
|
using namespace std;
|
||||||
|
|
||||||
|
using namespace utf8;
|
||||||
|
|
||||||
|
int main(int argc, char** argv)
|
||||||
|
{
|
||||||
|
if (argc != 2) {
|
||||||
|
cout << "\nUsage: win32test filename\n";
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
const char* test_file_path = argv[1];
|
||||||
|
// Open the test file (UTF-8 encoded text)
|
||||||
|
ifstream fs8(test_file_path, ios::binary);
|
||||||
|
if (!fs8.is_open()) {
|
||||||
|
cout << "Could not open " << test_file_path << endl;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
// get length
|
||||||
|
fs8.seekg(0, ios::end);
|
||||||
|
int length = fs8.tellg();
|
||||||
|
fs8.seekg(0, ios::beg);
|
||||||
|
|
||||||
|
// allocate the buffer (no vector - we are benchmarking conversions, not STL
|
||||||
|
char* buf = new char[length];
|
||||||
|
// fill the data
|
||||||
|
fs8.read(buf, length);
|
||||||
|
fs8.close();
|
||||||
|
cout << "UTF8 > UTF16\n";
|
||||||
|
// the UTF-16 result will not be larger than this (I hope :) )
|
||||||
|
vector<wchar_t> temputf16;
|
||||||
|
utf8::utf8to16(buf, buf + length, back_inserter(temputf16));
|
||||||
|
vector<wchar_t>::size_type wlength = temputf16.size();
|
||||||
|
wchar_t* utf16buf = new wchar_t[wlength];
|
||||||
|
|
||||||
|
{
|
||||||
|
memset (utf16buf, 0 , wlength * sizeof(wchar_t));
|
||||||
|
// utf-8 cpp:
|
||||||
|
cout << "utf8::utf8to16: ";
|
||||||
|
timer t(cout);
|
||||||
|
utf8::utf8to16(buf, buf + length, utf16buf);
|
||||||
|
t.print_time();
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
memset (utf16buf, 0 , wlength * sizeof(wchar_t));
|
||||||
|
// utf-8 cpp:
|
||||||
|
cout << "unchecked::utf8to16: ";
|
||||||
|
timer t(cout);
|
||||||
|
utf8::unchecked::utf8to16(buf, buf + length, utf16buf);
|
||||||
|
t.print_time();
|
||||||
|
}
|
||||||
|
// the UTF-16 result will not be larger than this (I hope :) )
|
||||||
|
wchar_t* utf16iconvbuf = new wchar_t[wlength];
|
||||||
|
{
|
||||||
|
memset (utf16iconvbuf, 0 , wlength * sizeof(wchar_t));
|
||||||
|
// win32
|
||||||
|
cout << "win32: ";
|
||||||
|
|
||||||
|
{
|
||||||
|
timer t(cout);
|
||||||
|
MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, buf, length, utf16iconvbuf, int(wlength));
|
||||||
|
t.print_time();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// just check the correctness while we are here:
|
||||||
|
if (!equal(utf16buf, utf16buf + wlength, utf16iconvbuf))
|
||||||
|
cout << "Different result!!!";
|
||||||
|
|
||||||
|
// the other way around
|
||||||
|
cout << "UTF16 to UTF-8\n";
|
||||||
|
{
|
||||||
|
//win32
|
||||||
|
memset(buf, 0, length);
|
||||||
|
cout<< "win32: ";
|
||||||
|
|
||||||
|
{
|
||||||
|
timer t(cout);
|
||||||
|
WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, utf16buf, int(wlength), buf, length, NULL, NULL);
|
||||||
|
t.print_time();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
memset (buf, 0 , length);
|
||||||
|
// utf-8 cpp:
|
||||||
|
cout << "unchecked::utf16to8: ";
|
||||||
|
timer t(cout);
|
||||||
|
utf8::unchecked::utf16to8(utf16buf, utf16buf + wlength, buf);
|
||||||
|
t.print_time();
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
memset (buf, 0 , length);
|
||||||
|
cout << "utf16to8: ";
|
||||||
|
timer t(cout);
|
||||||
|
utf8::utf16to8(utf16buf, utf16buf + wlength, buf);
|
||||||
|
t.print_time();
|
||||||
|
}
|
||||||
|
|
||||||
|
delete [] buf;
|
||||||
|
delete [] utf16buf;
|
||||||
|
}
|
|
@ -0,0 +1,50 @@
|
||||||
|
#! /usr/bin/perl
|
||||||
|
|
||||||
|
$report_name = './report.txt';
|
||||||
|
|
||||||
|
# Create the report file
|
||||||
|
die if !open(REPORT, ">$report_name");
|
||||||
|
|
||||||
|
# First, build everything
|
||||||
|
print REPORT "==================Make output==================\n";
|
||||||
|
close($report_name);
|
||||||
|
|
||||||
|
`make >> $report_name`;
|
||||||
|
die if !open(REPORT, ">>$report_name");
|
||||||
|
print REPORT "==================End of Make output==================\n";
|
||||||
|
print REPORT "\n";
|
||||||
|
# Now, run individual tests and create the report
|
||||||
|
print REPORT "==================Smoke Test ==================\n";
|
||||||
|
close($report_name);
|
||||||
|
chdir 'smoke_test';
|
||||||
|
`./smoketest >> ../$report_name`;
|
||||||
|
chdir '..';
|
||||||
|
die if !open(REPORT, ">>$report_name");
|
||||||
|
print REPORT "==================End of smoke test==================\n";
|
||||||
|
print REPORT "\n";
|
||||||
|
print REPORT "==================Regression Test ==================\n";
|
||||||
|
close($report_name);
|
||||||
|
chdir 'regression_tests';
|
||||||
|
`./regressiontest >> ../$report_name`;
|
||||||
|
chdir '..';
|
||||||
|
die if !open(REPORT, ">>$report_name");
|
||||||
|
print REPORT "==================End of regression test==================\n";
|
||||||
|
print REPORT "\n";
|
||||||
|
print REPORT "==================Negative Test ==================\n";
|
||||||
|
close($report_name);
|
||||||
|
chdir 'negative';
|
||||||
|
`./negative ../../test_data/negative/utf8_invalid.txt >> ../$report_name`;
|
||||||
|
chdir '..';
|
||||||
|
die if !open(REPORT, ">>$report_name");
|
||||||
|
print REPORT "==================End of negative test==================\n";
|
||||||
|
print REPORT "\n";
|
||||||
|
print REPORT "==================utf8reader runs ==================\n";
|
||||||
|
close($report_name);
|
||||||
|
chdir 'utf8reader';
|
||||||
|
`./utf8reader ../../test_data/utf8samples/quickbrown.txt >> ../$report_name`;
|
||||||
|
`./utf8reader ../../test_data/utf8samples/Unicode_transcriptions.html >> ../$report_name`;
|
||||||
|
`./utf8reader ../../test_data/utf8samples/UTF-8-demo.txt >> ../$report_name`;
|
||||||
|
chdir '..';
|
||||||
|
die if !open(REPORT, ">>$report_name");
|
||||||
|
print REPORT "==================End of utf8reader runs==================\n";
|
||||||
|
print REPORT "\n";
|
|
@ -0,0 +1,5 @@
|
||||||
|
CC = g++
|
||||||
|
CFLAGS = -g -Wall
|
||||||
|
|
||||||
|
smoketest: test.cpp ../../source/utf8.h ../../source/utf8/core.h ../../source/utf8/checked.h ../../source/utf8/unchecked.h
|
||||||
|
$(CC) $(CFLAGS) test.cpp -osmoketest
|
|
@ -0,0 +1,297 @@
|
||||||
|
#include <cstring>
|
||||||
|
#include <cassert>
|
||||||
|
#include <vector>
|
||||||
|
#include "../../source/utf8.h"
|
||||||
|
using namespace utf8;
|
||||||
|
using namespace std;
|
||||||
|
|
||||||
|
int main()
|
||||||
|
{
|
||||||
|
//append
|
||||||
|
unsigned char u[5] = {0,0,0,0,0};
|
||||||
|
|
||||||
|
append(0x0448, u);
|
||||||
|
assert (u[0] == 0xd1 && u[1] == 0x88 && u[2] == 0 && u[3] == 0 && u[4] == 0);
|
||||||
|
|
||||||
|
append(0x65e5, u);
|
||||||
|
assert (u[0] == 0xe6 && u[1] == 0x97 && u[2] == 0xa5 && u[3] == 0 && u[4] == 0);
|
||||||
|
|
||||||
|
append(0x3044, u);
|
||||||
|
assert (u[0] == 0xe3 && u[1] == 0x81 && u[2] == 0x84 && u[3] == 0 && u[4] == 0);
|
||||||
|
|
||||||
|
append(0x10346, u);
|
||||||
|
assert (u[0] == 0xf0 && u[1] == 0x90 && u[2] == 0x8d && u[3] == 0x86 && u[4] == 0);
|
||||||
|
|
||||||
|
|
||||||
|
//next
|
||||||
|
const char* twochars = "\xe6\x97\xa5\xd1\x88";
|
||||||
|
const char* w = twochars;
|
||||||
|
int cp = next(w, twochars + 6);
|
||||||
|
assert (cp == 0x65e5);
|
||||||
|
assert (w == twochars + 3);
|
||||||
|
|
||||||
|
const char* threechars = "\xf0\x90\x8d\x86\xe6\x97\xa5\xd1\x88";
|
||||||
|
w = threechars;
|
||||||
|
cp = next(w, threechars + 9);
|
||||||
|
assert (cp == 0x10346);
|
||||||
|
assert (w == threechars + 4);
|
||||||
|
cp = next(w, threechars + 9);
|
||||||
|
assert (cp == 0x65e5);
|
||||||
|
assert (w == threechars + 7);
|
||||||
|
cp = next(w, threechars + 9);
|
||||||
|
assert (cp == 0x0448);
|
||||||
|
assert (w == threechars + 9);
|
||||||
|
|
||||||
|
//peek_next
|
||||||
|
const char* const cw = twochars;
|
||||||
|
cp = peek_next(cw, cw + 6);
|
||||||
|
assert (cp == 0x65e5);
|
||||||
|
assert (cw == twochars);
|
||||||
|
|
||||||
|
//prior
|
||||||
|
w = twochars + 3;
|
||||||
|
cp = prior (w, twochars);
|
||||||
|
assert (cp == 0x65e5);
|
||||||
|
assert (w == twochars);
|
||||||
|
|
||||||
|
w = threechars + 9;
|
||||||
|
cp = prior(w, threechars);
|
||||||
|
assert (cp == 0x0448);
|
||||||
|
assert (w == threechars + 7);
|
||||||
|
cp = prior(w, threechars);
|
||||||
|
assert (cp == 0x65e5);
|
||||||
|
assert (w == threechars + 4);
|
||||||
|
cp = prior(w, threechars);
|
||||||
|
assert (cp == 0x10346);
|
||||||
|
assert (w == threechars);
|
||||||
|
|
||||||
|
//previous (deprecated)
|
||||||
|
w = twochars + 3;
|
||||||
|
cp = previous (w, twochars - 1);
|
||||||
|
assert (cp == 0x65e5);
|
||||||
|
assert (w == twochars);
|
||||||
|
|
||||||
|
w = threechars + 9;
|
||||||
|
cp = previous(w, threechars - 1);
|
||||||
|
assert (cp == 0x0448);
|
||||||
|
assert (w == threechars + 7);
|
||||||
|
cp = previous(w, threechars -1);
|
||||||
|
assert (cp == 0x65e5);
|
||||||
|
assert (w == threechars + 4);
|
||||||
|
cp = previous(w, threechars - 1);
|
||||||
|
assert (cp == 0x10346);
|
||||||
|
assert (w == threechars);
|
||||||
|
|
||||||
|
// advance
|
||||||
|
w = twochars;
|
||||||
|
advance (w, 2, twochars + 6);
|
||||||
|
assert (w == twochars + 5);
|
||||||
|
|
||||||
|
// distance
|
||||||
|
size_t dist = utf8::distance(twochars, twochars + 5);
|
||||||
|
assert (dist == 2);
|
||||||
|
|
||||||
|
// utf32to8
|
||||||
|
int utf32string[] = {0x448, 0x65E5, 0x10346, 0};
|
||||||
|
vector<char> utf8result;
|
||||||
|
utf32to8(utf32string, utf32string + 3, back_inserter(utf8result));
|
||||||
|
assert (utf8result.size() == 9);
|
||||||
|
// try it with the return value;
|
||||||
|
char* utf8_end = utf32to8(utf32string, utf32string + 3, &utf8result[0]);
|
||||||
|
assert (utf8_end == &utf8result[0] + 9);
|
||||||
|
|
||||||
|
//utf8to32
|
||||||
|
vector<int> utf32result;
|
||||||
|
utf8to32(twochars, twochars + 5, back_inserter(utf32result));
|
||||||
|
assert (utf32result.size() == 2);
|
||||||
|
// try it with the return value;
|
||||||
|
int* utf32_end = utf8to32(twochars, twochars + 5, &utf32result[0]);
|
||||||
|
assert (utf32_end == &utf32result[0] + 2);
|
||||||
|
|
||||||
|
//utf16to8
|
||||||
|
unsigned short utf16string[] = {0x41, 0x0448, 0x65e5, 0xd834, 0xdd1e};
|
||||||
|
utf8result.clear();
|
||||||
|
utf16to8(utf16string, utf16string + 5, back_inserter(utf8result));
|
||||||
|
assert (utf8result.size() == 10);
|
||||||
|
// try it with the return value;
|
||||||
|
utf8_end = utf16to8 (utf16string, utf16string + 5, &utf8result[0]);
|
||||||
|
assert (utf8_end == &utf8result[0] + 10);
|
||||||
|
|
||||||
|
//utf8to16
|
||||||
|
char utf8_with_surrogates[] = "\xe6\x97\xa5\xd1\x88\xf0\x9d\x84\x9e";
|
||||||
|
vector <unsigned short> utf16result;
|
||||||
|
utf8to16(utf8_with_surrogates, utf8_with_surrogates + 9, back_inserter(utf16result));
|
||||||
|
assert (utf16result.size() == 4);
|
||||||
|
assert (utf16result[2] == 0xd834);
|
||||||
|
assert (utf16result[3] == 0xdd1e);
|
||||||
|
// try it with the return value;
|
||||||
|
unsigned short* utf16_end = utf8to16 (utf8_with_surrogates, utf8_with_surrogates + 9, &utf16result[0]);
|
||||||
|
assert (utf16_end == &utf16result[0] + 4);
|
||||||
|
|
||||||
|
//find_invalid
|
||||||
|
char utf_invalid[] = "\xe6\x97\xa5\xd1\x88\xfa";
|
||||||
|
char* invalid = find_invalid(utf_invalid, utf_invalid + 6);
|
||||||
|
assert (invalid == utf_invalid + 5);
|
||||||
|
|
||||||
|
//is_valid
|
||||||
|
bool bvalid = is_valid(utf_invalid, utf_invalid + 6);
|
||||||
|
assert (bvalid == false);
|
||||||
|
bvalid = is_valid(utf8_with_surrogates, utf8_with_surrogates + 9);
|
||||||
|
assert (bvalid == true);
|
||||||
|
|
||||||
|
//starts_with_bom
|
||||||
|
unsigned char byte_order_mark[] = {0xef, 0xbb, 0xbf};
|
||||||
|
bool bbom = starts_with_bom(byte_order_mark, byte_order_mark + sizeof(byte_order_mark));
|
||||||
|
assert (bbom == true);
|
||||||
|
bool no_bbom = starts_with_bom(threechars, threechars + sizeof(threechars));
|
||||||
|
assert (no_bbom == false);
|
||||||
|
|
||||||
|
//is_bom
|
||||||
|
bool unsafe_bbom = is_bom(byte_order_mark);
|
||||||
|
assert (unsafe_bbom == true);
|
||||||
|
|
||||||
|
|
||||||
|
//replace_invalid
|
||||||
|
char invalid_sequence[] = "a\x80\xe0\xa0\xc0\xaf\xed\xa0\x80z";
|
||||||
|
vector<char> replace_invalid_result(50);
|
||||||
|
replace_invalid (invalid_sequence, invalid_sequence + sizeof(invalid_sequence), replace_invalid_result.begin(), '?');
|
||||||
|
bvalid = is_valid(replace_invalid_result.begin(), replace_invalid_result.end());
|
||||||
|
assert (bvalid);
|
||||||
|
const char* fixed_invalid_sequence = "a????z";
|
||||||
|
assert (std::equal(replace_invalid_result.begin(), replace_invalid_result.begin() + sizeof(fixed_invalid_sequence), fixed_invalid_sequence));
|
||||||
|
|
||||||
|
// iterator
|
||||||
|
utf8::iterator<const char*> it(threechars, threechars, threechars + 9);
|
||||||
|
utf8::iterator<const char*> it2 = it;
|
||||||
|
assert (it2 == it);
|
||||||
|
assert (*it == 0x10346);
|
||||||
|
assert (*(++it) == 0x65e5);
|
||||||
|
assert ((*it++) == 0x65e5);
|
||||||
|
assert (*it == 0x0448);
|
||||||
|
assert (it != it2);
|
||||||
|
utf8::iterator<const char*> endit (threechars + 9, threechars, threechars + 9);
|
||||||
|
assert (++it == endit);
|
||||||
|
assert (*(--it) == 0x0448);
|
||||||
|
assert ((*it--) == 0x0448);
|
||||||
|
assert (*it == 0x65e5);
|
||||||
|
assert (--it == utf8::iterator<const char*>(threechars, threechars, threechars + 9));
|
||||||
|
assert (*it == 0x10346);
|
||||||
|
|
||||||
|
//////////////////////////////////////////////////////////
|
||||||
|
//// Unchecked variants
|
||||||
|
//////////////////////////////////////////////////////////
|
||||||
|
|
||||||
|
//append
|
||||||
|
memset(u, 0, 5);
|
||||||
|
append(0x0448, u);
|
||||||
|
assert (u[0] == 0xd1 && u[1] == 0x88 && u[2] == 0 && u[3] == 0 && u[4] == 0);
|
||||||
|
|
||||||
|
append(0x65e5, u);
|
||||||
|
assert (u[0] == 0xe6 && u[1] == 0x97 && u[2] == 0xa5 && u[3] == 0 && u[4] == 0);
|
||||||
|
|
||||||
|
append(0x10346, u);
|
||||||
|
assert (u[0] == 0xf0 && u[1] == 0x90 && u[2] == 0x8d && u[3] == 0x86 && u[4] == 0);
|
||||||
|
|
||||||
|
//next
|
||||||
|
w = twochars;
|
||||||
|
cp = unchecked::next(w);
|
||||||
|
assert (cp == 0x65e5);
|
||||||
|
assert (w == twochars + 3);
|
||||||
|
|
||||||
|
w = threechars;
|
||||||
|
cp = unchecked::next(w);
|
||||||
|
assert (cp == 0x10346);
|
||||||
|
assert (w == threechars + 4);
|
||||||
|
cp = unchecked::next(w);
|
||||||
|
assert (cp == 0x65e5);
|
||||||
|
assert (w == threechars + 7);
|
||||||
|
cp = unchecked::next(w);
|
||||||
|
assert (cp == 0x0448);
|
||||||
|
assert (w == threechars + 9);
|
||||||
|
|
||||||
|
//peek_next
|
||||||
|
cp = unchecked::peek_next(cw);
|
||||||
|
assert (cp == 0x65e5);
|
||||||
|
assert (cw == twochars);
|
||||||
|
|
||||||
|
|
||||||
|
//previous (calls prior internally)
|
||||||
|
|
||||||
|
w = twochars + 3;
|
||||||
|
cp = unchecked::previous (w);
|
||||||
|
assert (cp == 0x65e5);
|
||||||
|
assert (w == twochars);
|
||||||
|
|
||||||
|
w = threechars + 9;
|
||||||
|
cp = unchecked::previous(w);
|
||||||
|
assert (cp == 0x0448);
|
||||||
|
assert (w == threechars + 7);
|
||||||
|
cp = unchecked::previous(w);
|
||||||
|
assert (cp == 0x65e5);
|
||||||
|
assert (w == threechars + 4);
|
||||||
|
cp = unchecked::previous(w);
|
||||||
|
assert (cp == 0x10346);
|
||||||
|
assert (w == threechars);
|
||||||
|
|
||||||
|
// advance
|
||||||
|
w = twochars;
|
||||||
|
unchecked::advance (w, 2);
|
||||||
|
assert (w == twochars + 5);
|
||||||
|
|
||||||
|
// distance
|
||||||
|
dist = unchecked::distance(twochars, twochars + 5);
|
||||||
|
assert (dist == 2);
|
||||||
|
|
||||||
|
// utf32to8
|
||||||
|
utf8result.clear();
|
||||||
|
unchecked::utf32to8(utf32string, utf32string + 3, back_inserter(utf8result));
|
||||||
|
assert (utf8result.size() == 9);
|
||||||
|
// try it with the return value;
|
||||||
|
utf8_end = utf32to8(utf32string, utf32string + 3, &utf8result[0]);
|
||||||
|
assert(utf8_end == &utf8result[0] + 9);
|
||||||
|
|
||||||
|
//utf8to32
|
||||||
|
utf32result.clear();
|
||||||
|
unchecked::utf8to32(twochars, twochars + 5, back_inserter(utf32result));
|
||||||
|
assert (utf32result.size() == 2);
|
||||||
|
// try it with the return value;
|
||||||
|
utf32_end = utf8to32(twochars, twochars + 5, &utf32result[0]);
|
||||||
|
assert (utf32_end == &utf32result[0] + 2);
|
||||||
|
|
||||||
|
//utf16to8
|
||||||
|
utf8result.clear();
|
||||||
|
unchecked::utf16to8(utf16string, utf16string + 5, back_inserter(utf8result));
|
||||||
|
assert (utf8result.size() == 10);
|
||||||
|
// try it with the return value;
|
||||||
|
utf8_end = utf16to8 (utf16string, utf16string + 5, &utf8result[0]);
|
||||||
|
assert (utf8_end == &utf8result[0] + 10);
|
||||||
|
|
||||||
|
//utf8to16
|
||||||
|
utf16result.clear();
|
||||||
|
unchecked::utf8to16(utf8_with_surrogates, utf8_with_surrogates + 9, back_inserter(utf16result));
|
||||||
|
assert (utf16result.size() == 4);
|
||||||
|
assert (utf16result[2] == 0xd834);
|
||||||
|
assert (utf16result[3] == 0xdd1e);
|
||||||
|
// try it with the return value;
|
||||||
|
utf16_end = utf8to16 (utf8_with_surrogates, utf8_with_surrogates + 9, &utf16result[0]);
|
||||||
|
assert (utf16_end == &utf16result[0] + 4);
|
||||||
|
|
||||||
|
// iterator
|
||||||
|
utf8::unchecked::iterator<const char*> un_it(threechars);
|
||||||
|
utf8::unchecked::iterator<const char*> un_it2 = un_it;
|
||||||
|
assert (un_it2 == un_it);
|
||||||
|
assert (*un_it == 0x10346);
|
||||||
|
assert (*(++un_it) == 0x65e5);
|
||||||
|
assert ((*un_it++) == 0x65e5);
|
||||||
|
assert (un_it != un_it2);
|
||||||
|
assert (*un_it == 0x0448);
|
||||||
|
utf8::unchecked::iterator<const char*> un_endit (threechars + 9);
|
||||||
|
assert (++un_it == un_endit);
|
||||||
|
assert (*(--un_it) == 0x0448);
|
||||||
|
assert ((*un_it--) == 0x0448);
|
||||||
|
assert (*un_it == 0x65e5);
|
||||||
|
assert (--un_it == utf8::unchecked::iterator<const char*>(threechars));
|
||||||
|
assert (*un_it == 0x10346);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,5 @@
|
||||||
|
CC = g++
|
||||||
|
CFLAGS = -g -Wall -pedantic
|
||||||
|
|
||||||
|
utf8readertest: utf8reader.cpp ../../source/utf8.h ../../source/utf8/core.h ../../source/utf8/checked.h ../../source/utf8/unchecked.h
|
||||||
|
$(CC) $(CFLAGS) utf8reader.cpp -o utf8reader
|
|
@ -0,0 +1,160 @@
|
||||||
|
#include "../../source/utf8.h"
|
||||||
|
using namespace utf8;
|
||||||
|
|
||||||
|
#include <string>
|
||||||
|
#include <iostream>
|
||||||
|
#include <fstream>
|
||||||
|
#include <vector>
|
||||||
|
using namespace std;
|
||||||
|
|
||||||
|
int main(int argc, char** argv)
|
||||||
|
{
|
||||||
|
if (argc != 2) {
|
||||||
|
cout << "\nUsage: utfreader filename\n";
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
const char* TEST_FILE_PATH = argv[1];
|
||||||
|
// Open the test file
|
||||||
|
ifstream fs8(TEST_FILE_PATH);
|
||||||
|
if (!fs8.is_open()) {
|
||||||
|
cout << "Could not open " << TEST_FILE_PATH << endl;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read it line by line
|
||||||
|
unsigned int line_count = 0;
|
||||||
|
char byte;
|
||||||
|
while (!fs8.eof()) {
|
||||||
|
string line;
|
||||||
|
while ((byte = static_cast<char>(fs8.get())) != '\n' && !fs8.eof())
|
||||||
|
line.push_back(byte);
|
||||||
|
|
||||||
|
line_count++;
|
||||||
|
// Play around with each line and convert it to utf16
|
||||||
|
string::iterator line_start = line.begin();
|
||||||
|
string::iterator line_end = line.end();
|
||||||
|
line_end = find_invalid(line_start, line_end);
|
||||||
|
if (line_end != line.end())
|
||||||
|
cout << "Line " << line_count << ": Invalid utf-8 at byte " << int(line.end() - line_end) << '\n';
|
||||||
|
|
||||||
|
// Convert it to utf-16 and write to the file
|
||||||
|
vector<unsigned short> utf16_line;
|
||||||
|
utf8to16(line_start, line_end, back_inserter(utf16_line));
|
||||||
|
|
||||||
|
// Back to utf-8 and compare it to the original line.
|
||||||
|
string back_to_utf8;
|
||||||
|
utf16to8(utf16_line.begin(), utf16_line.end(), back_inserter(back_to_utf8));
|
||||||
|
if (back_to_utf8.compare(string(line_start, line_end)) != 0)
|
||||||
|
cout << "Line " << line_count << ": Conversion to UTF-16 and back failed" << '\n';
|
||||||
|
|
||||||
|
// Now, convert it to utf-32, back to utf-8 and compare
|
||||||
|
vector <unsigned> utf32_line;
|
||||||
|
utf8to32(line_start, line_end, back_inserter(utf32_line));
|
||||||
|
back_to_utf8.clear();
|
||||||
|
utf32to8(utf32_line.begin(), utf32_line.end(), back_inserter(back_to_utf8));
|
||||||
|
if (back_to_utf8.compare(string(line_start, line_end)) != 0)
|
||||||
|
cout << "Line " << line_count << ": Conversion to UTF-32 and back failed" << '\n';
|
||||||
|
|
||||||
|
// Now, iterate and back
|
||||||
|
unsigned char_count = 0;
|
||||||
|
string::iterator it = line_start;
|
||||||
|
while (it != line_end) {
|
||||||
|
unsigned int next_cp = peek_next(it, line_end);
|
||||||
|
if (next(it, line_end) != next_cp)
|
||||||
|
cout << "Line " << line_count << ": Error: peek_next gave a different result than next" << '\n';
|
||||||
|
char_count++;
|
||||||
|
}
|
||||||
|
if (char_count != utf32_line.size())
|
||||||
|
cout << "Line " << line_count << ": Error in iterating with next - wrong number of characters" << '\n';
|
||||||
|
|
||||||
|
string::iterator adv_it = line_start;
|
||||||
|
utf8::advance(adv_it, char_count, line_end);
|
||||||
|
if (adv_it != line_end)
|
||||||
|
cout << "Line " << line_count << ": Error in advance function" << '\n';
|
||||||
|
|
||||||
|
if (string::size_type(utf8::distance(line_start, line_end)) != char_count)
|
||||||
|
cout << "Line " << line_count << ": Error in distance function" << '\n';
|
||||||
|
|
||||||
|
while (it != line_start) {
|
||||||
|
previous(it, line.rend().base());
|
||||||
|
char_count--;
|
||||||
|
}
|
||||||
|
if (char_count != 0)
|
||||||
|
cout << "Line " << line_count << ": Error in iterating with previous - wrong number of characters" << '\n';
|
||||||
|
|
||||||
|
// Try utf8::iterator
|
||||||
|
utf8::iterator<string::iterator> u8it(line_start, line_start, line_end);
|
||||||
|
if (!utf32_line.empty() && *u8it != utf32_line.at(0))
|
||||||
|
cout << "Line " << line_count << ": Error in utf::iterator * operator" << '\n';
|
||||||
|
if (std::distance(u8it, utf8::iterator<string::iterator>(line_end, line_start, line_end)) != static_cast<int>(utf32_line.size()))
|
||||||
|
cout << "Line " << line_count << ": Error in using utf::iterator with std::distance - wrong number of characters" << '\n';
|
||||||
|
|
||||||
|
std::advance(u8it, utf32_line.size());
|
||||||
|
if (u8it != utf8::iterator<string::iterator>(line_end, line_start, line_end))
|
||||||
|
cout << "Line " << line_count << ": Error in using utf::iterator with std::advance" << '\n';
|
||||||
|
|
||||||
|
|
||||||
|
//======================== Now, the unchecked versions ======================
|
||||||
|
// Convert it to utf-16 and compare to the checked version
|
||||||
|
vector<unsigned short> utf16_line_unchecked;
|
||||||
|
unchecked::utf8to16(line_start, line_end, back_inserter(utf16_line_unchecked));
|
||||||
|
|
||||||
|
if (utf16_line != utf16_line_unchecked)
|
||||||
|
cout << "Line " << line_count << ": Error in unchecked::utf8to16" << '\n';
|
||||||
|
|
||||||
|
// Back to utf-8 and compare it to the original line.
|
||||||
|
back_to_utf8.clear();
|
||||||
|
unchecked::utf16to8(utf16_line_unchecked.begin(), utf16_line_unchecked.end(), back_inserter(back_to_utf8));
|
||||||
|
if (back_to_utf8.compare(string(line_start, line_end)) != 0)
|
||||||
|
cout << "Line " << line_count << ": Unchecked conversion to UTF-16 and back failed" << '\n';
|
||||||
|
|
||||||
|
// Now, convert it to utf-32, back to utf-8 and compare
|
||||||
|
vector <unsigned> utf32_line_unchecked;
|
||||||
|
unchecked::utf8to32(line_start, line_end, back_inserter(utf32_line_unchecked));
|
||||||
|
if (utf32_line != utf32_line_unchecked)
|
||||||
|
cout << "Line " << line_count << ": Error in unchecked::utf8to32" << '\n';
|
||||||
|
|
||||||
|
back_to_utf8.clear();
|
||||||
|
unchecked::utf32to8(utf32_line.begin(), utf32_line.end(), back_inserter(back_to_utf8));
|
||||||
|
if (back_to_utf8.compare(string(line_start, line_end)) != 0)
|
||||||
|
cout << "Line " << line_count << ": Unchecked conversion to UTF-32 and back failed" << '\n';
|
||||||
|
|
||||||
|
// Now, iterate and back
|
||||||
|
char_count = 0;
|
||||||
|
it = line_start;
|
||||||
|
while (it != line_end) {
|
||||||
|
unsigned int next_cp = unchecked::peek_next(it);
|
||||||
|
if (unchecked::next(it) != next_cp)
|
||||||
|
cout << "Line " << line_count << ": Error: unchecked::peek_next gave a different result than unchecked::next" << '\n';;
|
||||||
|
char_count++;
|
||||||
|
}
|
||||||
|
if (char_count != utf32_line.size())
|
||||||
|
cout << "Line " << line_count << ": Error in iterating with unchecked::next - wrong number of characters" << '\n';
|
||||||
|
|
||||||
|
adv_it = line_start;
|
||||||
|
utf8::unchecked::advance(adv_it, char_count);
|
||||||
|
if (adv_it != line_end)
|
||||||
|
cout << "Line " << line_count << ": Error in unchecked::advance function" << '\n';
|
||||||
|
|
||||||
|
if (string::size_type(utf8::unchecked::distance(line_start, line_end)) != char_count)
|
||||||
|
cout << "Line " << line_count << ": Error in unchecked::distance function" << '\n';
|
||||||
|
|
||||||
|
while (it != line_start) {
|
||||||
|
unchecked::previous(it);
|
||||||
|
char_count--;
|
||||||
|
}
|
||||||
|
if (char_count != 0)
|
||||||
|
cout << "Line " << line_count << ": Error in iterating with unchecked::previous - wrong number of characters" << '\n';
|
||||||
|
|
||||||
|
// Try utf8::unchecked::iterator
|
||||||
|
utf8::unchecked::iterator<string::iterator> un_u8it(line_start);
|
||||||
|
if (!utf32_line.empty() && *un_u8it != utf32_line.at(0))
|
||||||
|
cout << "Line " << line_count << ": Error in utf::unchecked::iterator * operator" << '\n';
|
||||||
|
if (std::distance(un_u8it, utf8::unchecked::iterator<string::iterator>(line_end)) != static_cast<int>(utf32_line.size()))
|
||||||
|
cout << "Line " << line_count << ": Error in using utf::unchecked::iterator with std::distance - wrong number of characters" << '\n';
|
||||||
|
|
||||||
|
std::advance(un_u8it, utf32_line.size());
|
||||||
|
if (un_u8it != utf8::unchecked::iterator<string::iterator>(line_end))
|
||||||
|
cout << "Line " << line_count << ": Error in using utf::unchecked::iterator with std::advance" << '\n';
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in New Issue