C++11中的UTF转换函数
UTF conversion functions in C++11
我正在寻找一组函数，用于在C++11中执行UTF字符转换。它应该包括到utf8、utf16和utf32中任意一个的转换。识别字节顺序标记的功能也会很有帮助。
Update：此处列出的函数在GitHub repo、.hpp、.cpp和测试中进行维护。某些UTF-16功能已被禁用，因为它们无法正常工作。utf.test.cpp文件中的"香蕉"测试证明了这个问题。
还包括一个用于识别字节顺序标记的"read_with_bom"函数。
#if _MSC_VER == 1900 //work around for bug in MS Visual C++ 2015 https://social.msdn.microsoft.com/Forums/en-US/8f40dcd8-c67f-4eba-9134-a19b9178e481/vs-2015-rc-linker-stdcodecvt-error?forum=vcgeneral
std::string to_utf8(const std::u16string &s)
{
std::wstring_convert<std::codecvt_utf8<int16_t>, int16_t> convert;
auto p = reinterpret_cast<const int16_t *>(s.data());
return convert.to_bytes(p, p + s.size());
}
std::string to_utf8(const std::u32string &s)
{
std::wstring_convert<std::codecvt_utf8<int32_t>, int32_t> convert;
auto p = reinterpret_cast<const int32_t *>(s.data());
return convert.to_bytes(p, p + s.size());
}
std::u16string to_utf16(const std::string &s)
{
std::wstring_convert<std::codecvt_utf8<int16_t>, int16_t> convert;
auto asInt = convert.from_bytes(s);
return std::u16string(reinterpret_cast<char16_t const *>(asInt.data()), asInt.length());
}
std::u32string to_utf32(const std::string &s)
{
std::wstring_convert<std::codecvt_utf8<int32_t>, int32_t> convert;
auto asInt = convert.from_bytes(s);
return std::u32string(reinterpret_cast<char32_t const *>(asInt.data()), asInt.length());
}
#else
std::string to_utf8(const std::u16string &s)
{
std::wstring_convert<std::codecvt_utf8<char16_t>, char16_t> conv;
return conv.to_bytes(s);
}
std::string to_utf8(const std::u32string &s)
{
std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> conv;
return conv.to_bytes(s);
}
std::u16string to_utf16(const std::string &s)
{
std::wstring_convert<std::codecvt_utf8<char16_t>, char16_t> convert;
return convert.from_bytes(s);
}
std::u32string to_utf32(const std::string &s)
{
std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> conv;
return conv.from_bytes(s);
}
#endif
std::u16string to_utf16(const std::u32string &s)
{
return to_utf16(to_utf8(s));
}
std::u32string to_utf32(const std::u16string &s) {
return to_utf32(to_utf8(s));
}
std::u32string read_with_bom(std::istream & src)
{
enum encoding {
encoding_utf32be = 0,
encoding_utf32le,
encoding_utf16be,
encoding_utf16le,
encoding_utf8,
encoding_ascii,
};
std::vector<std::string> boms = {
std::string("x00x00xFExFF", 4),
std::string("xFFxFEx00x00", 4),
std::string("xFExFF", 2),
std::string("xFFxFE", 2),
std::string("xEFxBBxBF", 3)
};
std::string buffer((std::istreambuf_iterator<char>(src)), std::istreambuf_iterator<char>());
encoding enc = encoding_ascii;
for (unsigned int i = 0; i < boms.size(); ++i) {
std::string testBom = boms[i];
if (buffer.compare(0, testBom.length(), testBom) == 0) {
enc = encoding(i);
buffer = buffer.substr(testBom.length());
break;
}
}
switch (enc) {
case encoding_utf32be:
{
if (buffer.length() % 4 != 0) {
throw std::logic_error("size in bytes must be a multiple of 4");
}
int count = buffer.length() / 4;
std::u32string temp = std::u32string(count, 0);
for (int i = 0; i < count; ++i) {
temp[i] = static_cast<char32_t>(buffer[i * 4 + 3] << 0 | buffer[i * 4 + 2] << 8 | buffer[i * 4 + 1] << 16 | buffer[i * 4 + 0] << 24);
}
return temp;
}
case encoding_utf32le:
{
if (buffer.length() % 4 != 0) {
throw std::logic_error("size in bytes must be a multiple of 4");
}
int count = buffer.length() / 4;
std::u32string temp = std::u32string(count, 0);
for (int i = 0; i < count; ++i) {
temp[i] = static_cast<char32_t>(buffer[i * 4 + 0] << 0 | buffer[i * 4 + 1] << 8 | buffer[i * 4 + 2] << 16 | buffer[i * 4 + 3] << 24);
}
return temp;
}
case encoding_utf16be:
{
if (buffer.length() % 2 != 0) {
throw std::logic_error("size in bytes must be a multiple of 2");
}
int count = buffer.length() / 2;
std::u16string temp = std::u16string(count, 0);
for (int i = 0; i < count; ++i) {
temp[i] = static_cast<char16_t>(buffer[i * 2 + 1] << 0 | buffer[i * 2 + 0] << 8);
}
return to_utf32(temp);
}
case encoding_utf16le:
{
if (buffer.length() % 2 != 0) {
throw std::logic_error("size in bytes must be a multiple of 2");
}
int count = buffer.length() / 2;
std::u16string temp = std::u16string(count, 0);
for (int i = 0; i < count; ++i) {
temp[i] = static_cast<char16_t>(buffer[i * 2 + 0] << 0 | buffer[i * 2 + 1] << 8);
}
return to_utf32(temp);
}
default:
return to_utf32(buffer);
}
}
我已经为此编写了一个小的utf_ranges库。它使用Range-V3和C++14。
它既有视图也有操作（如果您熟悉Range-V3术语），用于在三种主要UTF编码中的任何一种之间进行转换，可以使用和生成字节顺序标记，并基于bom执行endian转换。例如，将一个文件从未知的端号UTF-16读取为UTF-8 std::string，将七个unicode行结尾中的任何一个转换为n，如下所示：
std::ifstream source{path, std::ios::binary};
std::string str = utf::istreambuf<char16_t>(source)
| utf::view::consume_bom
| utf::view::utf8
| utf::view::line_end_convert;
这是我的Baby X的UTF-8代码(https://github.com/MalcolmMcLean/babyx)
static const unsigned int offsetsFromUTF8[6] =
{
0x00000000UL, 0x00003080UL, 0x000E2080UL,
0x03C82080UL, 0xFA082080UL, 0x82082080UL
};
static const unsigned char trailingBytesForUTF8[256] = {
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5
};
int bbx_isutf8z(const char *str)
{
int len = 0;
int pos = 0;
int nb;
int i;
int ch;
while(str[len])
len++;
while(pos < len && *str)
{
nb = bbx_utf8_skip(str);
if(nb < 1 || nb > 4)
return 0;
if(pos + nb > len)
return 0;
for(i=1;i<nb;i++)
if( (str[i] & 0xC0) != 0x80 )
return 0;
ch = bbx_utf8_getch(str);
if(ch < 0x80)
{
if(nb != 1)
return 0;
}
else if(ch < 0x8000)
{
if(nb != 2)
return 0;
}
else if(ch < 0x10000)
{
if(nb != 3)
return 0;
}
else if(ch < 0x110000)
{
if(nb != 4)
return 0;
}
pos += nb;
str += nb;
}
return 1;
}
int bbx_utf8_skip(const char *utf8)
{
return trailingBytesForUTF8[(unsigned char) *utf8] + 1;
}
int bbx_utf8_getch(const char *utf8)
{
int ch;
int nb;
nb = trailingBytesForUTF8[(unsigned char)*utf8];
ch = 0;
switch (nb)
{
/* these fall through deliberately */
case 3: ch += (unsigned char)*utf8++; ch <<= 6;
case 2: ch += (unsigned char)*utf8++; ch <<= 6;
case 1: ch += (unsigned char)*utf8++; ch <<= 6;
case 0: ch += (unsigned char)*utf8++;
}
ch -= offsetsFromUTF8[nb];
return ch;
}
int bbx_utf8_putch(char *out, int ch)
{
char *dest = out;
if (ch < 0x80)
{
*dest++ = (char)ch;
}
else if (ch < 0x800)
{
*dest++ = (ch>>6) | 0xC0;
*dest++ = (ch & 0x3F) | 0x80;
}
else if (ch < 0x10000)
{
*dest++ = (ch>>12) | 0xE0;
*dest++ = ((ch>>6) & 0x3F) | 0x80;
*dest++ = (ch & 0x3F) | 0x80;
}
else if (ch < 0x110000)
{
*dest++ = (ch>>18) | 0xF0;
*dest++ = ((ch>>12) & 0x3F) | 0x80;
*dest++ = ((ch>>6) & 0x3F) | 0x80;
*dest++ = (ch & 0x3F) | 0x80;
}
else
return 0;
return dest - out;
}
int bbx_utf8_charwidth(int ch)
{
if (ch < 0x80)
{
return 1;
}
else if (ch < 0x800)
{
return 2;
}
else if (ch < 0x10000)
{
return 3;
}
else if (ch < 0x110000)
{
return 4;
}
else
return 0;
}
int bbx_utf8_Nchars(const char *utf8)
{
int answer = 0;
while(*utf8)
{
utf8 += bbx_utf8_skip(utf8);
answer++;
}
return answer;
}
相关文章：
- 如何使用Rcpp将R函数转换为C++函数
- C++函数转换为 C# 函数
- 将 C 函数转换为 C++ 以检查数字是否有效
- C++:从重载函数转换为 std::function
- 将 C 函数转换为C++语言
- C++隐式构造函数转换，后跟类型向上转换
- 将 lambda 函数转换为具有混合 lambda 引入器和参数列表的函子结构
- 通过 Boost Python 将 Python 函数转换为 C++，用作回调
- 如何将 MATLAB 图像处理库内置函数转换为 MATLAB 编码器代码生成不支持的 C++?
- 如何将函数转换为 lambda 函数
- 自定义函数转换错误?
- 将 C# 哈希函数转换为C++
- 将 lambda 函数转换为另一个编译单元中的普通函数会缩短编译时间吗?
- 加快 R 性能或将 R 函数转换为C++函数
- 将成员函数转换为指向成员函数的指针
- 将MATLAB炒作函数转换为C
- Rcpp - 用二进制函数转换数字向量?
- C lambda函数转换误差
- 此语法中的构造函数转换错误
- 将函数转换为find_if lambda
